[{"id":1760646,"web_url":"http://patchwork.ozlabs.org/comment/1760646/","msgid":"<20170830.221508.1057740864304760206.davem@davemloft.net>","list_archive_url":null,"date":"2017-08-31T05:15:08","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":15,"url":"http://patchwork.ozlabs.org/api/people/15/","name":"David Miller","email":"davem@davemloft.net"},"content":"From: Saeed Mahameed \nDate: Thu, 31 Aug 2017 02:04:06 +0300\n\n> The following changes provide GRE tunnel offloads for mlx5 ethernet netdevice driver.\n> \n> For more details please see tag log message below.\n> Please pull and let me know if there's any problem.\n> \n> Note: this series doesn't conflict with the ongoing net mlx5 submission.\n\nLooks good, pulled.","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":"ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xjVtw4cFSz9s4q\n\tfor ;\n\tThu, 31 Aug 2017 15:15:20 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1750883AbdHaFPK (ORCPT );\n\tThu, 31 Aug 2017 01:15:10 -0400","from shards.monkeyblade.net ([184.105.139.130]:42012 \"EHLO\n\tshards.monkeyblade.net\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1750826AbdHaFPJ (ORCPT\n\t); Thu, 31 Aug 2017 01:15:09 -0400","from localhost (74-93-104-98-Washington.hfc.comcastbusiness.net\n\t[74.93.104.98]) (using TLSv1 with cipher AES256-SHA (256/256 bits))\n\t(Client did not present a certificate)\n\t(Authenticated sender: davem-davemloft)\n\tby shards.monkeyblade.net (Postfix) with ESMTPSA id 3DF5213D498E1;\n\tWed, 30 Aug 2017 22:15:09 -0700 (PDT)"],"Date":"Wed, 30 Aug 2017 22:15:08 -0700 (PDT)","Message-Id":"<20170830.221508.1057740864304760206.davem@davemloft.net>","To":"saeedm@mellanox.com","Cc":"netdev@vger.kernel.org","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","From":"David Miller ","In-Reply-To":"<20170830230409.15176-1-saeedm@mellanox.com>","References":"<20170830230409.15176-1-saeedm@mellanox.com>","X-Mailer":"Mew version 6.7 on Emacs 25.2 / Mule 6.0 (HANACHIRUSATO)","Mime-Version":"1.0","Content-Type":"Text/Plain; charset=us-ascii","Content-Transfer-Encoding":"7bit","X-Greylist":"Sender succeeded SMTP AUTH, not delayed by\n\tmilter-greylist-4.5.12 (shards.monkeyblade.net\n\t[149.20.54.216]); Wed, 30 Aug 2017 22:15:09 -0700 (PDT)","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1761018,"web_url":"http://patchwork.ozlabs.org/comment/1761018/","msgid":"<87fuc7ong0.fsf@stressinduktion.org>","list_archive_url":null,"date":"2017-08-31T13:51:11","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":18284,"url":"http://patchwork.ozlabs.org/api/people/18284/","name":"Hannes Frederic Sowa","email":"hannes@stressinduktion.org"},"content":"Saeed Mahameed writes:\n\n> The first patch from Gal and Ariel provides the mlx5 driver support for\n> ConnectX capability to perform IP version identification and matching in\n> order to distinguish between IPv4 and IPv6 without the need to specify the\n> encapsulation type, thus perform RSS in MPLS automatically without\n> specifying MPLS ethertyoe. This patch will also serve for inner GRE IPv4/6\n> classification for inner GRE RSS.\n\nI don't think this is legal at all or did I misunderstood something?\n\n\n\nThanks,\nHannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=stressinduktion.org\n\theader.i=@stressinduktion.org header.b=\"eGtyrba5\"; \n\tdkim=pass (2048-bit key;\n\tunprotected) header.d=messagingengine.com\n\theader.i=@messagingengine.com header.b=\"qsNKjaMt\"; \n\tdkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xjkLN0RGQz9sPt\n\tfor ;\n\tThu, 31 Aug 2017 23:51:24 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1751671AbdHaNvW (ORCPT );\n\tThu, 31 Aug 2017 09:51:22 -0400","from out1-smtp.messagingengine.com ([66.111.4.25]:59921 \"EHLO\n\tout1-smtp.messagingengine.com\" rhost-flags-OK-OK-OK-OK)\n\tby vger.kernel.org with ESMTP id S1751034AbdHaNvV (ORCPT\n\t); Thu, 31 Aug 2017 09:51:21 -0400","from compute7.internal (compute7.nyi.internal [10.202.2.47])\n\tby mailout.nyi.internal (Postfix) with ESMTP id 1B4BA20D4E;\n\tThu, 31 Aug 2017 09:51:20 -0400 (EDT)","from frontend2 ([10.202.2.161])\n\tby compute7.internal (MEProxy); Thu, 31 Aug 2017 09:51:20 -0400","from z.localhost.stressinduktion.org (unknown [217.192.177.51])\n\tby mail.messagingengine.com (Postfix) with ESMTPA id CE65A2492F;\n\tThu, 31 Aug 2017 09:51:18 -0400 (EDT)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tstressinduktion.org; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=mrLaZdguVeMrV/YYwQ\n\t5bB320jmGv3qaWsEUBVvAh0bQ=; b=eGtyrba59xfdGyWFaOPSpZ8xx9yhHVQAqB\n\tpAfhXNI+CFB20s0FGznC6/jpacUORomIut7uvZTvtnqjZcX7sEf+c/2r+XgVVnQt\n\tZhRITmH2Kkl90Rhtv8wU/kuD7Wt75Fh4yDbpT2WbMJoPgVJIXx1lh04sc8QTsLPV\n\tqKZyt7Wsv7m97p+jROw831VIXZYeVPPqr3d4jPtB3Tel2W9+S+BNuLLFTkEsWdOF\n\tV393K+XRBQhA1FsfYNH2zqU/7jKVUPo8h7ilAGCKpmZogOtES8yG3bTfOR9SSlFE\n\tCIFfWhTo6R7APY16Kqh0QZl//bURkp12fMvcn+x5N4HgxdHF5b4Q==","v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tmessagingengine.com; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=mrLaZdguVeMrV/YYwQ\n\t5bB320jmGv3qaWsEUBVvAh0bQ=; b=qsNKjaMtbsv66HcgzHZjBwayl8umSAon7T\n\tDMDpmoETcnqYWevcDAsnx+GB+ErMvx2wQ04OuaxBb+JNE6rkUIEbDzzieUat+EZS\n\tx2ZjRBXZGuw0O/lsoGOX1bK97SfXMYj0uD9EZTHn99a4L4biafMC4Exhgmd2Os1g\n\tGBujLKJRL2mbJWS1vEkx/+HPjtWQYIpZoPo5uM0IUNNv/gyEnJSTPbDmGw/amtYw\n\tc3+N81w+tJ7CbkRK9MFgYwP1nh3ceRVAX0GGmuz7sNsHL19gXn/Rf28wr/U3VRwa\n\tMfrU6cR5iDqPPOHVWhr5C4uZAqP5EgGUI0X/ni1QWVhlm+K3fqjw=="],"X-ME-Sender":"","X-Sasl-enc":"PCTlhDBLAFs4nj2K5JNQ9WoAqEJMcj5b/dFAeyHfu9YE 1504187479","From":"Hannes Frederic Sowa ","To":"Saeed Mahameed ","Cc":"\"David S. Miller\" , netdev@vger.kernel.org","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","References":"<20170830230409.15176-1-saeedm@mellanox.com>","Date":"Thu, 31 Aug 2017 15:51:11 +0200","In-Reply-To":"<20170830230409.15176-1-saeedm@mellanox.com> (Saeed Mahameed's\n\tmessage of \"Thu, 31 Aug 2017 02:04:06 +0300\")","Message-ID":"<87fuc7ong0.fsf@stressinduktion.org>","User-Agent":"Gnus/5.13 (Gnus v5.13) Emacs/25.2 (gnu/linux)","MIME-Version":"1.0","Content-Type":"text/plain","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762205,"web_url":"http://patchwork.ozlabs.org/comment/1762205/","msgid":"","list_archive_url":null,"date":"2017-09-02T23:01:29","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65131,"url":"http://patchwork.ozlabs.org/api/people/65131/","name":"Saeed Mahameed","email":"saeedm@dev.mellanox.co.il"},"content":"On Thu, Aug 31, 2017 at 6:51 AM, Hannes Frederic Sowa\n wrote:\n> Saeed Mahameed writes:\n>\n>> The first patch from Gal and Ariel provides the mlx5 driver support for\n>> ConnectX capability to perform IP version identification and matching in\n>> order to distinguish between IPv4 and IPv6 without the need to specify the\n>> encapsulation type, thus perform RSS in MPLS automatically without\n>> specifying MPLS ethertyoe. This patch will also serve for inner GRE IPv4/6\n>> classification for inner GRE RSS.\n>\n> I don't think this is legal at all or did I misunderstood something?\n>\n> \n\nIt seems you misunderstood the cover letter. The HW will still\nidentify MPLS (IPv4/IPv6) packets using a new bit we specify in the HW\nsteering rules rather than adding new specific rules with {MPLS\nethertype} X {IPv4,IPv6} to classify MPLS IPv{4,6} traffic, Same\nfunctionality a better and general way to approach it.\nBottom line the hardware is capable of processing MPLS headers and\nperform RSS on the inner packet (IPv4/6) without the need of the\ndriver to provide precise steering MPLS rules.\n\n>\n> Thanks,\n> Hannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=dev-mellanox-co-il.20150623.gappssmtp.com\n\theader.i=@dev-mellanox-co-il.20150623.gappssmtp.com\n\theader.b=\"b0cFtOTx\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xlBSj0NT7z9sPs\n\tfor ;\n\tSun, 3 Sep 2017 09:01:57 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1752813AbdIBXBw (ORCPT );\n\tSat, 2 Sep 2017 19:01:52 -0400","from mail-lf0-f43.google.com ([209.85.215.43]:35204 \"EHLO\n\tmail-lf0-f43.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1752761AbdIBXBv (ORCPT\n\t); Sat, 2 Sep 2017 19:01:51 -0400","by mail-lf0-f43.google.com with SMTP id g18so10239560lfl.2\n\tfor ; Sat, 02 Sep 2017 16:01:51 -0700 (PDT)","by 10.25.166.17 with HTTP; Sat, 2 Sep 2017 16:01:29 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=dev-mellanox-co-il.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=nZ8A2hRS/kc9IYCRYjqcb6JNyJAlHU+5s9FGeWamBG4=;\n\tb=b0cFtOTx9fFD0DfaLL9aNqrEut6WKafbxcrpwVR6CmOJ6n0SFdJqRgsxZlKkM5VnAw\n\tjCFq7D4pgSSK7CtzTb4KbN1L8yzn/xq3mkdrQHhScqdGkLz47a77hJFPLBZQzsn6f9r9\n\tUsCPIopVMjogTIpnVum6rQ47D1414K8A/QhAhWKQb7h8do7MbK6UkHMyFn9PZagXimCn\n\tfyXCFEM0ApTdqUh7ERHo8amD9i6rnZt22d6xsV2wZ4/ElVEilU15NSpyU5EMxCua0lrr\n\trSLf7uZyTBbHjpftyeDb5+cM76d+mfbYy8zMDZV1fe8Db41HRfG/4NrfuuKOmHRyO52L\n\tWTbg==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=nZ8A2hRS/kc9IYCRYjqcb6JNyJAlHU+5s9FGeWamBG4=;\n\tb=G3on3a8YwRbTL2Y9igpTf1e7diY0v2BCI57srLH+s17Q7Dh3l7YUyU7Kky5B0sHE4B\n\tTwNuuclmushSeRNU6rpwH1S/AAqUL1MAZ3VAiIr1kq1j1VRACDYJIu0ejxU6vB8R6W7G\n\tMdiWEErMY9zEUlJyfgVjoq8QQrLNL2R+IUPf47PtzA86u4123QCyB8F5ZdXJ/YdF8A3K\n\tugjgMpfSuz1uWP2LY5EzUDV+Vt2zGa/YW3W4scf3DnQ5pcQ3w/+6WJIoXybjp40DvOsY\n\tE1vHFWEozY31/ZiJpNJe2TbNj+sPtYFKxepiyRQDtz/GMw6vihKP1zOoTEjgUj3Y9bxB\n\tw2SA==","X-Gm-Message-State":"AHPjjUga8yk5tCSxaG33aylFVXoUXfonm4Iv9ZbHDsykZZ2gMdXWa+Wo\n\tB++1lDu0Ht4VoRnoAPRHi3tRYQvRmwSy","X-Google-Smtp-Source":"ADKCNb5+o/bQlqYGUt0k0WDglUuBv3ETVwavDku2xqu5jAecA9YqC3f7njFsvJfSCYnPvKOlX9eF1n06TqC9W2sxXlM=","X-Received":"by 10.25.115.11 with SMTP id o11mr1939351lfc.170.1504393310207; \n\tSat, 02 Sep 2017 16:01:50 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"<87fuc7ong0.fsf@stressinduktion.org>","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>","From":"Saeed Mahameed ","Date":"Sat, 2 Sep 2017 16:01:29 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Hannes Frederic Sowa ","Cc":"Saeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762209,"web_url":"http://patchwork.ozlabs.org/comment/1762209/","msgid":"<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>","list_archive_url":null,"date":"2017-09-03T01:32:29","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":18284,"url":"http://patchwork.ozlabs.org/api/people/18284/","name":"Hannes Frederic Sowa","email":"hannes@stressinduktion.org"},"content":"Hi Saeed,\n\nOn Sun, Sep 3, 2017, at 01:01, Saeed Mahameed wrote:\n> On Thu, Aug 31, 2017 at 6:51 AM, Hannes Frederic Sowa\n> wrote:\n> > Saeed Mahameed writes:\n> >\n> >> The first patch from Gal and Ariel provides the mlx5 driver support for\n> >> ConnectX capability to perform IP version identification and matching in\n> >> order to distinguish between IPv4 and IPv6 without the need to specify the\n> >> encapsulation type, thus perform RSS in MPLS automatically without\n> >> specifying MPLS ethertyoe. This patch will also serve for inner GRE IPv4/6\n> >> classification for inner GRE RSS.\n> >\n> > I don't think this is legal at all or did I misunderstood something?\n> >\n> > \n> \n> It seems you misunderstood the cover letter. The HW will still\n> identify MPLS (IPv4/IPv6) packets using a new bit we specify in the HW\n> steering rules rather than adding new specific rules with {MPLS\n> ethertype} X {IPv4,IPv6} to classify MPLS IPv{4,6} traffic, Same\n> functionality a better and general way to approach it.\n> Bottom line the hardware is capable of processing MPLS headers and\n> perform RSS on the inner packet (IPv4/6) without the need of the\n> driver to provide precise steering MPLS rules.\n\nSorry, I think I am still confused.\n\nI just want to make sure that you don't use the first nibble after the\nmpls bottom of stack label in any way as an indicator if that is an IPv4\nor IPv6 packet by default. It can be anything. The forward equivalence\nclass tells the stack which protocol you see.\n\nIf you match on the first nibble behind the MPLS bottom of stack label\nthe '4' or '6' respectively could be part of a MAC address with its\nfirst nibble being 4 or 6, because the particular pseudowire is EoMPLS\nand uses no control world.\n\nI wanted to mention it, because with addition of e.g. VPLS this could\ncause problems down the road and should at least be controllable? It is\nprobably better to use Entropy Labels in future.\n\nThanks,\nHannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=stressinduktion.org\n\theader.i=@stressinduktion.org header.b=\"ZpcxPzsu\"; \n\tdkim=pass (2048-bit key;\n\tunprotected) header.d=messagingengine.com\n\theader.i=@messagingengine.com header.b=\"fugsou9a\"; \n\tdkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xlFpW5fq7z9s8J\n\tfor ;\n\tSun, 3 Sep 2017 11:32:35 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1752819AbdICBcb (ORCPT );\n\tSat, 2 Sep 2017 21:32:31 -0400","from out1-smtp.messagingengine.com ([66.111.4.25]:38151 \"EHLO\n\tout1-smtp.messagingengine.com\" rhost-flags-OK-OK-OK-OK)\n\tby vger.kernel.org with ESMTP id S1752806AbdICBc3 (ORCPT\n\t); Sat, 2 Sep 2017 21:32:29 -0400","from compute7.internal (compute7.nyi.internal [10.202.2.47])\n\tby mailout.nyi.internal (Postfix) with ESMTP id 540EB20D5D;\n\tSat, 2 Sep 2017 21:32:29 -0400 (EDT)","from web3 ([10.202.2.213])\n\tby compute7.internal (MEProxy); Sat, 02 Sep 2017 21:32:29 -0400","by mailuser.nyi.internal (Postfix, from userid 99)\n\tid 1B2F39EAD8; Sat, 2 Sep 2017 21:32:29 -0400 (EDT)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tstressinduktion.org; h=cc:content-transfer-encoding:content-type\n\t:date:from:in-reply-to:message-id:mime-version:references\n\t:subject:to:x-me-sender:x-me-sender:x-sasl-enc; s=fm1; bh=yYO+jv\n\txdU83bcleEYVddKD+rDA8PRqi4fzenyVVDJkU=; b=ZpcxPzsumSxRla1VguZYmH\n\tKicTaoWatXyPgB8MTYtlledlkcanAT1iVbTix2EGaDCrBnIVsGUmYAOOZnnvuOOZ\n\thl/Qgep8A4MvK6AtMY7XWt3OSyRqk7bychQJVq1rkN8cx4ZJDZkvSNsfp1vabWpN\n\tTkVPhuq1efc7ZeNAt3nEmQ2/Wv7LihOyRmmUbO4ovOcwMHO+915IaZbsxM/U6nni\n\tMbiglY+FtRvdzt7hmwTX6JMcHsxNGh6hGBGuAsZChXCrSrSI1Ddge4JDc3K/ZfEU\n\tztB4JMXyjBIdGKsjuQ7bTkVE9wQH0W2MX+/wFbYAUla6lCGqskrCX6DBxUPkS1Kg\n\t==","v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tmessagingengine.com; h=cc:content-transfer-encoding:content-type\n\t:date:from:in-reply-to:message-id:mime-version:references\n\t:subject:to:x-me-sender:x-me-sender:x-sasl-enc; s=fm1; bh=yYO+jv\n\txdU83bcleEYVddKD+rDA8PRqi4fzenyVVDJkU=; b=fugsou9a9aL5e1P8lyGmN2\n\tohrcmxvlyUTYVpESheusVmMtwlkYOqT2dOxRwBFsv/60m02dkmDxo0YNy/sjqwp1\n\telRApcqDFCGqWqkhhi7Htw9SjnKgTG6JWicEbB/nXYUpYJa0uo7s/g44WLpbCvS0\n\tQGZlnbNjqfQ+Cdn8K7CE6w2l5yyvzVuXDz/KKFjMis1NKpOrO4DMaNidYmNJnmnx\n\t5eGBAD927km/74P0qyDz1W7RJUhgrhHq231/X4hrvKpJapuiU+Kxdjw+RE1EZi9P\n\tV/8CmbObb+dyxoCXu4AI0bhc3KLm3mQtgg9YusXNvMeUfCmo6h+hb+xipl++NT1w\n\t=="],"X-ME-Sender":"","Message-Id":"<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>","From":"Hannes Frederic Sowa ","To":"Saeed Mahameed ","Cc":"Saeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","MIME-Version":"1.0","Content-Transfer-Encoding":"7bit","Content-Type":"text/plain; charset=\"utf-8\"","X-Mailer":"MessagingEngine.com Webmail Interface - ajax-13b5a8c9","In-Reply-To":"","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","Date":"Sun, 03 Sep 2017 03:32:29 +0200","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762210,"web_url":"http://patchwork.ozlabs.org/comment/1762210/","msgid":"","list_archive_url":null,"date":"2017-09-03T01:37:57","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65986,"url":"http://patchwork.ozlabs.org/api/people/65986/","name":"Tom Herbert","email":"tom@herbertland.com"},"content":"On Sat, Sep 2, 2017 at 6:32 PM, Hannes Frederic Sowa\n wrote:\n> Hi Saeed,\n>\n> On Sun, Sep 3, 2017, at 01:01, Saeed Mahameed wrote:\n>> On Thu, Aug 31, 2017 at 6:51 AM, Hannes Frederic Sowa\n>> wrote:\n>> > Saeed Mahameed writes:\n>> >\n>> >> The first patch from Gal and Ariel provides the mlx5 driver support for\n>> >> ConnectX capability to perform IP version identification and matching in\n>> >> order to distinguish between IPv4 and IPv6 without the need to specify the\n>> >> encapsulation type, thus perform RSS in MPLS automatically without\n>> >> specifying MPLS ethertyoe. This patch will also serve for inner GRE IPv4/6\n>> >> classification for inner GRE RSS.\n>> >\n>> > I don't think this is legal at all or did I misunderstood something?\n>> >\n>> > \n>>\n>> It seems you misunderstood the cover letter. The HW will still\n>> identify MPLS (IPv4/IPv6) packets using a new bit we specify in the HW\n>> steering rules rather than adding new specific rules with {MPLS\n>> ethertype} X {IPv4,IPv6} to classify MPLS IPv{4,6} traffic, Same\n>> functionality a better and general way to approach it.\n>> Bottom line the hardware is capable of processing MPLS headers and\n>> perform RSS on the inner packet (IPv4/6) without the need of the\n>> driver to provide precise steering MPLS rules.\n>\n> Sorry, I think I am still confused.\n>\n> I just want to make sure that you don't use the first nibble after the\n> mpls bottom of stack label in any way as an indicator if that is an IPv4\n> or IPv6 packet by default. It can be anything. The forward equivalence\n> class tells the stack which protocol you see.\n>\n> If you match on the first nibble behind the MPLS bottom of stack label\n> the '4' or '6' respectively could be part of a MAC address with its\n> first nibble being 4 or 6, because the particular pseudowire is EoMPLS\n> and uses no control world.\n>\n> I wanted to mention it, because with addition of e.g. VPLS this could\n> cause problems down the road and should at least be controllable? It is\n> probably better to use Entropy Labels in future.\n>\nOr just use IPv6 with flow label for RSS (or MPLS/UDP, GRE/UDP if you\nprefer) then all this protocol specific DPI for RSS just goes away ;-)\n\nTom\n\n> Thanks,\n> Hannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=herbertland-com.20150623.gappssmtp.com\n\theader.i=@herbertland-com.20150623.gappssmtp.com\n\theader.b=\"sTCQHnMK\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xlFwr2G7Pz9s8J\n\tfor ;\n\tSun, 3 Sep 2017 11:38:04 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1752821AbdICBh7 (ORCPT );\n\tSat, 2 Sep 2017 21:37:59 -0400","from mail-qt0-f173.google.com ([209.85.216.173]:34510 \"EHLO\n\tmail-qt0-f173.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1752806AbdICBh6 (ORCPT\n\t); Sat, 2 Sep 2017 21:37:58 -0400","by mail-qt0-f173.google.com with SMTP id u11so13012644qtu.1\n\tfor ; Sat, 02 Sep 2017 18:37:58 -0700 (PDT)","by 10.237.61.196 with HTTP; Sat, 2 Sep 2017 18:37:57 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=herbertland-com.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=/lyya8EJ38X7q92/w81YT6IBGSi0GYTs46JcZKMEREE=;\n\tb=sTCQHnMKNuEYs72LEsxlzYk5IXVn9AtVg/GH7vzGN/6Y05gXdNlN+gBP/O71RfMl2g\n\tKFnaMEGqpOYcpw6l4pD96WT8MictiuH54jx5QJDDyLQ1v8t63VS7NONQDSgmccGlqhsD\n\trZbP4xjSiuq3ruNlu6Oo7ORQmf4sBrDwrxm86lGLhe7gYticbTv16juRRdI+iH5ajtVf\n\tNXnAE62UmvFma2JETn9pYA6xg598bSDAXt+rhfO+ZLbAdlCGb2iLfUfHmYERqnVd/Gyo\n\tWr3MJzqEITCcmGaUIlhPQIval5f8NDlA6PbV7hHxgZjYdHRXGXY9lonK9grLhttYZ7hw\n\tFxRg==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=/lyya8EJ38X7q92/w81YT6IBGSi0GYTs46JcZKMEREE=;\n\tb=p2VF2rS5kKpqlED8k/GQtJt9d0Cuf/lgoLuX88an1cXv3/pI+qyR7Bm7IFfTCz9dE+\n\tmljrQKM88zETMzqYYdmNpvV+yC1tZtS0r8HQrG6PUJb07hKgcSj01RESM4V0eR7DTDlE\n\tHxmYVD/GrwJNBcks6J/gmJQxMrdDjHMCFYOKD1ux+wmmiHxfLAUoxcEAy1blfsMR5YMS\n\tDIx+rrA2YUkxOCgiVSIsnwPbPXtC7WOP4uhZqiiE1NQyXyw3/H8hp8W7GV8DOC2xoE0q\n\t9IIXtcvOp+0LPdje+zUErzu06+RIL27gZ/UbR5HavctR/frLB+NnNXrlvy56mPGHtVzk\n\tlu3g==","X-Gm-Message-State":"AHPjjUiWMV+TWWnXogK8Z0eRzAkZc3D2X9uYMC5KqMLWtvpZF+Ed1mcY\n\t9X5x/QRP7h2a/YfhnAfQczqe/LaCRrnT","X-Google-Smtp-Source":"ADKCNb7Ifo60BpOxFcYqwGN1TAsjmpURPIzRe/sqlHBBlLq7cMu7fuuNaXbEoXfJQHGeBbmMKLB8neNcoK9AYYbP+lM=","X-Received":"by 10.200.23.92 with SMTP id u28mr9915965qtk.27.1504402677672;\n\tSat, 02 Sep 2017 18:37:57 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>","From":"Tom Herbert ","Date":"Sat, 2 Sep 2017 18:37:57 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Hannes Frederic Sowa ","Cc":"Saeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762216,"web_url":"http://patchwork.ozlabs.org/comment/1762216/","msgid":"","list_archive_url":null,"date":"2017-09-03T04:07:10","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65131,"url":"http://patchwork.ozlabs.org/api/people/65131/","name":"Saeed Mahameed","email":"saeedm@dev.mellanox.co.il"},"content":"On Sat, Sep 2, 2017 at 6:32 PM, Hannes Frederic Sowa\n wrote:\n> Hi Saeed,\n>\n> On Sun, Sep 3, 2017, at 01:01, Saeed Mahameed wrote:\n>> On Thu, Aug 31, 2017 at 6:51 AM, Hannes Frederic Sowa\n>> wrote:\n>> > Saeed Mahameed writes:\n>> >\n>> >> The first patch from Gal and Ariel provides the mlx5 driver support for\n>> >> ConnectX capability to perform IP version identification and matching in\n>> >> order to distinguish between IPv4 and IPv6 without the need to specify the\n>> >> encapsulation type, thus perform RSS in MPLS automatically without\n>> >> specifying MPLS ethertyoe. This patch will also serve for inner GRE IPv4/6\n>> >> classification for inner GRE RSS.\n>> >\n>> > I don't think this is legal at all or did I misunderstood something?\n>> >\n>> > \n>>\n>> It seems you misunderstood the cover letter. The HW will still\n>> identify MPLS (IPv4/IPv6) packets using a new bit we specify in the HW\n>> steering rules rather than adding new specific rules with {MPLS\n>> ethertype} X {IPv4,IPv6} to classify MPLS IPv{4,6} traffic, Same\n>> functionality a better and general way to approach it.\n>> Bottom line the hardware is capable of processing MPLS headers and\n>> perform RSS on the inner packet (IPv4/6) without the need of the\n>> driver to provide precise steering MPLS rules.\n>\n> Sorry, I think I am still confused.\n>\n> I just want to make sure that you don't use the first nibble after the\n> mpls bottom of stack label in any way as an indicator if that is an IPv4\n> or IPv6 packet by default. It can be anything. The forward equivalence\n> class tells the stack which protocol you see.\n>\n> If you match on the first nibble behind the MPLS bottom of stack label\n> the '4' or '6' respectively could be part of a MAC address with its\n> first nibble being 4 or 6, because the particular pseudowire is EoMPLS\n> and uses no control world.\n>\n> I wanted to mention it, because with addition of e.g. VPLS this could\n> cause problems down the road and should at least be controllable? It is\n> probably better to use Entropy Labels in future.\n>\n\nHi Hannes,\n\nI see your concern now, but still it has nothing to do with the\ndriver, the whole change is only to simplify driver code to not push\nfull blown matching steering rules into the HW, and simply replace it\nwith a one bit command.\n\nRegarding your concern, I will need to check with the HW guys and\nreview the processing algorithm that identifies IP packets over MPLs,\nand will get back to you.\n\nif there is really a problem, then yes, we might need to make it\ncontrollable ..\n\n> Thanks,\n> Hannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=dev-mellanox-co-il.20150623.gappssmtp.com\n\theader.i=@dev-mellanox-co-il.20150623.gappssmtp.com\n\theader.b=\"DqJm9Ct4\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xlKFW4Zh6z9sPk\n\tfor ;\n\tSun, 3 Sep 2017 14:07:43 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1751089AbdICEHd (ORCPT );\n\tSun, 3 Sep 2017 00:07:33 -0400","from mail-lf0-f44.google.com ([209.85.215.44]:38502 \"EHLO\n\tmail-lf0-f44.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1750795AbdICEHc (ORCPT\n\t); Sun, 3 Sep 2017 00:07:32 -0400","by mail-lf0-f44.google.com with SMTP id d202so10889624lfd.5\n\tfor ; Sat, 02 Sep 2017 21:07:32 -0700 (PDT)","by 10.25.166.17 with HTTP; Sat, 2 Sep 2017 21:07:10 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=dev-mellanox-co-il.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=WrAVekZ2DyDM6Vx/oLJcsNxrpgIod81rH0QoyU7qUKI=;\n\tb=DqJm9Ct4z2wjV+h7hWFaGAjCy/vCpobK5FtVSvTFRgJPUCRDuZBS6OewIQYQ8O5UHh\n\tNPBoT6XkbYCjoaXs8r5xEHRyjwq1k3H57jKgBvrhexkHltgkCBzyJQBJvDKG5hmH5Coh\n\tWCSMz2BCXY3j6gwrBdI44OOHNsmlqPKxAtrXhHzqdM3WwiIKHPlPzCg5BzHUyEWVjrKY\n\tbvh0jWXhnaVwS3MiCBjdnPt7L+c7hHqS+X73mHzb3/kXisFXQ7UpQHrLxU4tIKjLG/x1\n\t+GycuDWna8rgY4ORL5/ryuqh4O8xRUbRL6YLhxNp7keCVDvMBmt9v01bRS+S1xq5O58v\n\t81ng==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=WrAVekZ2DyDM6Vx/oLJcsNxrpgIod81rH0QoyU7qUKI=;\n\tb=oRbi0Aepecbz/b90GuT4broYXqLt4AOvogfwnFfXjF7TyJn5SbzPp5qFjVh8gHUAbd\n\t4FdXPRfMEzY1eeAD5LAy+BVeHZ7s7qp5wvydxkBMAi/mvzh5DYbecWFj0cW5tOQIhoPK\n\t1qsK7ptgYMd3FrXmPju6o74rgJ81gn7ctNJnW36Ri31x/unNQLbpWRg4K+tW9dLGaEkd\n\tfZbDNy+vIw1MIMkQ9MR/fv2Lf+IOWavwQV+UW1ncSfSg/uRsH/LBR1I40UPc3G/yHP35\n\tFSKKP+PnSaydWxCBDqXmfxxsr8zIzB+nKoRv4qunVFScDcUZ2qbFZMF2OIQ74FzyKK4/\n\tAq5A==","X-Gm-Message-State":"AHPjjUj2gc1zovtyTlFP5WqnAn9z/tGSOX+x4tZDM7B2lw60sKEL7FM9\n\tpOg6jSzzEP/XjLEdyEmEJ1GmJYu7/eb8","X-Google-Smtp-Source":"ADKCNb4AIUIiT8u7q/+mDpeelh6J2ZyOMBDgFxqgtHmLsJ8b1VkpviVxdXntC15xZ8/YtBuH6Z7CxpvIBpyYbd6odFQ=","X-Received":"by 10.25.115.11 with SMTP id o11mr2049935lfc.170.1504411651207; \n\tSat, 02 Sep 2017 21:07:31 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>","From":"Saeed Mahameed ","Date":"Sat, 2 Sep 2017 21:07:10 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Hannes Frederic Sowa ","Cc":"Saeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762217,"web_url":"http://patchwork.ozlabs.org/comment/1762217/","msgid":"","list_archive_url":null,"date":"2017-09-03T04:11:07","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65131,"url":"http://patchwork.ozlabs.org/api/people/65131/","name":"Saeed Mahameed","email":"saeedm@dev.mellanox.co.il"},"content":"On Sat, Sep 2, 2017 at 6:37 PM, Tom Herbert wrote:\n> On Sat, Sep 2, 2017 at 6:32 PM, Hannes Frederic Sowa\n> wrote:\n>> Hi Saeed,\n>>\n>> On Sun, Sep 3, 2017, at 01:01, Saeed Mahameed wrote:\n>>> On Thu, Aug 31, 2017 at 6:51 AM, Hannes Frederic Sowa\n>>> wrote:\n>>> > Saeed Mahameed writes:\n>>> >\n>>> >> The first patch from Gal and Ariel provides the mlx5 driver support for\n>>> >> ConnectX capability to perform IP version identification and matching in\n>>> >> order to distinguish between IPv4 and IPv6 without the need to specify the\n>>> >> encapsulation type, thus perform RSS in MPLS automatically without\n>>> >> specifying MPLS ethertyoe. This patch will also serve for inner GRE IPv4/6\n>>> >> classification for inner GRE RSS.\n>>> >\n>>> > I don't think this is legal at all or did I misunderstood something?\n>>> >\n>>> > \n>>>\n>>> It seems you misunderstood the cover letter. The HW will still\n>>> identify MPLS (IPv4/IPv6) packets using a new bit we specify in the HW\n>>> steering rules rather than adding new specific rules with {MPLS\n>>> ethertype} X {IPv4,IPv6} to classify MPLS IPv{4,6} traffic, Same\n>>> functionality a better and general way to approach it.\n>>> Bottom line the hardware is capable of processing MPLS headers and\n>>> perform RSS on the inner packet (IPv4/6) without the need of the\n>>> driver to provide precise steering MPLS rules.\n>>\n>> Sorry, I think I am still confused.\n>>\n>> I just want to make sure that you don't use the first nibble after the\n>> mpls bottom of stack label in any way as an indicator if that is an IPv4\n>> or IPv6 packet by default. It can be anything. The forward equivalence\n>> class tells the stack which protocol you see.\n>>\n>> If you match on the first nibble behind the MPLS bottom of stack label\n>> the '4' or '6' respectively could be part of a MAC address with its\n>> first nibble being 4 or 6, because the particular pseudowire is EoMPLS\n>> and uses no control world.\n>>\n>> I wanted to mention it, because with addition of e.g. VPLS this could\n>> cause problems down the road and should at least be controllable? It is\n>> probably better to use Entropy Labels in future.\n>>\n> Or just use IPv6 with flow label for RSS (or MPLS/UDP, GRE/UDP if you\n> prefer) then all this protocol specific DPI for RSS just goes away ;-)\n\nHi Tom,\n\nHow does MPLS/UDP or GRE/UDP RSS works without protocol specific DPI ?\nunlike vxlan those protocols are not over UDP and you can't just play\nwith the outer header udp src port, or do you ?\n\nCan you elaborate ?\n\nThanks,\nSaeed.\n\n>\n> Tom\n>\n>> Thanks,\n>> Hannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=dev-mellanox-co-il.20150623.gappssmtp.com\n\theader.i=@dev-mellanox-co-il.20150623.gappssmtp.com\n\theader.b=\"gsH/QOa7\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xlKL24Ts0z9sPk\n\tfor ;\n\tSun, 3 Sep 2017 14:11:38 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1751030AbdICELa (ORCPT );\n\tSun, 3 Sep 2017 00:11:30 -0400","from mail-lf0-f43.google.com ([209.85.215.43]:38825 \"EHLO\n\tmail-lf0-f43.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1750795AbdICEL3 (ORCPT\n\t); Sun, 3 Sep 2017 00:11:29 -0400","by mail-lf0-f43.google.com with SMTP id d202so10898703lfd.5\n\tfor ; Sat, 02 Sep 2017 21:11:29 -0700 (PDT)","by 10.25.166.17 with HTTP; Sat, 2 Sep 2017 21:11:07 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=dev-mellanox-co-il.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=YN20G0acm0gG7KWDJCMdSkscwhS8ifV4sVs3jsKSXIs=;\n\tb=gsH/QOa7P40ehYo/8Y8mzcjZW6/DAGaqKJIw+0fJBzglsQKb3FFbUdTktxVLhWcwyg\n\ttEdu28uLMMpCeqYCx11v48JFt55CY2TvOwC2Exiq57E2kIzocwOaLDgn0Zkz/IlmkNPm\n\tW0p0zjsUu79EfEkGC0vUbg+FBWIF8prkD83fuo9KSRZrdVKrhNab8bTRduagUxn1iD9O\n\tpUYnf7rPiUjQwgNgfi4R7UQUkIGh+AuQN7PFsvBM78drnyZGN2w83WAFrx3wlI/E3+C1\n\tqzqAdv5klcU4W/Pr26FJa0bHEYmFxwFnhxF8L4OKajVaNeSV5Z3S2nXRxCfDdGsTrHBw\n\tRJbA==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=YN20G0acm0gG7KWDJCMdSkscwhS8ifV4sVs3jsKSXIs=;\n\tb=sCN23cyDYaFkXtDRgSMn50WfV2I8Q99SN5EDhbYjI1mRvK5cbCxGAeFScrOEaaOiqM\n\tZ1P5NLTqLI2EQ/CSmQuJ1L7Ahk+974CLnkBjxDdNrGcu3UiI7E/rBUVi9lB9+sBJhK6Z\n\tXlwlR6+6sBKMkNqlda4GalkFtobUw8uVbPi8sGv84dXUpcM2DlU6sJ52f/eIN412uJGb\n\tvNQgavcsB2mcWMWQhtRXG5Z5Gu9XP+VkM2/adTJdEpO8Xl47P9HlJ2bEOiL7ArblXw5P\n\tzB6iJHgM4QShjMdaQgnfZoNvOePxoWx4Xts9wwJlcZHofMFsrulITnmk9IpFzrUUnY3k\n\tbiZw==","X-Gm-Message-State":"AHPjjUiFXpBleVkjPEOov0E5XIuhyCi+Y93XJQvbzrz8JY/Q9lxF+9M8\n\tVan/Q2cTHy/M0gk0Kn8tZnDo0tPb1oRy7uI=","X-Google-Smtp-Source":"ADKCNb5YBCLZTF0TIlhdYD0LyoMkaEpZfMvhjfxQ+D0Xoolo/dr4fIiCMkf3bUhnmvigqTyWVTwvsnPst6LBlTeG1NI=","X-Received":"by 10.25.1.80 with SMTP id 77mr601049lfb.182.1504411888429; Sat,\n\t02 Sep 2017 21:11:28 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t","From":"Saeed Mahameed ","Date":"Sat, 2 Sep 2017 21:11:07 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Tom Herbert ","Cc":"Hannes Frederic Sowa ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762290,"web_url":"http://patchwork.ozlabs.org/comment/1762290/","msgid":"","list_archive_url":null,"date":"2017-09-03T15:43:27","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65986,"url":"http://patchwork.ozlabs.org/api/people/65986/","name":"Tom Herbert","email":"tom@herbertland.com"},"content":"On Sat, Sep 2, 2017 at 9:11 PM, Saeed Mahameed\n wrote:\n> On Sat, Sep 2, 2017 at 6:37 PM, Tom Herbert wrote:\n>> On Sat, Sep 2, 2017 at 6:32 PM, Hannes Frederic Sowa\n>> wrote:\n>>> Hi Saeed,\n>>>\n>>> On Sun, Sep 3, 2017, at 01:01, Saeed Mahameed wrote:\n>>>> On Thu, Aug 31, 2017 at 6:51 AM, Hannes Frederic Sowa\n>>>> wrote:\n>>>> > Saeed Mahameed writes:\n>>>> >\n>>>> >> The first patch from Gal and Ariel provides the mlx5 driver support for\n>>>> >> ConnectX capability to perform IP version identification and matching in\n>>>> >> order to distinguish between IPv4 and IPv6 without the need to specify the\n>>>> >> encapsulation type, thus perform RSS in MPLS automatically without\n>>>> >> specifying MPLS ethertyoe. This patch will also serve for inner GRE IPv4/6\n>>>> >> classification for inner GRE RSS.\n>>>> >\n>>>> > I don't think this is legal at all or did I misunderstood something?\n>>>> >\n>>>> > \n>>>>\n>>>> It seems you misunderstood the cover letter. The HW will still\n>>>> identify MPLS (IPv4/IPv6) packets using a new bit we specify in the HW\n>>>> steering rules rather than adding new specific rules with {MPLS\n>>>> ethertype} X {IPv4,IPv6} to classify MPLS IPv{4,6} traffic, Same\n>>>> functionality a better and general way to approach it.\n>>>> Bottom line the hardware is capable of processing MPLS headers and\n>>>> perform RSS on the inner packet (IPv4/6) without the need of the\n>>>> driver to provide precise steering MPLS rules.\n>>>\n>>> Sorry, I think I am still confused.\n>>>\n>>> I just want to make sure that you don't use the first nibble after the\n>>> mpls bottom of stack label in any way as an indicator if that is an IPv4\n>>> or IPv6 packet by default. It can be anything. The forward equivalence\n>>> class tells the stack which protocol you see.\n>>>\n>>> If you match on the first nibble behind the MPLS bottom of stack label\n>>> the '4' or '6' respectively could be part of a MAC address with its\n>>> first nibble being 4 or 6, because the particular pseudowire is EoMPLS\n>>> and uses no control world.\n>>>\n>>> I wanted to mention it, because with addition of e.g. VPLS this could\n>>> cause problems down the road and should at least be controllable? It is\n>>> probably better to use Entropy Labels in future.\n>>>\n>> Or just use IPv6 with flow label for RSS (or MPLS/UDP, GRE/UDP if you\n>> prefer) then all this protocol specific DPI for RSS just goes away ;-)\n>\n> Hi Tom,\n>\n> How does MPLS/UDP or GRE/UDP RSS works without protocol specific DPI ?\n> unlike vxlan those protocols are not over UDP and you can't just play\n> with the outer header udp src port, or do you ?\n>\n> Can you elaborate ?\n>\nHi Saeed,\n\nAn encapsulator sets the UDP source port to be the flow entropy of the\npacket being encapsulated. So when the packet traverses the network\ndevices can base their hash just on the canonical 5-tuple which is\nsufficient for ECMP and RSS. IPv6 flow label is even better since the\nmiddleboxes don't even need to look at the transport header, a packet\nis steered based on the 3-tuple of addresses and flow label. In the\nLinux stack, udp_flow_src_port is used by UDP encapsulations to set\nthe source port. Flow label is similarly set by ip6_make_flowlabel.\nBoth of these functions use the skb->hash which is computed by calling\nflow dissector at most once per packet (if the packet was received\nwith an L4 HW hash or locally originated on a connection the hash does\nnot need to be computed).\n\nPlease look at https://people.netfilter.org/pablo/netdev0.1/papers/UDP-Encapsulation-in-Linux.pdf\nas well as Davem's \"Less is More\" presentation which highlights the\nvirtues of protocol generic HW mechanisms\n(https://www.youtube.com/watch?v=6VgmazGwL_Y).\n\nTom","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=herbertland-com.20150623.gappssmtp.com\n\theader.i=@herbertland-com.20150623.gappssmtp.com\n\theader.b=\"GI0ei4gr\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xlchR68xpz9t33\n\tfor ;\n\tMon, 4 Sep 2017 01:43:35 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1753065AbdICPna (ORCPT );\n\tSun, 3 Sep 2017 11:43:30 -0400","from mail-qk0-f179.google.com ([209.85.220.179]:35944 \"EHLO\n\tmail-qk0-f179.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1752915AbdICPn3 (ORCPT\n\t); Sun, 3 Sep 2017 11:43:29 -0400","by mail-qk0-f179.google.com with SMTP id o63so15210754qkb.3\n\tfor ; Sun, 03 Sep 2017 08:43:28 -0700 (PDT)","by 10.237.61.196 with HTTP; Sun, 3 Sep 2017 08:43:27 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=herbertland-com.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=ItuWVxXZ9D3nZRyLF/JmTFU2hcxOThCCWTtYmZdx7Aw=;\n\tb=GI0ei4grbtWLnNKr5qyj9D+laKp2koKw159xOHklmeuF9zmVKojoSgNv+Rz5YjWwKs\n\tywg/y0B5XHy0jbYDLsm0+ut2PMpfN7onmIPebVuZqrxVB/ZkkOyz0rAxZH9i3NWYi3gb\n\tOnKv20DPeKaCfSG9EvAZVKgDBzGYs/VKNX5Ex/9NKHPPp9XPEMYcJxs/Eqg4W5EpGPCw\n\tQidsCWAXtNfWfC2l4cwtyOYObXUi8ytsH7jFz0YEpcxqutYpdox9BzUTKVj7GnI+IoOH\n\tn2v/AR6GwQxcKLbWXAjmveGQQRHEnYmSV/y589z5DLxPnmB2xDVooIDGKiPD9ixfGB0/\n\tpThw==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=ItuWVxXZ9D3nZRyLF/JmTFU2hcxOThCCWTtYmZdx7Aw=;\n\tb=qgUH7qY5q03pdk+TLWzwC1aeFHLGBwEAtIT4cO90stKb9FPB5uxVJIRz2+E5OmI6BQ\n\tbgvrMeAjuKP2MOmFIr1OU0+/1TM1euVKrknV9xaQ58dkLqIfW/uT16OaBnaN51xbqlsA\n\taJ+mzq3+HxxTd7Pbt1+xsyTI4F1tnOuaxEufYN8hKBnJ4/mzi8ze+gOVqPFssKssip5N\n\trajUy3KsSzs5BzyOEO+5w+dIsgD4OwJazNPksUvhyYgyfweD6vQTiNiuR2Q4yJzWpasI\n\toadDhN6WZ5inMA1kfb9M/RLf1jPYzhIIsfrPWoop4VeFR8nGQtXSJmMAEhX2zytKrwpp\n\tF13g==","X-Gm-Message-State":"AHPjjUj7iGEVsxxqp8GCXb5wUfrH/w+XB+KS4LJO6Z4j6X8069d1RRPJ\n\tm9yCF2FjDTdl9lEx474/AiPWv2s3aUNJzDk=","X-Google-Smtp-Source":"ADKCNb79qpRavXvGGomOkJMP9dKrEtcgOUU7P5/EMuEYzyqRfjGziJGdET01t3mpJt93EziX8xu/LWPKrthUt/d5EpE=","X-Received":"by 10.55.48.71 with SMTP id w68mr9989191qkw.345.1504453408431;\n\tSun, 03 Sep 2017 08:43:28 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t","From":"Tom Herbert ","Date":"Sun, 3 Sep 2017 08:43:27 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Saeed Mahameed ","Cc":"Hannes Frederic Sowa ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762296,"web_url":"http://patchwork.ozlabs.org/comment/1762296/","msgid":"","list_archive_url":null,"date":"2017-09-03T16:17:08","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":64793,"url":"http://patchwork.ozlabs.org/api/people/64793/","name":"Or Gerlitz","email":"gerlitz.or@gmail.com"},"content":"On Sun, Sep 3, 2017 at 6:43 PM, Tom Herbert wrote:\n> On Sat, Sep 2, 2017 at 9:11 PM, Saeed Mahameed wrote:\n>> On Sat, Sep 2, 2017 at 6:37 PM, Tom Herbert wrote:\n>>> On Sat, Sep 2, 2017 at 6:32 PM, Hannes Frederic Sowa\n>>> wrote:\n>>>> Hi Saeed,\n>>>>\n>>>> On Sun, Sep 3, 2017, at 01:01, Saeed Mahameed wrote:\n>>>>> On Thu, Aug 31, 2017 at 6:51 AM, Hannes Frederic Sowa\n>>>>> wrote:\n>>>>> > Saeed Mahameed writes:\n>>>>> >\n>>>>> >> The first patch from Gal and Ariel provides the mlx5 driver support for\n>>>>> >> ConnectX capability to perform IP version identification and matching in\n>>>>> >> order to distinguish between IPv4 and IPv6 without the need to specify the\n>>>>> >> encapsulation type, thus perform RSS in MPLS automatically without\n>>>>> >> specifying MPLS ethertyoe. This patch will also serve for inner GRE IPv4/6\n>>>>> >> classification for inner GRE RSS.\n>>>>> >\n>>>>> > I don't think this is legal at all or did I misunderstood something?\n>>>>> >\n>>>>> > \n>>>>>\n>>>>> It seems you misunderstood the cover letter. The HW will still\n>>>>> identify MPLS (IPv4/IPv6) packets using a new bit we specify in the HW\n>>>>> steering rules rather than adding new specific rules with {MPLS\n>>>>> ethertype} X {IPv4,IPv6} to classify MPLS IPv{4,6} traffic, Same\n>>>>> functionality a better and general way to approach it.\n>>>>> Bottom line the hardware is capable of processing MPLS headers and\n>>>>> perform RSS on the inner packet (IPv4/6) without the need of the\n>>>>> driver to provide precise steering MPLS rules.\n>>>>\n>>>> Sorry, I think I am still confused.\n>>>>\n>>>> I just want to make sure that you don't use the first nibble after the\n>>>> mpls bottom of stack label in any way as an indicator if that is an IPv4\n>>>> or IPv6 packet by default. It can be anything. The forward equivalence\n>>>> class tells the stack which protocol you see.\n>>>>\n>>>> If you match on the first nibble behind the MPLS bottom of stack label\n>>>> the '4' or '6' respectively could be part of a MAC address with its\n>>>> first nibble being 4 or 6, because the particular pseudowire is EoMPLS\n>>>> and uses no control world.\n>>>>\n>>>> I wanted to mention it, because with addition of e.g. VPLS this could\n>>>> cause problems down the road and should at least be controllable? It is\n>>>> probably better to use Entropy Labels in future.\n>>>>\n>>> Or just use IPv6 with flow label for RSS (or MPLS/UDP, GRE/UDP if you\n>>> prefer) then all this protocol specific DPI for RSS just goes away ;-)\n\n>> How does MPLS/UDP or GRE/UDP RSS works without protocol specific DPI ?\n>> unlike vxlan those protocols are not over UDP and you can't just play\n>> with the outer header udp src port, or do you ?\n>> Can you elaborate ?\n\n> An encapsulator sets the UDP source port to be the flow entropy of the\n> packet being encapsulated. So when the packet traverses the network\n> devices can base their hash just on the canonical 5-tuple which is\n> sufficient for ECMP and RSS. IPv6 flow label is even better since the\n> middleboxes don't even need to look at the transport header, a packet\n> is steered based on the 3-tuple of addresses and flow label. In the\n> Linux stack, udp_flow_src_port is used by UDP encapsulations to set\n> the source port. Flow label is similarly set by ip6_make_flowlabel.\n> Both of these functions use the skb->hash which is computed by calling\n> flow dissector at most once per packet (if the packet was received\n> with an L4 HW hash or locally originated on a connection the hash does\n> not need to be computed).\n\nHi Tom,\n\nRe all sorts of udp encap, sure, we're all on the less-is-more thing and just\nRSS-ing on the ip+udp encap header.\n\nFor GRE, I was trying to fight back that rss-ing on inner, but as\nSaeed commented,\nwe didn't see something simple through which the HW can do spreading. To make\nsure I follow, you are saying that if this is gre6 tunneling\n\nnet-next.git]# git grep -p ip6_make_flowlabel net/ include/linux/ include/net/\ninclude/net/ipv6.h=static inline void iph_to_flow_copy_v6addrs(struct\nflow_keys *flow,\ninclude/net/ipv6.h:static inline __be32 ip6_make_flowlabel(struct net\n*net, struct sk_buff *skb,\ninclude/net/ipv6.h=static inline void ip6_set_txhash(struct sock *sk) { }\ninclude/net/ipv6.h:static inline __be32 ip6_make_flowlabel(struct net\n*net, struct sk_buff *skb,\nnet/ipv6/ip6_gre.c=static int ip6gre_header(struct sk_buff *skb,\nstruct net_device *dev,\nnet/ipv6/ip6_gre.c: ip6_make_flowlabel(dev_net(dev), skb,\nnet/ipv6/ip6_output.c=int ip6_xmit(const struct sock *sk, struct\nsk_buff *skb, struct flowi6 *fl6,\nnet/ipv6/ip6_output.c: ip6_flow_hdr(hdr, tclass,\nip6_make_flowlabel(net, skb, fl6->flowlabel,\nnet/ipv6/ip6_output.c=struct sk_buff *__ip6_make_skb(struct sock *sk,\nnet/ipv6/ip6_output.c: ip6_make_flowlabel(net, skb,\nfl6->flowlabel,\nnet/ipv6/ip6_tunnel.c=int ip6_tnl_xmit(struct sk_buff *skb, struct\nnet_device *dev, __u8 dsfield,\nnet/ipv6/ip6_tunnel.c: ip6_make_flowlabel(net, skb,\nfl6->flowlabel, true, fl6));\n\nthe sender side (ip6_tnl_xmit?) will set the IPv6 flow label in a\nsimilar manner done by udp_flow_src_port? and\nif the receiver side hashes on L3 IPv6 src/dst/flow label we'll get\nspreading? nice!\n\nStill, what do we do with IPv4 GRE tunnels? and what do we do with HW\nwhich isn't capable to RSS on flow label?\n\n> Please look at https://people.netfilter.org/pablo/netdev0.1/papers/UDP-Encapsulation-in-Linux.pdf\n> as well as Davem's \"Less is More\" presentation which highlights the\n> virtues of protocol generic HW mechanisms\n> (https://www.youtube.com/watch?v=6VgmazGwL_Y).","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=gmail.com header.i=@gmail.com\n\theader.b=\"tm+OViAd\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xldRH1SQ4z9t3V\n\tfor ;\n\tMon, 4 Sep 2017 02:17:15 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1753129AbdICQRL (ORCPT );\n\tSun, 3 Sep 2017 12:17:11 -0400","from mail-oi0-f43.google.com ([209.85.218.43]:35493 \"EHLO\n\tmail-oi0-f43.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1753076AbdICQRK (ORCPT\n\t); Sun, 3 Sep 2017 12:17:10 -0400","by mail-oi0-f43.google.com with SMTP id n18so30486932oig.2\n\tfor ; Sun, 03 Sep 2017 09:17:09 -0700 (PDT)","by 10.202.220.87 with HTTP; Sun, 3 Sep 2017 09:17:08 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=gmail.com; s=20161025;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=ChAL/kCMwTiS0zjmOO7CN35w0a9PNOH7TIgcsqUZtJU=;\n\tb=tm+OViAd4xU/UJGK9duHcFwWlAwptQNYghQfHYW6yGkHRfYInTBkHPdXHBL4cNO1j4\n\ti2d3cZIUO/5vbS3ni7e2HyXToAY61IrX9RlW2DD0XMEsQesrU2eCXXGixCP6zMyCIrVm\n\tV8F9L+jujPh35iwv4pBMf+RGcNiWo6V7ypAWD+ZOEGb5lP7j1O2etBuOsba/QzCPwCxk\n\txRVtlpSa0Wl/F1nM9b745+pfOpVA9qDhw7wmTu0EyRXgrND5sjmnFpVLSViINAWX5RPv\n\t0sUVrkH3oOgm5wdd3ZQ2NMGkOwJnGjV9iGB1VaFBwcZXadqJotfGc1QSUU0qN2mwy7EY\n\tXu2A==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=ChAL/kCMwTiS0zjmOO7CN35w0a9PNOH7TIgcsqUZtJU=;\n\tb=X92SndGn7kZWEg0F6xi5pf8tIdA8e+YYcRcVR31he77TNeEjIc7RKJ1m1BujjQTDxH\n\tgQQTH0mbyijq84dZyOJcBsFHOwvw2rCtzsDKaLuJWIzw4v5Ag+GUyoPg7IL5zQrjFNxc\n\tVlZrLpcudtPxkFb5XhOoF8s/YpuUDBln5amqOmaWFGJTNwkbOtUABYe4BmyV1uwSpgdR\n\tLIxx9v9pi3Aq+M4ZpgRYgMYKiyshI9lAg9OXqjykecb7DfndS/JZjyryyrWD1DsB+YrL\n\tUbkCSfnaNDP5hGA2dK9BeCMR/93sAcud1KKu5wyESmCgf9VVvouY1/OiU8M4jJNjuc9X\n\tICBA==","X-Gm-Message-State":"AHPjjUh2BzgvtfpFQq51FLw74zcJe1pb79k8NaWxtnnE+Ho7TV5zMwSm\n\t1IcMo1pnGRR7Oqru+r+nZ5ABjCXzfn0X","X-Google-Smtp-Source":"ADKCNb64dNnJ5U9Q1l4be++Ktz2rftS17Ho+MupMLJ1f6cvQVRuCkDxHKbjnLGbAwz2D6sRaLFQGVnx+qghiioY4f0s=","X-Received":"by 10.202.208.13 with SMTP id h13mr1584384oig.319.1504455429213; \n\tSun, 03 Sep 2017 09:17:09 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t","From":"Or Gerlitz ","Date":"Sun, 3 Sep 2017 19:17:08 +0300","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Tom Herbert ","Cc":"Saeed Mahameed ,\n\tHannes Frederic Sowa ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762303,"web_url":"http://patchwork.ozlabs.org/comment/1762303/","msgid":"","list_archive_url":null,"date":"2017-09-03T16:45:00","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65986,"url":"http://patchwork.ozlabs.org/api/people/65986/","name":"Tom Herbert","email":"tom@herbertland.com"},"content":"> Re all sorts of udp encap, sure, we're all on the less-is-more thing and just\n> RSS-ing on the ip+udp encap header.\n>\n> For GRE, I was trying to fight back that rss-ing on inner, but as\n> Saeed commented,\n> we didn't see something simple through which the HW can do spreading. To make\n> sure I follow, you are saying that if this is gre6 tunneling\n>\nIt's pretty common that HW does this since GRE is in widespread use for a while.\n\n> net-next.git]# git grep -p ip6_make_flowlabel net/ include/linux/ include/net/\n> include/net/ipv6.h=static inline void iph_to_flow_copy_v6addrs(struct\n> flow_keys *flow,\n> include/net/ipv6.h:static inline __be32 ip6_make_flowlabel(struct net\n> *net, struct sk_buff *skb,\n> include/net/ipv6.h=static inline void ip6_set_txhash(struct sock *sk) { }\n> include/net/ipv6.h:static inline __be32 ip6_make_flowlabel(struct net\n> *net, struct sk_buff *skb,\n> net/ipv6/ip6_gre.c=static int ip6gre_header(struct sk_buff *skb,\n> struct net_device *dev,\n> net/ipv6/ip6_gre.c: ip6_make_flowlabel(dev_net(dev), skb,\n> net/ipv6/ip6_output.c=int ip6_xmit(const struct sock *sk, struct\n> sk_buff *skb, struct flowi6 *fl6,\n> net/ipv6/ip6_output.c: ip6_flow_hdr(hdr, tclass,\n> ip6_make_flowlabel(net, skb, fl6->flowlabel,\n> net/ipv6/ip6_output.c=struct sk_buff *__ip6_make_skb(struct sock *sk,\n> net/ipv6/ip6_output.c: ip6_make_flowlabel(net, skb,\n> fl6->flowlabel,\n> net/ipv6/ip6_tunnel.c=int ip6_tnl_xmit(struct sk_buff *skb, struct\n> net_device *dev, __u8 dsfield,\n> net/ipv6/ip6_tunnel.c: ip6_make_flowlabel(net, skb,\n> fl6->flowlabel, true, fl6));\n>\nSeems right.\n\n> the sender side (ip6_tnl_xmit?) will set the IPv6 flow label in a\n> similar manner done by udp_flow_src_port? and\n> if the receiver side hashes on L3 IPv6 src/dst/flow label we'll get\n> spreading? nice!\n>\nAs long as the network devices support it.\n\n> Still, what do we do with IPv4 GRE tunnels? and what do we do with HW\n> which isn't capable to RSS on flow label?\n>\nThrow it out and buy hardware that supports flow label! ;-)\n\nSeriously though, flow labels are the only reasonable way that RSS can\nbe supported in IPv6. If a device tries to do DPI on IPv6 then they'll\neventually need to be able to parse of some number of extension\nheaders which unlike IPv4 is unbounded in size. So there are going to\nbe circumstances in which a device either doesn't understand an EH, or\nthe size of EHs blows it parsing buffer limit so it can't do the DPI.\nIMO, telling host implementations that we're not allowed to use\nextension headers because middleboxes can't support them is\nunacceptable...\n\nBtw, these same arguments apply as to why CHECKSUM_COMPLETE is the\nonly reasonable way to handle receive checksums in IPv6.\n\nTom","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=herbertland-com.20150623.gappssmtp.com\n\theader.i=@herbertland-com.20150623.gappssmtp.com\n\theader.b=\"YXLo3JpL\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xlf3Q5gz4z9t2f\n\tfor ;\n\tMon, 4 Sep 2017 02:45:06 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1753129AbdICQpC (ORCPT );\n\tSun, 3 Sep 2017 12:45:02 -0400","from mail-qt0-f193.google.com ([209.85.216.193]:37615 \"EHLO\n\tmail-qt0-f193.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1753068AbdICQpB (ORCPT\n\t); Sun, 3 Sep 2017 12:45:01 -0400","by mail-qt0-f193.google.com with SMTP id x29so3125281qtc.4\n\tfor ; Sun, 03 Sep 2017 09:45:01 -0700 (PDT)","by 10.237.61.196 with HTTP; Sun, 3 Sep 2017 09:45:00 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=herbertland-com.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=6DNOps8jy/q4z9VqxfZGU1IPngVc/F1Wo25pvoeZvAI=;\n\tb=YXLo3JpL603RBSuSouN8chuuz0ONMHxx7ofVoLOZ7InB8JSJNfMN17QiGuvdhnKyls\n\tyOldDEL4USsFz/02k4VaiTYin5W6dOiiG0E2kOAPwUSef5UgHN0fmqxoPHMlKEZo2xNK\n\tOwrf/+4o0PsdM30auUda7bedvH4Ieq1ybXqirMm5kVaVaPxEOR9Pfo8enaQgSog75hen\n\tw+MeY8otCnJ6Wraov9zdudwg7AqhG8ZMDvth+ypDf75hmQrFZrNxV1OBsY2TbhsWLVub\n\tVn4Nx8EIQH4K8iXqo5W4oWoBz2pJGVBrUhDSCeN4THSAziUnls9KC9Ha5IekZotKOKto\n\tXkWQ==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=6DNOps8jy/q4z9VqxfZGU1IPngVc/F1Wo25pvoeZvAI=;\n\tb=iVwfyO+yIdm0adRMRcwk3QKBHz7Zvgh8fguBx9hQ73Cc3alGX493jMZUp7wKIrh4+F\n\tAQSBkzcnyc1IFVjncWyhBmB9bU3jcrYM7MJdC0mrpv06Vrto1VrE818aqkQb515Bccpm\n\tsyRiunxIsQFBODFZSGzRMbGj0n8d0xuQQAByu0XT6NYEjijgdPvo96MpyBxxtptUVN3j\n\tAGhkiDE7vEAbU6eKoJGl9ZCx1f0234kyyWNpTFnnVTZoiQK66vw+6KjfUytT1DgNvLY9\n\ta35m2Pweb+Vl2A1sae/c2uXeIEKEeifuhay/WT5lccvcVNaqxFOWYMbfTquHmn1Rki4I\n\tYklQ==","X-Gm-Message-State":"AHPjjUi/llpi0fJlHjrcsR8/3kfE0tku6M2EDGEN8YDLPwl2g5oG63Eo\n\t3/eId0V8f3Z5p7VF4FJGuVHQv5B/Wu6M","X-Google-Smtp-Source":"ADKCNb7EeeuI0VoB0eq3D66yS5T0G2Rb8enYazBYOv1h4pfcdgodYMq1rKDRx6uFTFR64sT3sMAxwtoQpJHNPJq+/+I=","X-Received":"by 10.237.53.213 with SMTP id d21mr12230485qte.286.1504457100948;\n\tSun, 03 Sep 2017 09:45:00 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t","From":"Tom Herbert ","Date":"Sun, 3 Sep 2017 09:45:00 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Or Gerlitz ","Cc":"Saeed Mahameed ,\n\tHannes Frederic Sowa ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762325,"web_url":"http://patchwork.ozlabs.org/comment/1762325/","msgid":"","list_archive_url":null,"date":"2017-09-03T18:58:55","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":64793,"url":"http://patchwork.ozlabs.org/api/people/64793/","name":"Or Gerlitz","email":"gerlitz.or@gmail.com"},"content":"On Sun, Sep 3, 2017 at 7:45 PM, Tom Herbert wrote:\n>> Re all sorts of udp encap, sure, we're all on the less-is-more thing and just\n>> RSS-ing on the ip+udp encap header.\n\n>> For GRE, I was trying to fight back that rss-ing on inner, but as\n>> Saeed commented,\n>> we didn't see something simple through which the HW can do spreading. To\n>> make sure I follow, you are saying that if this is gre6 tunneling\n\n> It's pretty common that HW does this since GRE is in widespread use for a while.\n\nECMP? I guess so\n\n\n>> the sender side (ip6_tnl_xmit?) will set the IPv6 flow label in a\n>> similar manner done by udp_flow_src_port? and\n>> if the receiver side hashes on L3 IPv6 src/dst/flow label we'll get\n>> spreading? nice!\n\n> As long as the network devices support it.\n\nyeah, hopefully upcoming devices will support the thing\n\n>> Still, what do we do with IPv4 GRE tunnels? and what do we do with HW\n>> which isn't capable to RSS on flow label?\n\n> Throw it out and buy hardware that supports flow label! ;-)\n\nstill, we came across IPv4 GRE customer environments\n\n> Seriously though, flow labels are the only reasonable way that RSS can\n> be supported in IPv6. If a device tries to do DPI on IPv6 then they'll\n> eventually need to be able to parse of some number of extension\n> headers which unlike IPv4 is unbounded in size. So there are going to\n> be circumstances in which a device either doesn't understand an EH, or\n> the size of EHs blows it parsing buffer limit so it can't do the DPI.\n> IMO, telling host implementations that we're not allowed to use\n> extension headers because middleboxes can't support them is\n> unacceptable...\n\nmakes sense to me\n\n> Btw, these same arguments apply as to why CHECKSUM_COMPLETE is the\n> only reasonable way to handle receive checksums in IPv6.\n\nsupported on mlx5","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=gmail.com header.i=@gmail.com\n\theader.b=\"szXjjqtM\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xlj813MWTz9sRm\n\tfor ;\n\tMon, 4 Sep 2017 05:04:17 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1751509AbdICS66 (ORCPT );\n\tSun, 3 Sep 2017 14:58:58 -0400","from mail-oi0-f52.google.com ([209.85.218.52]:33902 \"EHLO\n\tmail-oi0-f52.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1750930AbdICS64 (ORCPT\n\t); Sun, 3 Sep 2017 14:58:56 -0400","by mail-oi0-f52.google.com with SMTP id h70so880321oic.1\n\tfor ; Sun, 03 Sep 2017 11:58:56 -0700 (PDT)","by 10.202.220.87 with HTTP; Sun, 3 Sep 2017 11:58:55 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=gmail.com; s=20161025;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=Ayy5E4uN1WzguK97cUeuVBz9noquoSZ0nZV+ucKQLwo=;\n\tb=szXjjqtM29o8qc64AFZrpgMxaTMEJU5PunCb5nAxVcRKnl4kvnW3EEOK62EgKjpUQs\n\tVmVRKUvDMHzE+k9be+Ys5kwAYB+pJhw2ACs/WpQyw0ZfMwVpCguKF9o8i32l0B9+K7jC\n\tw/Q84i//qeoiGU1svXTwfGLOhqrzhfIRR4g2QtKOTCbaBK6aDM55E1ubFoLb8SPT/uJh\n\tdMTTjt0SeKkX45v2pJpGzf+i5f44ONcmy9JSVloYFxFn00q8gnQKIa3PEqIu/xFZiuRu\n\tqEWnDdGxyV5MjR57iWhdg3EoWSBOeqdkcXGUkWukbNhp09Hxq8DCmUwwJuurSZYoOG+V\n\tnSaQ==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=Ayy5E4uN1WzguK97cUeuVBz9noquoSZ0nZV+ucKQLwo=;\n\tb=oEStmmcyCcuF8mIsIX2pdfN2gPU024/6IbtFRByeWXkpS1QyfstZh9QKYrVzYsTU2P\n\tyBFotoHy556hgfiKoSspI3SsYbsF2qnLlbEe3As/j4XGr3jM/QFwdHpwr/VzG+1NrW5r\n\tcGr4I7NEr2GXUPff86C0K4eEfoKL1SAWf73xgYGTnS4+wb10+YtzHezAH+7nGlGKTK99\n\tkq+BJgeoGHwq/6H6Bq2IGkRAfVTbhAHM7cV9JYYkHstMcswjwUgRGRPf0IWlwI61ESaY\n\tXNNz/VdCfgihYBI2gSu8L683pydxJwrDZ3qdNzHh4sG8xeIJSI9QDzQVFuR5LiQljExN\n\tfxMA==","X-Gm-Message-State":"AHPjjUgeFncFETbkDTt5RD/iu9hf/CE9feNIkvRNgYXbSNENIvMfkAbk\n\tXewWJTFukc117nfqx30FCJ4VIsErZQ==","X-Google-Smtp-Source":"ADKCNb451Pm7NjhXgY0UOnnsiaaBGijMyCwfOEyjNegEHoIrFGYObSZYy+IghMeto/BktikyN4Tm1T61RDuDAeem73U=","X-Received":"by 10.202.8.144 with SMTP id 138mr8773131oii.318.1504465136243; \n\tSun, 03 Sep 2017 11:58:56 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t\n\t","From":"Or Gerlitz ","Date":"Sun, 3 Sep 2017 21:58:55 +0300","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Tom Herbert ","Cc":"Saeed Mahameed ,\n\tHannes Frederic Sowa ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762539,"web_url":"http://patchwork.ozlabs.org/comment/1762539/","msgid":"<87shg2by9a.fsf@stressinduktion.org>","list_archive_url":null,"date":"2017-09-04T09:37:21","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":18284,"url":"http://patchwork.ozlabs.org/api/people/18284/","name":"Hannes Frederic Sowa","email":"hannes@stressinduktion.org"},"content":"Hello,\n\nSaeed Mahameed writes:\n\n[...]\n\n> On Sat, Sep 2, 2017 at 6:32 PM, Hannes Frederic Sowa\n> wrote:\n>> Sorry, I think I am still confused.\n>>\n>> I just want to make sure that you don't use the first nibble after the\n>> mpls bottom of stack label in any way as an indicator if that is an IPv4\n>> or IPv6 packet by default. It can be anything. The forward equivalence\n>> class tells the stack which protocol you see.\n>>\n>> If you match on the first nibble behind the MPLS bottom of stack label\n>> the '4' or '6' respectively could be part of a MAC address with its\n>> first nibble being 4 or 6, because the particular pseudowire is EoMPLS\n>> and uses no control world.\n>>\n>> I wanted to mention it, because with addition of e.g. VPLS this could\n>> cause problems down the road and should at least be controllable? It is\n>> probably better to use Entropy Labels in future.\n>>\n>\n> Hi Hannes,\n>\n> I see your concern now, but still it has nothing to do with the\n> driver, the whole change is only to simplify driver code to not push\n> full blown matching steering rules into the HW, and simply replace it\n> with a one bit command.\n\nSorry, I very much got the impression from the cover letter plus the\ndescriptions of the patches.\n\n> Regarding your concern, I will need to check with the HW guys and\n> review the processing algorithm that identifies IP packets over MPLs,\n> and will get back to you.\n>\n> if there is really a problem, then yes, we might need to make it\n> controllable ..\n\nThanks for looking into this. I do think it can be added as a feature\nbut I very much think such logic should be disabled by default.\n\nThanks,\nHannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=stressinduktion.org\n\theader.i=@stressinduktion.org header.b=\"bO5YMkUK\"; \n\tdkim=pass (2048-bit key;\n\tunprotected) header.d=messagingengine.com\n\theader.i=@messagingengine.com header.b=\"SxFOKXkJ\"; \n\tdkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xm4Wf05Ywz9s7C\n\tfor ;\n\tMon, 4 Sep 2017 19:37:33 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1753422AbdIDJh3 (ORCPT );\n\tMon, 4 Sep 2017 05:37:29 -0400","from out1-smtp.messagingengine.com ([66.111.4.25]:38023 \"EHLO\n\tout1-smtp.messagingengine.com\" rhost-flags-OK-OK-OK-OK)\n\tby vger.kernel.org with ESMTP id S1753364AbdIDJh2 (ORCPT\n\t); Mon, 4 Sep 2017 05:37:28 -0400","from compute7.internal (compute7.nyi.internal [10.202.2.47])\n\tby mailout.nyi.internal (Postfix) with ESMTP id 2711220BB9;\n\tMon, 4 Sep 2017 05:37:28 -0400 (EDT)","from frontend1 ([10.202.2.160])\n\tby compute7.internal (MEProxy); Mon, 04 Sep 2017 05:37:28 -0400","from z.localhost.stressinduktion.org (unknown [217.192.177.51])\n\tby mail.messagingengine.com (Postfix) with ESMTPA id 44D007FA75;\n\tMon, 4 Sep 2017 05:37:27 -0400 (EDT)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tstressinduktion.org; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=gvpvGP+fCgLUlmBuLl\n\tiqZZHRwMMO537guDCTA58grIw=; b=bO5YMkUKpV/7v1E9DkU9P+XbH79x0O85dN\n\tXUdcYzhwaHV8J1jLJqjrOlHrp5gJTKMGBShq4+8JchxWC15um1SaaLh+Hny2zguz\n\tveR91wqXLHpQzZooT1deO6wuF8PWyuyiBrYU1xTOqQSLC9tiLgvEQEz+Q6rLHv27\n\tdowiQ3ohhN8WcKyA13pQ/97mS69V/rRjaBwbgA7oVbntGqOE17SXR7bC5NdS4zO8\n\tKwFOioDjtTPf2RMC5vbwjnUgEDxeMK3giAwWRFluRu4tSu5FE9DK6Lm5OscY/Uzb\n\tDZ5y5LagCKzoq4Io8yTsbuv+/XpNOZaSE0L5rSYvDhBTbaRcCYRg==","v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tmessagingengine.com; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=gvpvGP+fCgLUlmBuLl\n\tiqZZHRwMMO537guDCTA58grIw=; b=SxFOKXkJRBD3hvHlBXkROj0IjQKHSIMGoq\n\t+Fbpua0pbEN+B1pLaviVUcZ/rNyI61/hLe7FuGpm9tbqM/KZrg9F9vN9Yd6pSb/3\n\tsUd6i/T+Kmw5Ulp9zk7nE5WLQ9ml2ecYA1hwHTKIZe1AcXqOKI7LGsLOvZWMOIOY\n\tb+xn6+6Cuq+7mW3aHbD5BkTTZGIiYQ1is7Jfc0YfHUs2n7pCU062k4R9KWOkmDTI\n\t52tRcSFmHAOytYyqRnCEUrExcRLfdsItz0Jwtl5Ef+WgebuXyyx8VkM5K9MB7+di\n\tAQmYg8t3pWkb8XMcg/dkyyrX7TtUKQYz5XsleGZC0FpkrAsy5oXg=="],"X-ME-Sender":"","X-Sasl-enc":"atprEPWMENIr9dP8sKbG/2F7cW+YdITSzVyfSltShocn 1504517847","From":"Hannes Frederic Sowa ","To":"Saeed Mahameed ","Cc":"Saeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t","Date":"Mon, 04 Sep 2017 11:37:21 +0200","In-Reply-To":"\n\t(Saeed Mahameed's message of \"Sat, 2 Sep 2017 21:07:10 -0700\")","Message-ID":"<87shg2by9a.fsf@stressinduktion.org>","User-Agent":"Gnus/5.13 (Gnus v5.13) Emacs/25.2 (gnu/linux)","MIME-Version":"1.0","Content-Type":"text/plain","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762651,"web_url":"http://patchwork.ozlabs.org/comment/1762651/","msgid":"<87pob6bmjv.fsf@stressinduktion.org>","list_archive_url":null,"date":"2017-09-04T13:50:12","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":18284,"url":"http://patchwork.ozlabs.org/api/people/18284/","name":"Hannes Frederic Sowa","email":"hannes@stressinduktion.org"},"content":"Tom Herbert writes:\n\n> An encapsulator sets the UDP source port to be the flow entropy of the\n> packet being encapsulated. So when the packet traverses the network\n> devices can base their hash just on the canonical 5-tuple which is\n> sufficient for ECMP and RSS. IPv6 flow label is even better since the\n> middleboxes don't even need to look at the transport header, a packet\n> is steered based on the 3-tuple of addresses and flow label. In the\n> Linux stack, udp_flow_src_port is used by UDP encapsulations to set\n> the source port. Flow label is similarly set by ip6_make_flowlabel.\n> Both of these functions use the skb->hash which is computed by calling\n> flow dissector at most once per packet (if the packet was received\n> with an L4 HW hash or locally originated on a connection the hash does\n> not need to be computed).\n\nThis would require the MPLS stack copying the flowlabel of IPv6\nconnections between MPLS routers over their whole lifetime in the MPLS\nnetwork. The same would hold for MPLS encapsulated inside UDP, the\nsource port needs to be kept constant. This is very impractical. The\nhash for the flow label can often not be recomputed by interim routers,\nbecause they might lack the knowledge of the upper layer protocol.\n\nUDP source port entropy still has the problem that we don't respect the\nsource port as RSS entropy by default in network cards, because of\npossible fragmentation and thus possible reordering of packets. GRE does\nnot have this problem and is way easier to identify by hardware.\n\nBasically we need to tell network cards that they can use specific\ndestination UDP ports where we allow the source port to be used in RSS\nhash calculation. I don't see how this is any easier than just using GRE\nwith a defined protocol field? I do like the combination of ipv6\nflowlabel + GRE.\n\nBtw. people are using the GRE Key as additional entropy without looking\ninto the GRE payload.\n\nBye,\nHannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=stressinduktion.org\n\theader.i=@stressinduktion.org header.b=\"AKqIeptw\"; \n\tdkim=pass (2048-bit key;\n\tunprotected) header.d=messagingengine.com\n\theader.i=@messagingengine.com header.b=\"McyCDoyA\"; \n\tdkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xmB7H12xWz9t2c\n\tfor ;\n\tMon, 4 Sep 2017 23:50:19 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1753622AbdIDNuR (ORCPT );\n\tMon, 4 Sep 2017 09:50:17 -0400","from out1-smtp.messagingengine.com ([66.111.4.25]:49615 \"EHLO\n\tout1-smtp.messagingengine.com\" rhost-flags-OK-OK-OK-OK)\n\tby vger.kernel.org with ESMTP id S1753566AbdIDNuP (ORCPT\n\t); Mon, 4 Sep 2017 09:50:15 -0400","from compute7.internal (compute7.nyi.internal [10.202.2.47])\n\tby mailout.nyi.internal (Postfix) with ESMTP id 7087E20BE5;\n\tMon, 4 Sep 2017 09:50:15 -0400 (EDT)","from frontend1 ([10.202.2.160])\n\tby compute7.internal (MEProxy); Mon, 04 Sep 2017 09:50:15 -0400","from z.localhost.stressinduktion.org (unknown [217.192.177.51])\n\tby mail.messagingengine.com (Postfix) with ESMTPA id C89537E954;\n\tMon, 4 Sep 2017 09:50:13 -0400 (EDT)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tstressinduktion.org; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=k7Jq+qvVfyQNi64lur\n\trTXyb2lOJH3PoiZEQl7hxgLgI=; b=AKqIeptwkUqKaRBfwfob6C5ada07DDMMbP\n\t33r6zBn1j+fEQGclsk/Yc4RvEFHm1LvhYjVNVpYzv4ocEkjo32sqk9LLNDfPkIfo\n\t1r0LFpWcl7msCTLlTCM5LdEtjRG/iwYnA9u6JQ7zPzuv5EZ5v2nNsi4jFWxLTBPK\n\tNJZjXzbDcy/JSJNLHSkw/jslc9kSD2SQ0cIjUjolxJySkgql1mhIKpy4TMO9r5j8\n\t/m+tWUEyApMXx1XB/Xuum7NkXeWkTe7uBNrl1nUbAu+BClR08aCCeJHUjqXAMnZr\n\tBw5cZPr4U82KiW8+NFa6L5tkW6teIxy3J6dQpymIzzcArI+l8gdA==","v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tmessagingengine.com; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=k7Jq+qvVfyQNi64lur\n\trTXyb2lOJH3PoiZEQl7hxgLgI=; b=McyCDoyA2Y1qSFrw5/UsfM7sPsw30aXlNk\n\tkJJn2FG0AwuXizAvS5v/HwqsouXnanu7iQq4UFo8Dl9W/LCWoXQHLAqZ33xW+JBd\n\tGHUuV3uJorH7mr87q1xz4Oz5X9FS1LwD9fQqvw8gPYap5q+ace6ko0/rX1lOOIFA\n\t/GqPTWG4jyGXbEH0YvuO3HbmJL8gn7x5ERqvUKMNBw7BDFtpHOWYAFbTgtV0g2bg\n\tr0nVY/upHBiEDtsYJ0ToRBS9Z0b0qnJapIqnqZXFo1lwmAxVWnIZOCh+vyf5R+Vc\n\t6DzGTTeiqEPhnzs3tGJjGEVMqeOqqmp3ncPU1qfop1A+elIF1aYg=="],"X-ME-Sender":"","X-Sasl-enc":"xktr6nvErZtp1ZHzbWWq6ZSd4nGH3assfCy8cLdj0LWT 1504533014","From":"Hannes Frederic Sowa ","To":"Tom Herbert ","Cc":"Saeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t","Date":"Mon, 04 Sep 2017 15:50:12 +0200","In-Reply-To":"\n\t(Tom Herbert's message of \"Sun, 3 Sep 2017 08:43:27 -0700\")","Message-ID":"<87pob6bmjv.fsf@stressinduktion.org>","User-Agent":"Gnus/5.13 (Gnus v5.13) Emacs/25.2 (gnu/linux)","MIME-Version":"1.0","Content-Type":"text/plain","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762750,"web_url":"http://patchwork.ozlabs.org/comment/1762750/","msgid":"","list_archive_url":null,"date":"2017-09-04T16:15:42","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65986,"url":"http://patchwork.ozlabs.org/api/people/65986/","name":"Tom Herbert","email":"tom@herbertland.com"},"content":"On Mon, Sep 4, 2017 at 6:50 AM, Hannes Frederic Sowa\n wrote:\n> Tom Herbert writes:\n>\n>> An encapsulator sets the UDP source port to be the flow entropy of the\n>> packet being encapsulated. So when the packet traverses the network\n>> devices can base their hash just on the canonical 5-tuple which is\n>> sufficient for ECMP and RSS. IPv6 flow label is even better since the\n>> middleboxes don't even need to look at the transport header, a packet\n>> is steered based on the 3-tuple of addresses and flow label. In the\n>> Linux stack, udp_flow_src_port is used by UDP encapsulations to set\n>> the source port. Flow label is similarly set by ip6_make_flowlabel.\n>> Both of these functions use the skb->hash which is computed by calling\n>> flow dissector at most once per packet (if the packet was received\n>> with an L4 HW hash or locally originated on a connection the hash does\n>> not need to be computed).\n>\n> This would require the MPLS stack copying the flowlabel of IPv6\n> connections between MPLS routers over their whole lifetime in the MPLS\n> network. The same would hold for MPLS encapsulated inside UDP, the\n> source port needs to be kept constant. This is very impractical. The\n> hash for the flow label can often not be recomputed by interim routers,\n> because they might lack the knowledge of the upper layer protocol.\n>\nHannes,\n\nWhen the flow label is set the packet will traverse the network and be\nECMP routed regardless of whether the payload is MPLS at anything\nelse-- the important characteristic is that network devices don't need\nto know how to parse MPLS (or GRE, or IPIP, or L2TP, ESP, or ...) to\nprovide good ECMP. At a source the flow label or UDP source port needs\nto be generated. That can be based on DPI, derived from the MPLS\nentropy label, use SPI in ESP, etc. I don't see anything special about\nMPLS in this regard.\n\n> UDP source port entropy still has the problem that we don't respect the\n> source port as RSS entropy by default in network cards, because of\n> possible fragmentation and thus possible reordering of packets. GRE does\n> not have this problem and is way easier to identify by hardware.\n>\n> Basically we need to tell network cards that they can use specific\n> destination UDP ports where we allow the source port to be used in RSS\n> hash calculation. I don't see how this is any easier than just using GRE\n> with a defined protocol field? I do like the combination of ipv6\n> flowlabel + GRE.\n>\nNo, we don't any more want port specific configuration in NICs! The\nNIC should just fallback to 3-tuple hash when it see MF or offset set\nin IPv4 header. But even if it doesn't implement this, receiving OOO\nfragments is hardly the end of the world-- IP packets are always\nallowed to be received OOO. If something breaks because in order\ndelivery is assumed then that is the bug that needs to be fixed. So at\nbest handling fragmentation in this manner is proposed om\noptimization whose benefits will pale to getting good ECMP and RSS\nwhen encapsulation is in use.\n\n> Btw. people are using the GRE Key as additional entropy without looking\n> into the GRE payload.\n>\nSure some are, but the GRE key is not defined to be flow entropy so\nit's not ubiquitous it used for that so it gives sufficient entropy or\nis even constant per flow. GRE/UDP (RFC8086) was primarily written to\nallow a more consistent method (as was RFC7510 for doing MPLS/UDP).\n\nTom","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=herbertland-com.20150623.gappssmtp.com\n\theader.i=@herbertland-com.20150623.gappssmtp.com\n\theader.b=\"VVNDuX6l\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xmFM724KGz9s7m\n\tfor ;\n\tTue, 5 Sep 2017 02:15:47 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1753780AbdIDQPp (ORCPT );\n\tMon, 4 Sep 2017 12:15:45 -0400","from mail-qk0-f169.google.com ([209.85.220.169]:35257 \"EHLO\n\tmail-qk0-f169.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1753764AbdIDQPn (ORCPT\n\t); Mon, 4 Sep 2017 12:15:43 -0400","by mail-qk0-f169.google.com with SMTP id r141so3328879qke.2\n\tfor ; Mon, 04 Sep 2017 09:15:43 -0700 (PDT)","by 10.237.61.196 with HTTP; Mon, 4 Sep 2017 09:15:42 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=herbertland-com.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=oI4FAFQKHMYGvM8VpY2otttvR7uGU2raWibisvuWFys=;\n\tb=VVNDuX6lF/HkxXt5QRicJwKu9bZ8tbpXq0VLX55rW6/1HfX9FwTn1wWsjfP4lbOub3\n\taIy1ybGxBB30aunhcB5kG7pkvxmvEkm3irbqGlgJfSyc5zT5Y6H+vkFnz+wnYg+O8AWU\n\tfofOwgjkQ7vu6kNiGnnX4rN1mSjr2wOxFTtWBdMzgJwuaEMh7rv64tFSl9gDkDicvx1F\n\tSe0SOXxmwTsSCzzhFAnF9DjN/oXFEIv0Sb9i8SpuLq4r8M49lV/0Iy1x9sgO8xTfJxui\n\tHhWgcKViiy/XfGBQLTXEjzTFkIrz18UFukXuDqDnFGiNSbKcZBR35S2bwhjP5LHAnbC/\n\tFd9A==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=oI4FAFQKHMYGvM8VpY2otttvR7uGU2raWibisvuWFys=;\n\tb=XwelhapwLH4gekkto2EA5YXsHVZgzisU3hLOTYMENotT9wj2e/BMx6AMOOK1rd13zT\n\tMxKF51TqK9PwjbW+/aNaPcpZqMIvFzHYUkG3nccDiGR+B837eUx3gv3UlwS0POIBS38s\n\tEyEDyIi56zdcnxTQ8zgAxWzmIB0ZL+NKECePKeuVgEyj1J3nS/nOmhoFby60QrbcBBSl\n\tpfWZk1d+ViazdFePw1z/Hb65qREqCr1ZbpqfRcHAZiDqZH1o8rZLl4FsxENze8zzu0H5\n\tUr/HojGYwBrFK0kiyEmgpSMH/g9CTKhWwZNc8dbbDn1wBmzf9H5w2sDimIBVekTvKLpT\n\tJcdg==","X-Gm-Message-State":"AHPjjUgp2o0hsEjemWrzsQM2/R2/ChuiTp4Ac5NtRPJCNmHPR7t8Qw63\n\tMXddBC4KIpw7RvA9vW1RUFzoPaiNvS5i","X-Google-Smtp-Source":"ADKCNb60dueAAX6nsxuqfhfN3zxEmB1jkUoRdlBKSnuqQsGGTEIcx7RFNY/WeTF++M4gbVJ4nsn9ckDtZl/iyxLDWdI=","X-Received":"by 10.55.94.69 with SMTP id s66mr1314147qkb.295.1504541742952;\n\tMon, 04 Sep 2017 09:15:42 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"<87pob6bmjv.fsf@stressinduktion.org>","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>","From":"Tom Herbert ","Date":"Mon, 4 Sep 2017 09:15:42 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Hannes Frederic Sowa ","Cc":"Saeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762783,"web_url":"http://patchwork.ozlabs.org/comment/1762783/","msgid":"<8760cytnif.fsf@stressinduktion.org>","list_archive_url":null,"date":"2017-09-04T16:52:08","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":18284,"url":"http://patchwork.ozlabs.org/api/people/18284/","name":"Hannes Frederic Sowa","email":"hannes@stressinduktion.org"},"content":"Hello Tom,\n\nTom Herbert writes:\n\n> On Mon, Sep 4, 2017 at 6:50 AM, Hannes Frederic Sowa\n> wrote:\n>> Tom Herbert writes:\n>>\n>>> An encapsulator sets the UDP source port to be the flow entropy of the\n>>> packet being encapsulated. So when the packet traverses the network\n>>> devices can base their hash just on the canonical 5-tuple which is\n>>> sufficient for ECMP and RSS. IPv6 flow label is even better since the\n>>> middleboxes don't even need to look at the transport header, a packet\n>>> is steered based on the 3-tuple of addresses and flow label. In the\n>>> Linux stack, udp_flow_src_port is used by UDP encapsulations to set\n>>> the source port. Flow label is similarly set by ip6_make_flowlabel.\n>>> Both of these functions use the skb->hash which is computed by calling\n>>> flow dissector at most once per packet (if the packet was received\n>>> with an L4 HW hash or locally originated on a connection the hash does\n>>> not need to be computed).\n>>\n>> This would require the MPLS stack copying the flowlabel of IPv6\n>> connections between MPLS routers over their whole lifetime in the MPLS\n>> network. The same would hold for MPLS encapsulated inside UDP, the\n>> source port needs to be kept constant. This is very impractical. The\n>> hash for the flow label can often not be recomputed by interim routers,\n>> because they might lack the knowledge of the upper layer protocol.\n>>\n> Hannes,\n>\n> When the flow label is set the packet will traverse the network and be\n> ECMP routed regardless of whether the payload is MPLS at anything\n> else-- the important characteristic is that network devices don't need\n> to know how to parse MPLS (or GRE, or IPIP, or L2TP, ESP, or ...) to\n> provide good ECMP. At a source the flow label or UDP source port needs\n> to be generated. That can be based on DPI, derived from the MPLS\n> entropy label, use SPI in ESP, etc. I don't see anything special about\n> MPLS in this regard.\n\nThe MPLS circuit is only end to end in terms of IP processing if MPLS is\nused for multitenant separation.\n\nNormally the IP connection is done between two label switch routers,\nthus is not end to end. One LSR will decapsulate the packet and throw\nthe IP header away, do the label processing and will reencapsulate it\nwith the new next hop information. To keep the assigned entropy alive it\nwould have to save the UDP source port or flowlabel and patch the\noutgoing IP header again. This is certainly possible it just seems more\nunnatural.\n\nNormally every next hop does MPLS processing and thus the packet\ntraverses up the stack. Special purpose (entropy) MPLS labels allow the\nstack to achieve RSS just based on the label stack and will be\nend-to-end in a MPLS cloud.\n\n>> UDP source port entropy still has the problem that we don't respect the\n>> source port as RSS entropy by default in network cards, because of\n>> possible fragmentation and thus possible reordering of packets. GRE does\n>> not have this problem and is way easier to identify by hardware.\n>>\n>> Basically we need to tell network cards that they can use specific\n>> destination UDP ports where we allow the source port to be used in RSS\n>> hash calculation. I don't see how this is any easier than just using GRE\n>> with a defined protocol field? I do like the combination of ipv6\n>> flowlabel + GRE.\n>>\n> No, we don't any more want port specific configuration in NICs! The\n> NIC should just fallback to 3-tuple hash when it see MF or offset set\n> in IPv4 header. But even if it doesn't implement this, receiving OOO\n> fragments is hardly the end of the world-- IP packets are always\n> allowed to be received OOO. If something breaks because in order\n> delivery is assumed then that is the bug that needs to be fixed. So at\n> best handling fragmentation in this manner is proposed om\n> optimization whose benefits will pale to getting good ECMP and RSS\n> when encapsulation is in use.\n\nThe problem is that you end up having two streams, one fragmented and\none non-fragmented, but actually they belong to the same stream. It is\nknown to break stuff, see:\n\n\n\nI would agree with you, but we can't break existing setups,\nunfortunately.\n\n>> Btw. people are using the GRE Key as additional entropy without looking\n>> into the GRE payload.\n>>\n> Sure some are, but the GRE key is not defined to be flow entropy so\n> it's not ubiquitous it used for that so it gives sufficient entropy or\n> is even constant per flow. GRE/UDP (RFC8086) was primarily written to\n> allow a more consistent method (as was RFC7510 for doing MPLS/UDP).\n\nI agree with that, just wanted to mention it.\n\nBye,\nHannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=stressinduktion.org\n\theader.i=@stressinduktion.org header.b=\"SqDmzSy3\"; \n\tdkim=pass (2048-bit key;\n\tunprotected) header.d=messagingengine.com\n\theader.i=@messagingengine.com header.b=\"aa+RxI8Y\"; \n\tdkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xmG9C02CTz9sR9\n\tfor ;\n\tTue, 5 Sep 2017 02:52:15 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1753941AbdIDQwM (ORCPT );\n\tMon, 4 Sep 2017 12:52:12 -0400","from out1-smtp.messagingengine.com ([66.111.4.25]:38277 \"EHLO\n\tout1-smtp.messagingengine.com\" rhost-flags-OK-OK-OK-OK)\n\tby vger.kernel.org with ESMTP id S1753847AbdIDQwL (ORCPT\n\t); Mon, 4 Sep 2017 12:52:11 -0400","from compute7.internal (compute7.nyi.internal [10.202.2.47])\n\tby mailout.nyi.internal (Postfix) with ESMTP id B35D220EB0;\n\tMon, 4 Sep 2017 12:52:10 -0400 (EDT)","from frontend1 ([10.202.2.160])\n\tby compute7.internal (MEProxy); Mon, 04 Sep 2017 12:52:10 -0400","from z.localhost.stressinduktion.org (unknown [185.72.66.238])\n\tby mail.messagingengine.com (Postfix) with ESMTPA id D8B437F9D2;\n\tMon, 4 Sep 2017 12:52:09 -0400 (EDT)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tstressinduktion.org; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=Y0WxD8DAtxCGm3Mavq\n\t6MoSnmz3/56wkpF5WNfsrGF3c=; b=SqDmzSy3Wd/ZU0YfZQeL7RZwfSnenwUgmN\n\tq2YvTg8E62rcqCqbZ5pphwMs246JRAtDJU3NU8JwWaulG+GIeTdrpc7Fw821bdoZ\n\tALAa+/3xlpjy/FTAvaWQhRLhPQqYp+Sy7UUzurySdj3dkwOCWeGXgKpssvNG0q62\n\t73JAnpDNknI/Er7b+zL8knlA+C4U0iuoSU/RVnjVWCg2LRj2c8oHr54IaTTcoTh2\n\t7TAER5U/uLXnFoWwuc+i8+HPZ2lgDNdRkaIHUTgZ3aWATWFra1ovyhtpXFewdVWw\n\t2XYtH2mVzx8nNRP+1dpWrAUzL51eUBY9ucblv70vsX/giob5jKIg==","v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tmessagingengine.com; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=Y0WxD8DAtxCGm3Mavq\n\t6MoSnmz3/56wkpF5WNfsrGF3c=; b=aa+RxI8Y9givZuBUeweqlIKzAz83+j7ZPo\n\tggQDZzPeXjR4qyWMK4z1Q5rgk7mAGeAJGcJperBoR/wu01DvRvX0ybCzikI9VSKT\n\tADr8TnUtXMTyK724hFwTuNrctvhHZzfr9gknjrH2QgRMkawtyp+X2KRqumH1Vv/o\n\t/louR3QDTqC/u8WErk1prB1dFoMS3XIl5y8NU8eeuFiqQsTbGbDhJldpcGKBpiCY\n\tN0MG6yn02NUKboQ4jPbkvv9SiUrlg/vCVpk1RpX3nDB//QzOv+3HnrTjsZZc6B6G\n\tm0lHRHTCppbA55ZaHDzmPsn1TuI2+iq7rghWHeaDRe78y1ctZr4g=="],"X-ME-Sender":"","X-Sasl-enc":"ml/GiP4AeU06D2r8pJtkOQqIxaHNwl2iAPiong4MNmPt 1504543930","From":"Hannes Frederic Sowa ","To":"Tom Herbert ","Cc":"Saeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>\n\t","Date":"Mon, 04 Sep 2017 18:52:08 +0200","In-Reply-To":"\n\t(Tom Herbert's message of \"Mon, 4 Sep 2017 09:15:42 -0700\")","Message-ID":"<8760cytnif.fsf@stressinduktion.org>","User-Agent":"Gnus/5.13 (Gnus v5.13) Emacs/25.2 (gnu/linux)","MIME-Version":"1.0","Content-Type":"text/plain","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762795,"web_url":"http://patchwork.ozlabs.org/comment/1762795/","msgid":"","list_archive_url":null,"date":"2017-09-04T17:11:21","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65986,"url":"http://patchwork.ozlabs.org/api/people/65986/","name":"Tom Herbert","email":"tom@herbertland.com"},"content":"> The problem is that you end up having two streams, one fragmented and\n> one non-fragmented, but actually they belong to the same stream. It is\n> known to break stuff, see:\n>\n> \n>\n> I would agree with you, but we can't break existing setups,\n> unfortunately.\n>\nI'm not sure what \"existing setups\" means here. If this is referring\nto the Internet then out of order packets and fragments abound-- any\nassumption of in order delivery for IP packets is useless there.\n\nBtw, TCP has exactly the same problem in this regard that UDP has with\nregard to fragmentation. The reason that TCP isn't considered an issue\nis the assumption that TCP will do PMTU discovery and set DF (I would\nbet that devices don't actually check for DF and vendors don't test\nwhen it's not set!).\n\nTom","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=herbertland-com.20150623.gappssmtp.com\n\theader.i=@herbertland-com.20150623.gappssmtp.com\n\theader.b=\"nrE+/8Kq\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xmGbP3hSRz9t16\n\tfor ;\n\tTue, 5 Sep 2017 03:11:29 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1753890AbdIDRLY (ORCPT );\n\tMon, 4 Sep 2017 13:11:24 -0400","from mail-qt0-f173.google.com ([209.85.216.173]:35210 \"EHLO\n\tmail-qt0-f173.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1753910AbdIDRLX (ORCPT\n\t); Mon, 4 Sep 2017 13:11:23 -0400","by mail-qt0-f173.google.com with SMTP id k2so4082411qte.2\n\tfor ; Mon, 04 Sep 2017 10:11:23 -0700 (PDT)","by 10.237.61.196 with HTTP; Mon, 4 Sep 2017 10:11:21 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=herbertland-com.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=hSgImcOgIMEb8jE5i5oy1irkYF2WrStbUvBSoC2pST8=;\n\tb=nrE+/8KqcAH6epzAYpzor7adNi1Xr6L/xWKNJnQwRMBqVGEIKOkuU1hTkINUSi8U33\n\t+HaDKurw7RIY6QIh4X+Zx6tIc0tfEstgqeaDXHZXQi3NHy1kJ7I10F5JN64xvdpY7M5f\n\t0yPpMX+msCLU4uoBks7ovfgVw/ofOZCv3dkGLoYgTXwJeFmS9H99wPZvzDNZKsb0WIvX\n\tly1RjoOld1auMCuf5JUsNURzdC608yPi3aem4GR/ZGW6sb4cOzdEtIoH1iuYSPYdqEs1\n\tkwuOdOm3U8OJWHb6pWgW9RoiMcBwdpIy/qn6pnRCQwZ0E+nCFk+cyLEdVULcagLcKd9j\n\tsG/g==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=hSgImcOgIMEb8jE5i5oy1irkYF2WrStbUvBSoC2pST8=;\n\tb=txZuun3AZ/3C59uw8nVg3UVxoZUi5idUR1U2uWhS3Rkf4fUTh2o7G/X+YnFmQlEV3S\n\tQZnPsOS+P9HoIBJ7L6ed50OeCOUhm+B+9FQgAlq0lmHrxzWFNr4qyk+JpbKDbFihIqnd\n\tXvVLB0sLkO6niVDaZiviY6jRAguhT/A0K3soMrzR2M+2PiD8YgG5yuWEgK09ZEL6OS3Q\n\tgLmbhUraBIdc+tCkAo19K7IopOD60P6vG9HfIBsrsXtQbilyZFuJFa80xztUJWhMPXw4\n\tGfq+x0mF4QUSoVm7B95SyTgIFWoLfwtU1kbJ2p8Nj83faTAtDdN5os1/llVx7QDVmngQ\n\tSyOw==","X-Gm-Message-State":"AHPjjUhvoBsbPoMYQ8cPYj71Nhyw39tQoRe2FsqN0e2qGz8D/heXJ9Ou\n\ttBtOVvqM9b4xAx92UOwaGsnMGeY5+mlW","X-Google-Smtp-Source":"ADKCNb5FNFZKwnPeGK/nkmppVFE6Ya1fxAQunPZSdjEOdJnoHgzZFxNtyreCpJSld3EAjgpSLpz8BlKJV1rXH2IZVFk=","X-Received":"by 10.200.42.243 with SMTP id c48mr1503289qta.205.1504545082470; \n\tMon, 04 Sep 2017 10:11:22 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"<8760cytnif.fsf@stressinduktion.org>","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>\n\t\n\t<8760cytnif.fsf@stressinduktion.org>","From":"Tom Herbert ","Date":"Mon, 4 Sep 2017 10:11:21 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Hannes Frederic Sowa ","Cc":"Saeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762823,"web_url":"http://patchwork.ozlabs.org/comment/1762823/","msgid":"<87k21ez6r9.fsf@stressinduktion.org>","list_archive_url":null,"date":"2017-09-04T17:57:30","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":18284,"url":"http://patchwork.ozlabs.org/api/people/18284/","name":"Hannes Frederic Sowa","email":"hannes@stressinduktion.org"},"content":"Hi Tom,\n\nTom Herbert writes:\n\n>> The problem is that you end up having two streams, one fragmented and\n>> one non-fragmented, but actually they belong to the same stream. It is\n>> known to break stuff, see:\n>>\n>> \n>>\n>> I would agree with you, but we can't break existing setups,\n>> unfortunately.\n>>\n> I'm not sure what \"existing setups\" means here. If this is referring\n> to the Internet then out of order packets and fragments abound-- any\n> assumption of in order delivery for IP packets is useless there.\n\nNetwork cards don't know where traffic originates from, even on the same\ncard. Clouds nowadays try to convince everyone to virtualize existing\nworkloads. So I *assume* that at least for traffic that seems to be in\none L2 domain out of order should not occur or things will break.\n\nEthernet for a long time appeared to do very much in order delivery and\nguarantees that. Thus for people it appeared that UDP packets are also\nstrictly in order. Encapsulating Ethernet inside UDP thus must preserve\nthose properties, especially if used for virtualization. Unfortunately\nfragmentation happens and the stack has to deal with it somehow.\n\nI do know about some software that uses UDP in multicast that is prone\nto misbehave in case of OoO frames. It uses a small reassemble queue but\nif reordering gets too high, it will simply drop data. This might be\nacceptable up to a specific degree.\n\nI guess one application you could harm is plain old simple syslog, which\noften generated fragmented packets. Luckily one often has time stamps in\nthere to reassemble the log lines but one had to do this manually.\n\nL2 domains are not bound to local networks anymore, thanks to vxlan etc.\n\nIf you are in a controlled environment and you do know your software\nthat runs perfectly well, certainly, no doubt, UDP hashing can be\nenabled. I would argue we can't do that generally.\n\n> Btw, TCP has exactly the same problem in this regard that UDP has with\n> regard to fragmentation. The reason that TCP isn't considered an issue\n> is the assumption that TCP will do PMTU discovery and set DF (I would\n> bet that devices don't actually check for DF and vendors don't test\n> when it's not set!).\n\nI don't know.\n\nFor the application the stream will appear in order at the socket level\nand OoO or fragmentation won't break anything. End systems will have a\nharder time reassembling the stream and thus fast paths couldn't be\nused. Is that what you are referring to?\n\nThanks,\nHannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=stressinduktion.org\n\theader.i=@stressinduktion.org header.b=\"AFQxTsrS\"; \n\tdkim=pass (2048-bit key;\n\tunprotected) header.d=messagingengine.com\n\theader.i=@messagingengine.com header.b=\"EAi87iVI\"; \n\tdkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xmHcc2yMpz9s7h\n\tfor ;\n\tTue, 5 Sep 2017 03:57:36 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1753961AbdIDR5e (ORCPT );\n\tMon, 4 Sep 2017 13:57:34 -0400","from out1-smtp.messagingengine.com ([66.111.4.25]:38869 \"EHLO\n\tout1-smtp.messagingengine.com\" rhost-flags-OK-OK-OK-OK)\n\tby vger.kernel.org with ESMTP id S1753949AbdIDR5d (ORCPT\n\t); Mon, 4 Sep 2017 13:57:33 -0400","from compute7.internal (compute7.nyi.internal [10.202.2.47])\n\tby mailout.nyi.internal (Postfix) with ESMTP id AAC1E20E7D;\n\tMon, 4 Sep 2017 13:57:32 -0400 (EDT)","from frontend2 ([10.202.2.161])\n\tby compute7.internal (MEProxy); Mon, 04 Sep 2017 13:57:32 -0400","from z.localhost.stressinduktion.org (unknown [185.72.66.238])\n\tby mail.messagingengine.com (Postfix) with ESMTPA id BF609243E0;\n\tMon, 4 Sep 2017 13:57:31 -0400 (EDT)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tstressinduktion.org; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=eoLVLMUsvk6d5b7KVb\n\tmGOGi7Hlxn9Te9M38oD+VH7Sw=; b=AFQxTsrSrVbmSp37asXc59C/vBIVa59kT2\n\tYSwH/cFARMlWQ7ON3I1zPpliNGGkt6b88zDyn6MRAzolQVusp1EBi1A6QddGqO/X\n\tuY8JEMKKXcNcR9ejw+CfPkvy0zpQwIAv1iUO4XUmG9k99yBQSKzChhgstACuod59\n\tCBFHOYLYYl0krFmspeIX02Ufj6AFutq9hpRQuwSgW2uxp1MTFUPT0logNJHex4e3\n\tAo3rIWv4Tpk0efGc79NGte05kaP4rYRyLtVBKufIZwf8ySIm7lf76PaTBYaeUb79\n\tRjObj7MGdSTzQVw0yDCa8jZvUR2y82sYolcOhWTKwQ6zrSsVFhHQ==","v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tmessagingengine.com; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=eoLVLMUsvk6d5b7KVb\n\tmGOGi7Hlxn9Te9M38oD+VH7Sw=; b=EAi87iVIDJAeFbVmFxiXdSkx2zb5kWKm19\n\tTKMtEuEvv2BQUzNrBNOMuNZQixUyxuOBuOTnuKySv42qu7+n5Lb+DZ6J+m3WR2pZ\n\taVigOqT0gnE+YLDxKTaEmYdGtSGhZtuijfq1tR5+XjzCDLhGrFfQteUkxASp6+0p\n\tlM/UO8N946t10HQelKiW8GbVzgHae1b4s0WGnbaemj9ovlBa87Dd8dVJMxJqFr8p\n\tZBPDH9Ggak7B+/BT8dFbPRESeT1qJs3+ElvsvnoTQtj5aAY8WcKYYp/xx02emtx0\n\tJDOE2vkWzNlVUDfRqqFlr7CotnQboBv6uoWoNhukhqxMMo0hNKLA=="],"X-ME-Sender":"","X-Sasl-enc":"B0OVdByYhNUkt51vha4fBX6/Wce6wdF/wcuS79GIPbc3 1504547852","From":"Hannes Frederic Sowa ","To":"Tom Herbert ","Cc":"Saeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>\n\t\n\t<8760cytnif.fsf@stressinduktion.org>\n\t","Date":"Mon, 04 Sep 2017 19:57:30 +0200","In-Reply-To":"\n\t(Tom Herbert's message of \"Mon, 4 Sep 2017 10:11:21 -0700\")","Message-ID":"<87k21ez6r9.fsf@stressinduktion.org>","User-Agent":"Gnus/5.13 (Gnus v5.13) Emacs/25.2 (gnu/linux)","MIME-Version":"1.0","Content-Type":"text/plain","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1762841,"web_url":"http://patchwork.ozlabs.org/comment/1762841/","msgid":"","list_archive_url":null,"date":"2017-09-04T18:56:00","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65986,"url":"http://patchwork.ozlabs.org/api/people/65986/","name":"Tom Herbert","email":"tom@herbertland.com"},"content":"On Mon, Sep 4, 2017 at 10:57 AM, Hannes Frederic Sowa\n wrote:\n> Hi Tom,\n>\n> Tom Herbert writes:\n>\n>>> The problem is that you end up having two streams, one fragmented and\n>>> one non-fragmented, but actually they belong to the same stream. It is\n>>> known to break stuff, see:\n>>>\n>>> \n>>>\n>>> I would agree with you, but we can't break existing setups,\n>>> unfortunately.\n>>>\n>> I'm not sure what \"existing setups\" means here. If this is referring\n>> to the Internet then out of order packets and fragments abound-- any\n>> assumption of in order delivery for IP packets is useless there.\n>\n> Network cards don't know where traffic originates from, even on the same\n> card. Clouds nowadays try to convince everyone to virtualize existing\n> workloads. So I *assume* that at least for traffic that seems to be in\n> one L2 domain out of order should not occur or things will break.\n>\n> Ethernet for a long time appeared to do very much in order delivery and\n> guarantees that. Thus for people it appeared that UDP packets are also\n> strictly in order. Encapsulating Ethernet inside UDP thus must preserve\n> those properties, especially if used for virtualization. Unfortunately\n> fragmentation happens and the stack has to deal with it somehow.\n>\nThere is absolutely no requirement in IP that packets are delivered in\norder-- there never has been and there never will be! If the ULP, like\nEthernet encapsulation, requires in order deliver then it needs to\nimplement that itself like TCP, GRE, and other protocols ensure that\nwith sequence numbers and reassembly. All of these hoops we do make\nsure that packets always follow the same path and are always in order\nare done for benefit of middlebox devices like stateful firewalls that\nhave force us to adopt their concept of network architecture-- in the\nlong run this is self-defeating and kills our ability to innovate.\n\nI'm not saying that we shouldn't consider legacy devices, but we\nshould scrutinize new development or solutions that perpetuate\nincorrect design or bad assumptions.\n\nTom","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=herbertland-com.20150623.gappssmtp.com\n\theader.i=@herbertland-com.20150623.gappssmtp.com\n\theader.b=\"lxscctdL\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xmJw74jDqz9s7c\n\tfor ;\n\tTue, 5 Sep 2017 04:56:07 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1753974AbdIDS4D (ORCPT );\n\tMon, 4 Sep 2017 14:56:03 -0400","from mail-qk0-f178.google.com ([209.85.220.178]:36530 \"EHLO\n\tmail-qk0-f178.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1753953AbdIDS4B (ORCPT\n\t); Mon, 4 Sep 2017 14:56:01 -0400","by mail-qk0-f178.google.com with SMTP id z143so4603023qkb.3\n\tfor ; Mon, 04 Sep 2017 11:56:01 -0700 (PDT)","by 10.237.61.196 with HTTP; Mon, 4 Sep 2017 11:56:00 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=herbertland-com.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=og8hu1U/Shqeu2OQfhyKdJnqwSnu1/r9/+Bvy0ffmA0=;\n\tb=lxscctdLkdwDA5gd3Rvw2fhECJtmPua18h4Mz5xURj93BOhY5wuiJhzST82w4bT/Pn\n\tT/V8j0pzyBYSb1Q5TqlPrtR/6J9lzR427L2yRnVCKUlPgM29FeA5pIHAtFmtXTHdxv2w\n\t3HNt8cRlC+JaPmvUYxCAFT+pjOYixHs0NRQvFQIQNt61x8Ce+JfKsJnVfHrFGiW2VvKE\n\t0VjoVEuk17acOel9SHYa84jOmXa7veqY4FyzxHoaaKChGlzlPHaDlaecIrSh3n10/yCU\n\t63kyJ1/TYgpCPXJv/vwcqf3W3XSARHu6BYJz5YrtH/K0ypD/zqd+x2awQmLB7ryPDesG\n\tp3DQ==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=og8hu1U/Shqeu2OQfhyKdJnqwSnu1/r9/+Bvy0ffmA0=;\n\tb=KN74UDFwhDYRWJp4SzopdDegWvo9hbr4zvcVFfEw85Lh8eqxOU0qiTQ5aNgkAPHVOi\n\tDR8Uo+aFrDq2PN3MnoRuxms7fhX2bVQWFl2UoYrdWCzNNYMHxGmVXfLoZFxrzq6cVU95\n\tX8c+t5mUGmbeWpreYG/3UuRYVRzNATHXQxDYy39MCIayfCJvsmJw0/3LRSC1RWPIQWYV\n\tdZbF/nKCxyLxCsOVx/9Xs4TtelxN0iUL7VDu80kH3RHLHhgQR3MDBK1czcBppiTZEaPD\n\tu+8QkblKMsLIFMm/G97EIt/Xs19HfcJjNu0Vi5U3VsoCju5igQJEery9GAGi9wkJU7UM\n\tyg6Q==","X-Gm-Message-State":"AHPjjUisW37arxL8NtjSyK+dnh3FNvVE/UWkDFJtk87HvV2Fq1f6MIYk\n\tVarHNOArMTKevf8XS9y3YPtjz7hzBWvC","X-Google-Smtp-Source":"ADKCNb7zbT0SblbthEll88vkUc8jjAMpwNqqCt0P/eNDwYTTKA9cAqbJNSq4YeZFk23NhbfhbPV9HJOEYlhIfHbOr2o=","X-Received":"by 10.55.94.69 with SMTP id s66mr1860778qkb.295.1504551361168;\n\tMon, 04 Sep 2017 11:56:01 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"<87k21ez6r9.fsf@stressinduktion.org>","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>\n\t\n\t<8760cytnif.fsf@stressinduktion.org>\n\t\n\t<87k21ez6r9.fsf@stressinduktion.org>","From":"Tom Herbert ","Date":"Mon, 4 Sep 2017 11:56:00 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Hannes Frederic Sowa ","Cc":"Saeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1763230,"web_url":"http://patchwork.ozlabs.org/comment/1763230/","msgid":"<87r2vls8hp.fsf@stressinduktion.org>","list_archive_url":null,"date":"2017-09-05T11:14:10","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":18284,"url":"http://patchwork.ozlabs.org/api/people/18284/","name":"Hannes Frederic Sowa","email":"hannes@stressinduktion.org"},"content":"Tom Herbert writes:\n\n> There is absolutely no requirement in IP that packets are delivered in\n> order-- there never has been and there never will be! If the ULP, like\n> Ethernet encapsulation, requires in order deliver then it needs to\n> implement that itself like TCP, GRE, and other protocols ensure that\n> with sequence numbers and reassembly. All of these hoops we do make\n> sure that packets always follow the same path and are always in order\n> are done for benefit of middlebox devices like stateful firewalls that\n> have force us to adopt their concept of network architecture-- in the\n> long run this is self-defeating and kills our ability to innovate.\n>\n> I'm not saying that we shouldn't consider legacy devices, but we\n> should scrutinize new development or solutions that perpetuate\n> incorrect design or bad assumptions.\n\nSo configure RSS per port and ensure no fragments are send to those\nports. This is possible and rather easy to do. It solves the problem\nwith legacy software and it spreads out packets for your applications.\n\nIt is not perfect but it is working and solves both problems.\n\nBye,\nHannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=stressinduktion.org\n\theader.i=@stressinduktion.org header.b=\"laLckvzq\"; \n\tdkim=pass (2048-bit key;\n\tunprotected) header.d=messagingengine.com\n\theader.i=@messagingengine.com header.b=\"LVDyUFqQ\"; \n\tdkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xmkcq1SGpz9sPk\n\tfor ;\n\tTue, 5 Sep 2017 21:14:19 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1750916AbdIELOQ (ORCPT );\n\tTue, 5 Sep 2017 07:14:16 -0400","from out1-smtp.messagingengine.com ([66.111.4.25]:56071 \"EHLO\n\tout1-smtp.messagingengine.com\" rhost-flags-OK-OK-OK-OK)\n\tby vger.kernel.org with ESMTP id S1750755AbdIELON (ORCPT\n\t); Tue, 5 Sep 2017 07:14:13 -0400","from compute7.internal (compute7.nyi.internal [10.202.2.47])\n\tby mailout.nyi.internal (Postfix) with ESMTP id 54B6D20C07;\n\tTue, 5 Sep 2017 07:14:13 -0400 (EDT)","from frontend2 ([10.202.2.161])\n\tby compute7.internal (MEProxy); Tue, 05 Sep 2017 07:14:13 -0400","from z.localhost.stressinduktion.org (unknown [217.192.177.51])\n\tby mail.messagingengine.com (Postfix) with ESMTPA id BED70240AF;\n\tTue, 5 Sep 2017 07:14:11 -0400 (EDT)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tstressinduktion.org; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=ZVAJmGnUPHOxg+I3uO\n\t1rjJu0uDBmKzJOFqDEsgll9tQ=; b=laLckvzqr4rA70ihfkbbsTCLHKH4nsd5eI\n\tsIStNwy8g/tnCfX6HyGy9UIrkykf/iGy3wXA0VbTp7IXDZ/YH0VaYoTdEdotq8lY\n\twwA4PfjrfNTXP+Bumpl9mwHH+uheHlLUM2foMmI1wnguGMVp0pZs30SA42J53UY4\n\tLw7LeZHLwZfQ3rSiDsdWmojVdR3cGj2bVZcIZj4EtyhdGOR5xXsEJqwR4eVdPJji\n\tHHgW0uN1tf22iF+k3PUSfpydjnV4zoJOCTfcyvgBmHqUFmIFFmNg02p02mjoLoOb\n\tuukj1Ih+UPlf0EqgmbSrs9oYtJL9NEtnooH6VeXyUFtWPCF5tPWA==","v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tmessagingengine.com; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=ZVAJmGnUPHOxg+I3uO\n\t1rjJu0uDBmKzJOFqDEsgll9tQ=; b=LVDyUFqQe+eKNqUhx+cNW7EFwtMTaYsxFy\n\teQOnE3siSubmj8JH0qAyKNvDfOcXgtRIjIBPkOrAwyMVmNga5kCO5cI00LWTLok0\n\tac/XxyWQ1fF0j941hJwZjqTojuImSoRIS8Xbydwb+OGPh3oQ6nNb/911NyxSfVOg\n\t2fv8kTnwCF3D2KrJmCFJ6szp22WdenZhNVaxseCHyrgj5I2SirjSJOVepcYyZcXC\n\tLqeB2L9GBZg0P04E3ZvbtOwCq25X1fAw7Pkfy7L6tnPp180M+pnEDlGPa4X6DqFH\n\tEGFjdDnZ8VrnWaNNFp9uXwaMv6xu3RP+lRg38zIfSVrqcmm/dVbQ=="],"X-ME-Sender":"","X-Sasl-enc":"JS2uy6isdXWjN6h6FVIx59gpxzx95O0NQRIhTnCxo/ZT 1504610052","From":"Hannes Frederic Sowa ","To":"Tom Herbert ","Cc":"Saeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>\n\t\n\t<8760cytnif.fsf@stressinduktion.org>\n\t\n\t<87k21ez6r9.fsf@stressinduktion.org>\n\t","Date":"Tue, 05 Sep 2017 13:14:10 +0200","In-Reply-To":"\n\t(Tom Herbert's message of \"Mon, 4 Sep 2017 11:56:00 -0700\")","Message-ID":"<87r2vls8hp.fsf@stressinduktion.org>","User-Agent":"Gnus/5.13 (Gnus v5.13) Emacs/25.2 (gnu/linux)","MIME-Version":"1.0","Content-Type":"text/plain","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1763468,"web_url":"http://patchwork.ozlabs.org/comment/1763468/","msgid":"","list_archive_url":null,"date":"2017-09-05T16:02:33","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65986,"url":"http://patchwork.ozlabs.org/api/people/65986/","name":"Tom Herbert","email":"tom@herbertland.com"},"content":"On Tue, Sep 5, 2017 at 4:14 AM, Hannes Frederic Sowa\n wrote:\n> Tom Herbert writes:\n>\n>> There is absolutely no requirement in IP that packets are delivered in\n>> order-- there never has been and there never will be! If the ULP, like\n>> Ethernet encapsulation, requires in order deliver then it needs to\n>> implement that itself like TCP, GRE, and other protocols ensure that\n>> with sequence numbers and reassembly. All of these hoops we do make\n>> sure that packets always follow the same path and are always in order\n>> are done for benefit of middlebox devices like stateful firewalls that\n>> have force us to adopt their concept of network architecture-- in the\n>> long run this is self-defeating and kills our ability to innovate.\n>>\n>> I'm not saying that we shouldn't consider legacy devices, but we\n>> should scrutinize new development or solutions that perpetuate\n>> incorrect design or bad assumptions.\n>\n> So configure RSS per port and ensure no fragments are send to those\n> ports. This is possible and rather easy to do. It solves the problem\n> with legacy software and it spreads out packets for your applications.\n>\n> It is not perfect but it is working and solves both problems.\n>\nHannes,\n\nI don't see how that solves anything. The purpose of RSS is to\ndistribute the load of protocol packets across queues. This needs to\nwork for UDP applications. For instance, if I were building a QUIC\nserver I'd want the sort of flow distribution that a TCP server would\ngive. You can't do that by configuring a few ports in the device.\n\nIf I were to suggest any HW change it would be to not do DPI on\nfragments (MF or offset is set). This ensures that all packets of the\nfragment train get hashed to the same queue and is on fact what RPS\nhas been doing for years without any complaints.\n\nBut even before I'd make that recommendation, I'd really like\nunderstand what the problem actually is. The only thing I can garner\nfrom this discussion and the Intel patch is that when fragments are\nreceived OOO that is perceived as a problem. But the by the protocol\nspecification clearly says this is not a problem. So the questions\nare: who is negatively affected by this? Is this a problem because\nsome artificial test that checks for everything to be in order is now\nfailing? Is this affecting real users? Is this an issue in the stack\nor really with some implementation outside of the stack? If it is an\nimplementation outside of the stack, then are we just bandaid'ing over\nsomeone else's incorrect implementation by patching the kernel (like\nwould have be the case if we change the kernel to interoperate with\nFacebook's switch that couldn't handle OOO in twstate).\n\nThanks,\nTom\n\n> Bye,\n> Hannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=herbertland-com.20150623.gappssmtp.com\n\theader.i=@herbertland-com.20150623.gappssmtp.com\n\theader.b=\"KuETupJc\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xms1W3kHSz9t16\n\tfor ;\n\tWed, 6 Sep 2017 02:02:39 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1752034AbdIEQCh (ORCPT );\n\tTue, 5 Sep 2017 12:02:37 -0400","from mail-qk0-f172.google.com ([209.85.220.172]:35656 \"EHLO\n\tmail-qk0-f172.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1751979AbdIEQCg (ORCPT\n\t); Tue, 5 Sep 2017 12:02:36 -0400","by mail-qk0-f172.google.com with SMTP id r141so12861832qke.2\n\tfor ; Tue, 05 Sep 2017 09:02:36 -0700 (PDT)","by 10.237.61.196 with HTTP; Tue, 5 Sep 2017 09:02:33 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=herbertland-com.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=KmR/gRyj+SejLJkhFYGH5shptaRgP4gBWzcn/p4YgaY=;\n\tb=KuETupJcqvsgzWOveXRyqBRATvWeh/iyBYmFPH1ToPqSEOTyxihrzdjM2IAJ0++zq9\n\tYPl2sv0onwaWwFJPxrA3FTdwN1MCnpJ+HYTvpQJPM3kdsCgYhmef/zoTweuxkXQok3Gm\n\tbc0lkn5IyxWLANtzzPb0EzC7LA+310GYFYX/mhsINmehB0oP7cTQXcOmMUJgfatguVdH\n\tCrizVxWIScwn7zus/JOCiWoHWxkw8RSwqH1XiaAgkIWsuRsqIs/zZcXFfO4lAc037+IA\n\t5m9pikr2jIej0BIjQmelGtJvizdcAwYIR8Cp3Ovtn/5KgE97Gi3mbTQVUHvvMNIYiBad\n\tnanA==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=KmR/gRyj+SejLJkhFYGH5shptaRgP4gBWzcn/p4YgaY=;\n\tb=TSK9hslpPczbJit+DJsiF3ps75ij6q8Fyp7u3wxrCpr7MkX3oF18S15oYEoIpCP4No\n\tCsxqsZ8N/Pdq+cXybllk7yrRiWU+MIphrUblJFBYjDDaJ+JCwqmQ9HO8oXocYmdUzJHO\n\tdE9Wk2namsJ7DOZlRPV3dQTM89Q4SctGCY2UoBdHUYuDCUob5hAPJl/JncZ3/3lBiPsC\n\t+L2rF+dqLkaVZkn5uJmtnnq0AZdhmxKTf9mvWoFg514UtwDcLP1nX+spNDNSzTYwTw2o\n\t+2KJbd/2bJacC3OaVoKuYn+CSp6WH44U6S9E9IJ5pmUaCEcNiWnuPoKBf/rryC2ZN21/\n\tlUaA==","X-Gm-Message-State":"AHPjjUiUOI/tBYLd9MaK+tLZk3mqHE2Ih2e2Ntb1OS3EzO6hfY9yBJge\n\tHzg12myilV3hJU7jPDAqQ3dASWkuAvNS","X-Google-Smtp-Source":"ADKCNb6EsfgItM32NH3fp6Jvp7haohuyDGxazLyEB0yh1AbPvot3DZ00/Y7jqPEZ2a4kDYFrI1RzHUPeBxZN2Y6SxoE=","X-Received":"by 10.55.48.71 with SMTP id w68mr5424425qkw.345.1504627354729;\n\tTue, 05 Sep 2017 09:02:34 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"<87r2vls8hp.fsf@stressinduktion.org>","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>\n\t\n\t<8760cytnif.fsf@stressinduktion.org>\n\t\n\t<87k21ez6r9.fsf@stressinduktion.org>\n\t\n\t<87r2vls8hp.fsf@stressinduktion.org>","From":"Tom Herbert ","Date":"Tue, 5 Sep 2017 09:02:33 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Hannes Frederic Sowa ","Cc":"Saeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1763593,"web_url":"http://patchwork.ozlabs.org/comment/1763593/","msgid":"<87fuc1q7en.fsf@stressinduktion.org>","list_archive_url":null,"date":"2017-09-05T19:20:32","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":18284,"url":"http://patchwork.ozlabs.org/api/people/18284/","name":"Hannes Frederic Sowa","email":"hannes@stressinduktion.org"},"content":"Hi Tom,\n\nTom Herbert writes:\n\n> On Tue, Sep 5, 2017 at 4:14 AM, Hannes Frederic Sowa\n> wrote:\n>> Tom Herbert writes:\n>>\n>>> There is absolutely no requirement in IP that packets are delivered in\n>>> order-- there never has been and there never will be! If the ULP, like\n>>> Ethernet encapsulation, requires in order deliver then it needs to\n>>> implement that itself like TCP, GRE, and other protocols ensure that\n>>> with sequence numbers and reassembly. All of these hoops we do make\n>>> sure that packets always follow the same path and are always in order\n>>> are done for benefit of middlebox devices like stateful firewalls that\n>>> have force us to adopt their concept of network architecture-- in the\n>>> long run this is self-defeating and kills our ability to innovate.\n>>>\n>>> I'm not saying that we shouldn't consider legacy devices, but we\n>>> should scrutinize new development or solutions that perpetuate\n>>> incorrect design or bad assumptions.\n>>\n>> So configure RSS per port and ensure no fragments are send to those\n>> ports. This is possible and rather easy to do. It solves the problem\n>> with legacy software and it spreads out packets for your applications.\n>>\n>> It is not perfect but it is working and solves both problems.\n>>\n> Hannes,\n>\n> I don't see how that solves anything. The purpose of RSS is to\n> distribute the load of protocol packets across queues. This needs to\n> work for UDP applications. For instance, if I were building a QUIC\n> server I'd want the sort of flow distribution that a TCP server would\n> give. You can't do that by configuring a few ports in the device.\n\nI seriously am with you and I think you are on the right track. I would\nalso much rather like to see fragmentation *not* happening and would\nlike to see hardware logic *not* parsing deep down into packets and\nprotocols. We can agree on that!\n\nAlbeit I don't understand the problem with my approach:\n\nIf you know that you are running a QUIC server and know your environment\n(safe against reordering), run `ethtool -N ethX rx-flow-hash udp4 sdfn'\non the box and you would get spreading of UDP packets based on the UDP\nsrc+dst port numbers to the rx queues. Otherwise only source and\ndestination addresses are being considered, which is the safe default\n(maybe this is sometimes enough).\n\nIntel network cards warn you if you actually switch the mode:\n\n-- >8 --\ndrivers/net/ethernet/intel/fm10k/fm10k_ethtool.c: \"enabling UDP RSS: fragmented packets may arrive out of order to the stack above\\n\");\ndrivers/net/ethernet/intel/igb/igb_ethtool.c: \"enabling UDP RSS: fragmented packets may arrive out of order to the stack above\\n\");\ndrivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c: e_warn(drv, \"enabling UDP RSS: fragmented packets may arrive out of order to the stack above\\n\");\n-- 8< --\n\nI argue that we can't change the default.\n\nFurthermore I tried to argue that in a cloud where you don't know which\napplications you run within the encapsulated networks, it is very hard\nto predict which protocols can be considered safe against those\neffects. I agree, that most of the time, it probably should not\nmatter. But given the rule about kernel backwards compatibility I do\nhave doubts that this change would go unnoticed.\n\nI am the devil's advocate here and argue it is on us to show that is\nsafe before we change the semantics.\n\n> If I were to suggest any HW change it would be to not do DPI on\n> fragments (MF or offset is set). This ensures that all packets of the\n> fragment train get hashed to the same queue and is on fact what RPS\n> has been doing for years without any complaints.\n\nThe same UDP flow could consist out of fragments and non-fragmented\npackets. Even if hardware would act according to what you wrote,\nreordering is very much possible within the same UDP flow.\n\nIn case of QUIC, as far as I know, it uses destination ports 80 and\n443. Like we already notify NICs about vxlan and geneve port numbers, we\ncould selectively enable RSS for UDP ports on those destination port\nnumbers to tell the NIC it won't receive fragments on those ports and\nthus do RSS. The same way the NIC could be notified if reordering on\nthis port is possible. This could be done for example via a lightweight\nsetsockopt.\n\nThe situation with encapsulation is even more complicated:\n\nWe are basically only interested in the UDP/vxlan/Ethernet/IP/UDP\nconstellation. If we do the fragmentation inside the vxlan tunnel and\ncarry over the skb hash to all resulting UDP/vxlan packets source ports,\nwe are fine and reordering on the receiver NIC won't happen in this\ncase. If the fragmentation happens on the outer UDP header, this will\nresult in reordering of the inner L2 flow. Unfortunately this depends on\nhow the vxlan tunnel was set up, how other devices do that and (I\nbelieve so) on the kernel version.\n\nGiven the fact that we tell the NICs the receiving port of vxlan tunnels\nanyway (and I only can assume) they are doing RSS on that, we probably\nhave reordering anyway on those tunnels. So maybe you are right and it\ndoesn't matter in the end for encapsulated packets, by accident.\n\n> But even before I'd make that recommendation, I'd really like\n> understand what the problem actually is. The only thing I can garner\n> from this discussion and the Intel patch is that when fragments are\n> received OOO that is perceived as a problem. But the by the protocol\n> specification clearly says this is not a problem. So the questions\n> are: who is negatively affected by this? Is this a problem because\n> some artificial test that checks for everything to be in order is now\n> failing? Is this affecting real users? Is this an issue in the stack\n> or really with some implementation outside of the stack? If it is an\n> implementation outside of the stack, then are we just bandaid'ing over\n> someone else's incorrect implementation by patching the kernel (like\n> would have be the case if we change the kernel to interoperate with\n> Facebook's switch that couldn't handle OOO in twstate).\n\nI agree with that and I would also love to hear more feedback on\nthis. Unfortunately I don't know.\n\nBye,\nHannes","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=stressinduktion.org\n\theader.i=@stressinduktion.org header.b=\"Wo5mKD8C\"; \n\tdkim=pass (2048-bit key;\n\tunprotected) header.d=messagingengine.com\n\theader.i=@messagingengine.com header.b=\"DVwq4C4E\"; \n\tdkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xmxPz5Lkgz9sNc\n\tfor ;\n\tWed, 6 Sep 2017 05:20:39 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1752643AbdIETUh (ORCPT );\n\tTue, 5 Sep 2017 15:20:37 -0400","from out1-smtp.messagingengine.com ([66.111.4.25]:49925 \"EHLO\n\tout1-smtp.messagingengine.com\" rhost-flags-OK-OK-OK-OK)\n\tby vger.kernel.org with ESMTP id S1751955AbdIETUg (ORCPT\n\t); Tue, 5 Sep 2017 15:20:36 -0400","from compute7.internal (compute7.nyi.internal [10.202.2.47])\n\tby mailout.nyi.internal (Postfix) with ESMTP id 8E199212FC;\n\tTue, 5 Sep 2017 15:20:35 -0400 (EDT)","from frontend1 ([10.202.2.160])\n\tby compute7.internal (MEProxy); Tue, 05 Sep 2017 15:20:35 -0400","from z.localhost.stressinduktion.org (unknown [185.72.66.238])\n\tby mail.messagingengine.com (Postfix) with ESMTPA id 352517FA60;\n\tTue, 5 Sep 2017 15:20:34 -0400 (EDT)"],"DKIM-Signature":["v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tstressinduktion.org; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=vzASGqhXphMVHdEz98\n\tVEXi5+dyurILnCI6R1VV8kzFk=; b=Wo5mKD8Ccmk4061MppX44KNAampCax/WQg\n\tZTVcecSy6ef4pWGjPbMRUGsKLffBZtSanWn3EBcyfbU0DC71DMbQyOVZ0KA0XGxJ\n\tRUpIOLwg0IF+Po3CA+JqAGempd2mCEI522OgsrXLXrucKSmK2HfAGUsGEo444y/O\n\tDg0pohHHk3Ro5R8aJFR6FZ84ug3xLVIF5imziFWbmIqVmsXnGHUp7YRSdkQWPNU1\n\tZsRKRFZ4fUUMMAN1R6vUKUyIyhQ98ORMpJKnlEdJv/PlnnDKVMvfvDg8lKY45/hj\n\t40gAQc3QMxr/d2RRbdtKh3SlxXJ6cuotgeyOHn0baQHXjUwqEscw==","v=1; a=rsa-sha256; c=relaxed/relaxed; d=\n\tmessagingengine.com; h=cc:content-type:date:from:in-reply-to\n\t:message-id:mime-version:references:subject:to:x-me-sender\n\t:x-me-sender:x-sasl-enc:x-sasl-enc; s=fm1; bh=vzASGqhXphMVHdEz98\n\tVEXi5+dyurILnCI6R1VV8kzFk=; b=DVwq4C4EAihgvTuJm09jejmlbgDotmIUh7\n\t/CTvWPotiU3h+HtnuOtX/aeLmMq1ovc2hhI3p7bcNnvgubKFQTbv67H0nJMdHjG4\n\tsnPmXb4cNv2k8tSXcpcqURvRSZjB5vdI5H0ryO/U/gzoYf3qcxSGenYdB9CWP2R8\n\tZaqVJG8A+05Op1C8ASD6Q1mtVZfWv+JE3C9yCV0yCSy6Zcp9T+hRqE2EF/zmZOr+\n\txYDarrpKx+y4omsf+4XXtL+WFcYl5KJRQCS2kMh7imsW+gKF0XzmSJBZHDjlTwAD\n\tJME8ptISKpBQTrnD9KHBeuypaaWEMtc/6m9RtpYUWFdHQV6Qjf/A=="],"X-ME-Sender":"","X-Sasl-enc":"nFKP5dgXm9i4DR2+159DSsO4YsRZL1Xw4EZ9I9CYqk3W 1504639235","From":"Hannes Frederic Sowa ","To":"Tom Herbert ","Cc":"Saeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>\n\t\n\t<8760cytnif.fsf@stressinduktion.org>\n\t\n\t<87k21ez6r9.fsf@stressinduktion.org>\n\t\n\t<87r2vls8hp.fsf@stressinduktion.org>\n\t","Date":"Tue, 05 Sep 2017 21:20:32 +0200","In-Reply-To":"\n\t(Tom Herbert's message of \"Tue, 5 Sep 2017 09:02:33 -0700\")","Message-ID":"<87fuc1q7en.fsf@stressinduktion.org>","User-Agent":"Gnus/5.13 (Gnus v5.13) Emacs/25.2 (gnu/linux)","MIME-Version":"1.0","Content-Type":"text/plain","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1763642,"web_url":"http://patchwork.ozlabs.org/comment/1763642/","msgid":"","list_archive_url":null,"date":"2017-09-05T21:13:05","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65986,"url":"http://patchwork.ozlabs.org/api/people/65986/","name":"Tom Herbert","email":"tom@herbertland.com"},"content":"> The situation with encapsulation is even more complicated:\n>\n> We are basically only interested in the UDP/vxlan/Ethernet/IP/UDP\n> constellation. If we do the fragmentation inside the vxlan tunnel and\n> carry over the skb hash to all resulting UDP/vxlan packets source ports,\n> we are fine and reordering on the receiver NIC won't happen in this\n> case. If the fragmentation happens on the outer UDP header, this will\n> result in reordering of the inner L2 flow. Unfortunately this depends on\n> how the vxlan tunnel was set up, how other devices do that and (I\n> believe so) on the kernel version.\n>\nThis really isn't that complicated. The assumption that an IP network\nalways delivers packets in order is simply wrong. The inventors of\nVXLAN must have know full well that when you use IP, packets can and\neventually will be delivered out of order. This isn't just because of\nfragmentation, there are many other reasons that packets can be\ndelivered OOO. This also must have been known when IP/GRE and any\nother protocol that carries L2 over IP was invented. If OOO is an\nissue for these protocols then they need to be fixed-- this is not a\nconcern with IP protocol nor the stack.\n\nTom","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=herbertland-com.20150623.gappssmtp.com\n\theader.i=@herbertland-com.20150623.gappssmtp.com\n\theader.b=\"NPnWWFK+\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xmzvt6SsWz9sNV\n\tfor ;\n\tWed, 6 Sep 2017 07:13:14 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1752742AbdIEVNK (ORCPT );\n\tTue, 5 Sep 2017 17:13:10 -0400","from mail-qt0-f169.google.com ([209.85.216.169]:35067 \"EHLO\n\tmail-qt0-f169.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1752312AbdIEVNH (ORCPT\n\t); Tue, 5 Sep 2017 17:13:07 -0400","by mail-qt0-f169.google.com with SMTP id k2so15408627qte.2\n\tfor ; Tue, 05 Sep 2017 14:13:07 -0700 (PDT)","by 10.200.53.168 with HTTP; Tue, 5 Sep 2017 14:13:05 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=herbertland-com.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=SeTVKk+Kg5tN10+RV2ByH3a1bdDpYjrO81pTNnWVdss=;\n\tb=NPnWWFK+0InpkGQOzvTmYEMpqTJBwvPwp7/J7+kTE0MfOv584beAENmIMoGXrtkIdl\n\tuZObAggQGjvbPCr+GUzOLSeel7iL6Pi19w74hxbFWjYVt/TqYdlZXA2Ar2FJcKlj100H\n\tNSptPCfX9PowEUk8i0SAstZWPl/DJFs6qQs6y6Zm09hiQMXRc1xzdgStgEekVuf+3YyJ\n\tl+JKTscBUidixpe/MjwVev1CX1w7G5bUkK/0aKexRKCtfq8+W5B1Q7UJc5lYA30rmyr7\n\tWHFB19dVwLv1e4je4I9s7RIZYChMx2+WgsuvyrmwZoyeI3xwdiUrv/xu8umWVLLj4U05\n\tWlSw==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=SeTVKk+Kg5tN10+RV2ByH3a1bdDpYjrO81pTNnWVdss=;\n\tb=l4VNg4l4KDRXpWn6cBEi/Xh7Rk0J/m8YVw1Pezuhsw2ObmB2kR8j9PrSpFY4vmGrO4\n\tOS4gqdhQcUIgqV+7ySJYufGAwvwBwOfRfhONv+/XMzhwlwrr0FaR3mNZ4JgiYlry5NQ+\n\tqJr8aiObdvI7/JWa/UGz4HPmOBDiLYI5GW4j/J/RWwi9KEaExQZMTtJeIaKNw6Nviqcq\n\tb+DNLvNlGtpLUtIDT0goO2ye0oTBZb1VZEQ7yrBnqJJd5STiE5uBpmaBrgzTAKjdXPRJ\n\t371h04g67/SfDe5BsmHShtU0dyWLrNEJREDNPKZSISxuqWtG0Wws7CKD91HZ8/GprLsI\n\tVYyw==","X-Gm-Message-State":"AHPjjUh4VxdUw4Z8b72fyGJ2AAQXPqr9WPS31xNuNiGdU+htXTnoQp0b\n\tc0RsSUuQUm9bnEo+MGDuOTWVtojmQ+49","X-Google-Smtp-Source":"ADKCNb648yao4Qq5/f7U8PKJseXlJdO4nUnKm+uzABhMRZ4Zxq6ld1G6SU8zDWjlcKpdQUijIfIflLqkc6dNFHthlUo=","X-Received":"by 10.200.42.243 with SMTP id c48mr586847qta.205.1504645987187; \n\tTue, 05 Sep 2017 14:13:07 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"<87fuc1q7en.fsf@stressinduktion.org>","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>\n\t\n\t<8760cytnif.fsf@stressinduktion.org>\n\t\n\t<87k21ez6r9.fsf@stressinduktion.org>\n\t\n\t<87r2vls8hp.fsf@stressinduktion.org>\n\t\n\t<87fuc1q7en.fsf@stressinduktion.org>","From":"Tom Herbert ","Date":"Tue, 5 Sep 2017 14:13:05 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Hannes Frederic Sowa ","Cc":"Saeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1763793,"web_url":"http://patchwork.ozlabs.org/comment/1763793/","msgid":"","list_archive_url":null,"date":"2017-09-06T03:06:24","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":252,"url":"http://patchwork.ozlabs.org/api/people/252/","name":"Alexander Duyck","email":"alexander.duyck@gmail.com"},"content":"On Tue, Sep 5, 2017 at 2:13 PM, Tom Herbert wrote:\n>> The situation with encapsulation is even more complicated:\n>>\n>> We are basically only interested in the UDP/vxlan/Ethernet/IP/UDP\n>> constellation. If we do the fragmentation inside the vxlan tunnel and\n>> carry over the skb hash to all resulting UDP/vxlan packets source ports,\n>> we are fine and reordering on the receiver NIC won't happen in this\n>> case. If the fragmentation happens on the outer UDP header, this will\n>> result in reordering of the inner L2 flow. Unfortunately this depends on\n>> how the vxlan tunnel was set up, how other devices do that and (I\n>> believe so) on the kernel version.\n>>\n> This really isn't that complicated. The assumption that an IP network\n> always delivers packets in order is simply wrong. The inventors of\n> VXLAN must have know full well that when you use IP, packets can and\n> eventually will be delivered out of order. This isn't just because of\n> fragmentation, there are many other reasons that packets can be\n> delivered OOO. This also must have been known when IP/GRE and any\n> other protocol that carries L2 over IP was invented. If OOO is an\n> issue for these protocols then they need to be fixed-- this is not a\n> concern with IP protocol nor the stack.\n>\n> Tom\n\nAs far as a little background on the original patch I believe the\nissue that was fixed by the patch was a video streaming application\nthat was sending/receiving a mix of fragmented and non-fragmented\npackets. Receiving them out of order due to the fragmentation was\ncausing issues with stutters in the video and so we ended up disabling\nUDP by default in the NICs listed. We decided to go that way as UDP\nRSS was viewed as a performance optimization, while the out-of-order\nproblems were viewed as a functionality issue.\n\nThe default for i40e is to have UDP RSS hashing enabled if I recall\ncorrectly. Basically as we move away from enterprise to cloud I\nsuspect that is going to be the case more and more since all the UDP\ntunnels require either port recognition or UDP RSS to be enabled. For\nnow we carry the ability to enable UDP RSS if desired in the legacy\ndrivers, and I believe we have some white papers somewhere that\nsuggest enabling it if you are planning to use UDP based tunnels.\n\n- Alex","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=gmail.com header.i=@gmail.com\n\theader.b=\"XHPpt2fp\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xn7lT38g4z9t2R\n\tfor ;\n\tWed, 6 Sep 2017 13:06:29 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1753113AbdIFDG1 (ORCPT );\n\tTue, 5 Sep 2017 23:06:27 -0400","from mail-qk0-f194.google.com ([209.85.220.194]:34461 \"EHLO\n\tmail-qk0-f194.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1752226AbdIFDG0 (ORCPT\n\t); Tue, 5 Sep 2017 23:06:26 -0400","by mail-qk0-f194.google.com with SMTP id d70so3206204qkc.1\n\tfor ; Tue, 05 Sep 2017 20:06:25 -0700 (PDT)","by 10.140.85.211 with HTTP; Tue, 5 Sep 2017 20:06:24 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=gmail.com; s=20161025;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=No650/hOSF8jsQ6fVcprDw9rfPh/2aOmqvNDtIbGijo=;\n\tb=XHPpt2fpeFEIKyLorxg03jGCd0CvpvZrGomurVwt0TfZOy6T82AaPuSQUCqHZlmj2P\n\t+ulfUZCKjD9RNWDjWngPLriR5S1H0lThC2j+D9gYxDC07rEhO9eXoJUusc3KiTemtgJG\n\tJDnosBi0PwekHV5wDO5iXej+gDnqBGjlQoDCMFpnVatQsPRDW/OCyv21E7UW2Bz8vvgB\n\te/eK/4M2iXElnUaC4bwiTOjVPYiySAtYZoUulKrkw4puYO1CRLgaIxVzbhnWo3hJoeJ2\n\tm3Y6XqGSvsZ/nhNMffX+LfRccjduCPxq0Uoho+mdWJcXqYlHl4k7+QIslbSFJjoRDNgF\n\t98/w==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=No650/hOSF8jsQ6fVcprDw9rfPh/2aOmqvNDtIbGijo=;\n\tb=j/pbGNFn1qjFlKJsnkM27uTvev5V8C2LdkOFTWZn28LanTzwZ4UpZMcLCKEggbpubY\n\tiIVzkRjLZBCM7b5gq9JXr3XacJmn+fy2PtgvrSVKOhje4/+lJTK5TGMxdmddQek2vZgb\n\t0EtFlc437D75KvqkCBvqzv8TxDPWLy8N2MR8+uVRufHuCbjNOtUkKSkWdaR3DeomJ9Ex\n\tUShvs7PmvwNJZItYhOvUr4VUHme+sE2pIck/9DIg5MHMtXvplx/6Uj9kwt8DHK/7Kd7w\n\tG7/sDOtDuBQCAO7HBcIMk8oOo6ntnj1gvqyHLqKykOJtvWGZGuT6WgvLuNmisVtoKdE0\n\tlyJA==","X-Gm-Message-State":"AHPjjUjYdDfnN8OrYnuYlt4fxQjzQjwVjAnLhUP0h7Emjew4vEP9xdkG\n\tAw2PSJjZhvSj7V4CMM/YfgXfoKCuyg==","X-Google-Smtp-Source":"ADKCNb6+h45X/EVMR2OQWUP6q4dD2mLz3pDib3oiEcxTSQwC8iXrsoaWd/tv03d19HCHWkLZ3C3Eh/eTIQkqLKRJN2s=","X-Received":"by 10.55.190.198 with SMTP id o189mr1468691qkf.103.1504667185286;\n\tTue, 05 Sep 2017 20:06:25 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>\n\t\n\t<8760cytnif.fsf@stressinduktion.org>\n\t\n\t<87k21ez6r9.fsf@stressinduktion.org>\n\t\n\t<87r2vls8hp.fsf@stressinduktion.org>\n\t\n\t<87fuc1q7en.fsf@stressinduktion.org>\n\t","From":"Alexander Duyck ","Date":"Tue, 5 Sep 2017 20:06:24 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Tom Herbert ","Cc":"Hannes Frederic Sowa ,\n\tSaeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1764231,"web_url":"http://patchwork.ozlabs.org/comment/1764231/","msgid":"","list_archive_url":null,"date":"2017-09-06T16:17:50","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65986,"url":"http://patchwork.ozlabs.org/api/people/65986/","name":"Tom Herbert","email":"tom@herbertland.com"},"content":"On Tue, Sep 5, 2017 at 8:06 PM, Alexander Duyck\n wrote:\n> On Tue, Sep 5, 2017 at 2:13 PM, Tom Herbert wrote:\n>>> The situation with encapsulation is even more complicated:\n>>>\n>>> We are basically only interested in the UDP/vxlan/Ethernet/IP/UDP\n>>> constellation. If we do the fragmentation inside the vxlan tunnel and\n>>> carry over the skb hash to all resulting UDP/vxlan packets source ports,\n>>> we are fine and reordering on the receiver NIC won't happen in this\n>>> case. If the fragmentation happens on the outer UDP header, this will\n>>> result in reordering of the inner L2 flow. Unfortunately this depends on\n>>> how the vxlan tunnel was set up, how other devices do that and (I\n>>> believe so) on the kernel version.\n>>>\n>> This really isn't that complicated. The assumption that an IP network\n>> always delivers packets in order is simply wrong. The inventors of\n>> VXLAN must have know full well that when you use IP, packets can and\n>> eventually will be delivered out of order. This isn't just because of\n>> fragmentation, there are many other reasons that packets can be\n>> delivered OOO. This also must have been known when IP/GRE and any\n>> other protocol that carries L2 over IP was invented. If OOO is an\n>> issue for these protocols then they need to be fixed-- this is not a\n>> concern with IP protocol nor the stack.\n>>\n>> Tom\n>\n> As far as a little background on the original patch I believe the\n> issue that was fixed by the patch was a video streaming application\n> that was sending/receiving a mix of fragmented and non-fragmented\n> packets. Receiving them out of order due to the fragmentation was\n> causing issues with stutters in the video and so we ended up disabling\n> UDP by default in the NICs listed. We decided to go that way as UDP\n> RSS was viewed as a performance optimization, while the out-of-order\n> problems were viewed as a functionality issue.\n>\nHi Alex,\n\nThanks for the details! Were you able to find the root cause for this?\nIn particular, it would be interesting to know if it is the kernel or\ndevice that introduced the jitter, or if it's the application that\ndoesn't handle OOO well...\n\nTom","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=herbertland-com.20150623.gappssmtp.com\n\theader.i=@herbertland-com.20150623.gappssmtp.com\n\theader.b=\"EYhk0php\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xnTJj1Xv3z9s7F\n\tfor ;\n\tThu, 7 Sep 2017 02:17:57 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1755352AbdIFQRz (ORCPT );\n\tWed, 6 Sep 2017 12:17:55 -0400","from mail-qk0-f175.google.com ([209.85.220.175]:35224 \"EHLO\n\tmail-qk0-f175.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1752462AbdIFQRw (ORCPT\n\t); Wed, 6 Sep 2017 12:17:52 -0400","by mail-qk0-f175.google.com with SMTP id r141so20663402qke.2\n\tfor ; Wed, 06 Sep 2017 09:17:52 -0700 (PDT)","by 10.237.61.196 with HTTP; Wed, 6 Sep 2017 09:17:50 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=herbertland-com.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=miAXJFFdeOgyETgyiFhEA16aQ7sdEEAMk6dQ/9B+f0U=;\n\tb=EYhk0phppos1clLN/OYfz4qqHH6B3MBawK6aAzaUnGjkPyNBXQogj2T1AoayIwv3Yy\n\tYduBMoC720q6NKrdu6D4lGCp+CPD+Dmz/bRPswUYvYHYvy32gbkgEq2KutxJOxOYE6uF\n\tj1wiP1g1RwmkXPX07qmohCgQFPYmdccL1EbphnuwTMEw+GPI9QMEW99h+M1VONhLeFB8\n\tOjhetkzBPHdkZAJ2Un9Uhc5J0tNZ1Wk576zfGJjrgiLEua3rxngLfEG+0XR2LqUMbcGX\n\tGxzKVaUBQhAWmXc4iGvgbEl/F11vhQir2FnsHG2dJWJdhJQItI2r/kEGrbcNlpbqW1jA\n\t9x4Q==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=miAXJFFdeOgyETgyiFhEA16aQ7sdEEAMk6dQ/9B+f0U=;\n\tb=kPjBW0DxE5bapyC64cYmSCK2ZdAAs0t/Swnp1LHApswu1S9b5fOH14yYppLQr/7wHb\n\twBptv6TGLY9DoenuubRFe33VeeAbvBXI/iSdsV3Gv8jiYOe3v38ruEPYzu7mOP0b+0b0\n\tsbFXWWrSeAaMo5EMhAyywwXPybHu2j9m1UZneMmmVR6FF7wUxItmxVnlS6dSDDL+m31k\n\tfjPfUAUkgKYAzzuWkd2JXnWfWna2IVBjhD1CK7KrNK82GrmwKWGNdU8WhPUO/2KVPAsp\n\tgFjUDgjTw3A2DpK2XUij06xVJZY5rJN/p59NWGiYvxGOH+ogtnmObRUMuX708tUozkoM\n\tTa8g==","X-Gm-Message-State":"AHPjjUgDDm7ZAIsk6N7AISryMmmEmPuwnio8A8pbk3HYRPcPVddEWRZ3\n\tB3Khc2I1v8QWkaTHboju3oOSeACmb0r0","X-Google-Smtp-Source":"ADKCNb7DcX1jmoHgCMmniYhM+rjYa+QwjIhIYSNNLnPHJOHhkEswT6VPQ0ONqtWpcFfRCRIJswjx/7+q5UWgK3CKvDQ=","X-Received":"by 10.55.133.6 with SMTP id h6mr4056445qkd.17.1504714671875; Wed,\n\t06 Sep 2017 09:17:51 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>\n\t\n\t<8760cytnif.fsf@stressinduktion.org>\n\t\n\t<87k21ez6r9.fsf@stressinduktion.org>\n\t\n\t<87r2vls8hp.fsf@stressinduktion.org>\n\t\n\t<87fuc1q7en.fsf@stressinduktion.org>\n\t\n\t","From":"Tom Herbert ","Date":"Wed, 6 Sep 2017 09:17:50 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Alexander Duyck ","Cc":"Hannes Frederic Sowa ,\n\tSaeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1764285,"web_url":"http://patchwork.ozlabs.org/comment/1764285/","msgid":"","list_archive_url":null,"date":"2017-09-06T17:43:13","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":252,"url":"http://patchwork.ozlabs.org/api/people/252/","name":"Alexander Duyck","email":"alexander.duyck@gmail.com"},"content":"On Wed, Sep 6, 2017 at 9:17 AM, Tom Herbert wrote:\n> On Tue, Sep 5, 2017 at 8:06 PM, Alexander Duyck\n> wrote:\n>> On Tue, Sep 5, 2017 at 2:13 PM, Tom Herbert wrote:\n>>>> The situation with encapsulation is even more complicated:\n>>>>\n>>>> We are basically only interested in the UDP/vxlan/Ethernet/IP/UDP\n>>>> constellation. If we do the fragmentation inside the vxlan tunnel and\n>>>> carry over the skb hash to all resulting UDP/vxlan packets source ports,\n>>>> we are fine and reordering on the receiver NIC won't happen in this\n>>>> case. If the fragmentation happens on the outer UDP header, this will\n>>>> result in reordering of the inner L2 flow. Unfortunately this depends on\n>>>> how the vxlan tunnel was set up, how other devices do that and (I\n>>>> believe so) on the kernel version.\n>>>>\n>>> This really isn't that complicated. The assumption that an IP network\n>>> always delivers packets in order is simply wrong. The inventors of\n>>> VXLAN must have know full well that when you use IP, packets can and\n>>> eventually will be delivered out of order. This isn't just because of\n>>> fragmentation, there are many other reasons that packets can be\n>>> delivered OOO. This also must have been known when IP/GRE and any\n>>> other protocol that carries L2 over IP was invented. If OOO is an\n>>> issue for these protocols then they need to be fixed-- this is not a\n>>> concern with IP protocol nor the stack.\n>>>\n>>> Tom\n>>\n>> As far as a little background on the original patch I believe the\n>> issue that was fixed by the patch was a video streaming application\n>> that was sending/receiving a mix of fragmented and non-fragmented\n>> packets. Receiving them out of order due to the fragmentation was\n>> causing issues with stutters in the video and so we ended up disabling\n>> UDP by default in the NICs listed. We decided to go that way as UDP\n>> RSS was viewed as a performance optimization, while the out-of-order\n>> problems were viewed as a functionality issue.\n>>\n> Hi Alex,\n>\n> Thanks for the details! Were you able to find the root cause for this?\n> In particular, it would be interesting to know if it is the kernel or\n> device that introduced the jitter, or if it's the application that\n> doesn't handle OOO well...\n>\n> Tom\n\nIt is hard to say since my memory of the events from 7 years ago is\npretty vague at this point, but I'm pretty sure it was the\napplication. Basically getting the frames out of order was causing\nthem to have to drop video data if I recall correctly.\n\n- Alex","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=gmail.com header.i=@gmail.com\n\theader.b=\"U0D80uzy\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xnWCC09nfz9t2d\n\tfor ;\n\tThu, 7 Sep 2017 03:43:19 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1751875AbdIFRnQ (ORCPT );\n\tWed, 6 Sep 2017 13:43:16 -0400","from mail-qt0-f196.google.com ([209.85.216.196]:35270 \"EHLO\n\tmail-qt0-f196.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1750739AbdIFRnO (ORCPT\n\t); Wed, 6 Sep 2017 13:43:14 -0400","by mail-qt0-f196.google.com with SMTP id p55so4308861qtc.2\n\tfor ; Wed, 06 Sep 2017 10:43:14 -0700 (PDT)","by 10.140.85.211 with HTTP; Wed, 6 Sep 2017 10:43:13 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=gmail.com; s=20161025;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=OoJXfLZeKQFJMRGv2+kPKShcHWxHwCEZ/WgmnisDtsI=;\n\tb=U0D80uzy3NkPxZFZGtb3QuhJtqX2jWecIqpmLdiZfjN6LPeEcc3EAEARCYeB0u4/+n\n\tGUOe5ZaeGNwhSP97HWF8CH0mS3kNOHEUl6uf3p53txnpDmCYpVslBh0LJqLv1BtBjEv9\n\t2V6dyk3ZML/AG7siPTVfCVeNLnpDBTk4X2ppiA6NlWvQcGaOgQ/Er76Z6+dEMGM+pb/R\n\tl6wPFgjnaACSYnigM4lsfmoyU+Bm9zAmMg0MbV0ENzaZ3Q+7lV/rUuOSfnUTYGemUE21\n\tBeaf8gEebaxNGKVWegk6GSst5Cl/78UzzF5baQ3V6TRGoYWT0WBI7ZML1LkQZ9+ckHn7\n\tcbwA==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=OoJXfLZeKQFJMRGv2+kPKShcHWxHwCEZ/WgmnisDtsI=;\n\tb=ZU0G+kBTInpj/j6/YDxevQ7wT//QpQWEaYbEXX8V9va8xH0Got/hWpcaaIXozTvJoS\n\ty47tSkcBnPTHRmKOMm0GYFoiXhuRzQs0p6ViGxFY8Mh3bPcEJiDvhrgRtXikW/BqQbi0\n\tn1d3Tu132BWuPG5ML0q3AM5l0+PWjQaa3YtBShGrdweHuSh0K2wUaes1yUy/D3ePEBpN\n\tvWZMhWdxn3qpCFnjKU8FSu3zxoffqmDLnMb6c4eEarbwiKlWTWeVjwmeukSsBrUNQI/+\n\tf1wChfOXgaLFfISE8cGc8AVrdozXi8BYaWxqaqPcDjsdCjuzW2X2pkKUIu9hMaj9BS0f\n\tyaag==","X-Gm-Message-State":"AHPjjUhzOuMDOYvRzyt2186jNpmD/97jWTbIBKFgC7QXU0qfepWxBncd\n\tH8lsdj7ux2im8sxf9A9UP9ZAJeugbA==","X-Google-Smtp-Source":"AOwi7QAM5xAb5JISEp9XxlKZlWSltiCzJOEs+HBNj5qSK1VFGVHm09KlAetFL1XAYBV5aBgu323wRoIZUWNdf7rLAb4=","X-Received":"by 10.237.46.71 with SMTP id j65mr4636570qtd.163.1504719794024; \n\tWed, 06 Sep 2017 10:43:14 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>\n\t\n\t<8760cytnif.fsf@stressinduktion.org>\n\t\n\t<87k21ez6r9.fsf@stressinduktion.org>\n\t\n\t<87r2vls8hp.fsf@stressinduktion.org>\n\t\n\t<87fuc1q7en.fsf@stressinduktion.org>\n\t\n\t\n\t","From":"Alexander Duyck ","Date":"Wed, 6 Sep 2017 10:43:13 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Tom Herbert ","Cc":"Hannes Frederic Sowa ,\n\tSaeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}},{"id":1764312,"web_url":"http://patchwork.ozlabs.org/comment/1764312/","msgid":"","list_archive_url":null,"date":"2017-09-06T19:01:16","subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","submitter":{"id":65986,"url":"http://patchwork.ozlabs.org/api/people/65986/","name":"Tom Herbert","email":"tom@herbertland.com"},"content":"On Wed, Sep 6, 2017 at 10:43 AM, Alexander Duyck\n wrote:\n> On Wed, Sep 6, 2017 at 9:17 AM, Tom Herbert wrote:\n>> On Tue, Sep 5, 2017 at 8:06 PM, Alexander Duyck\n>> wrote:\n>>> On Tue, Sep 5, 2017 at 2:13 PM, Tom Herbert wrote:\n>>>>> The situation with encapsulation is even more complicated:\n>>>>>\n>>>>> We are basically only interested in the UDP/vxlan/Ethernet/IP/UDP\n>>>>> constellation. If we do the fragmentation inside the vxlan tunnel and\n>>>>> carry over the skb hash to all resulting UDP/vxlan packets source ports,\n>>>>> we are fine and reordering on the receiver NIC won't happen in this\n>>>>> case. If the fragmentation happens on the outer UDP header, this will\n>>>>> result in reordering of the inner L2 flow. Unfortunately this depends on\n>>>>> how the vxlan tunnel was set up, how other devices do that and (I\n>>>>> believe so) on the kernel version.\n>>>>>\n>>>> This really isn't that complicated. The assumption that an IP network\n>>>> always delivers packets in order is simply wrong. The inventors of\n>>>> VXLAN must have know full well that when you use IP, packets can and\n>>>> eventually will be delivered out of order. This isn't just because of\n>>>> fragmentation, there are many other reasons that packets can be\n>>>> delivered OOO. This also must have been known when IP/GRE and any\n>>>> other protocol that carries L2 over IP was invented. If OOO is an\n>>>> issue for these protocols then they need to be fixed-- this is not a\n>>>> concern with IP protocol nor the stack.\n>>>>\n>>>> Tom\n>>>\n>>> As far as a little background on the original patch I believe the\n>>> issue that was fixed by the patch was a video streaming application\n>>> that was sending/receiving a mix of fragmented and non-fragmented\n>>> packets. Receiving them out of order due to the fragmentation was\n>>> causing issues with stutters in the video and so we ended up disabling\n>>> UDP by default in the NICs listed. We decided to go that way as UDP\n>>> RSS was viewed as a performance optimization, while the out-of-order\n>>> problems were viewed as a functionality issue.\n>>>\n>> Hi Alex,\n>>\n>> Thanks for the details! Were you able to find the root cause for this?\n>> In particular, it would be interesting to know if it is the kernel or\n>> device that introduced the jitter, or if it's the application that\n>> doesn't handle OOO well...\n>>\n>> Tom\n>\n> It is hard to say since my memory of the events from 7 years ago is\n> pretty vague at this point, but I'm pretty sure it was the\n> application. Basically getting the frames out of order was causing\n> them to have to drop video data if I recall correctly.\n>\nOh, I didn't notice that patch was from 2010. Maybe the application\nhas been fixed by now! :-)\n\nPerhaps, it's time to try to turn UDP hashing on again by default?\nEven if NICs aren't doing this, there are network devices that are\nlooking at UDP for ECMP and that doesn't seem to be causing widespread\nproblems currently...\n\nTom","headers":{"Return-Path":"","X-Original-To":"patchwork-incoming@ozlabs.org","Delivered-To":"patchwork-incoming@ozlabs.org","Authentication-Results":["ozlabs.org;\n\tspf=none (mailfrom) smtp.mailfrom=vger.kernel.org\n\t(client-ip=209.132.180.67; helo=vger.kernel.org;\n\tenvelope-from=netdev-owner@vger.kernel.org;\n\treceiver=)","ozlabs.org; dkim=pass (2048-bit key;\n\tunprotected) header.d=herbertland-com.20150623.gappssmtp.com\n\theader.i=@herbertland-com.20150623.gappssmtp.com\n\theader.b=\"KddqmKUE\"; dkim-atps=neutral"],"Received":["from vger.kernel.org (vger.kernel.org [209.132.180.67])\n\tby ozlabs.org (Postfix) with ESMTP id 3xnXxH6xxkz9s81\n\tfor ;\n\tThu, 7 Sep 2017 05:01:23 +1000 (AEST)","(majordomo@vger.kernel.org) by vger.kernel.org via listexpand\n\tid S1752176AbdIFTBT (ORCPT );\n\tWed, 6 Sep 2017 15:01:19 -0400","from mail-qt0-f194.google.com ([209.85.216.194]:34251 \"EHLO\n\tmail-qt0-f194.google.com\" rhost-flags-OK-OK-OK-OK) by vger.kernel.org\n\twith ESMTP id S1751969AbdIFTBS (ORCPT\n\t); Wed, 6 Sep 2017 15:01:18 -0400","by mail-qt0-f194.google.com with SMTP id q8so3678173qtb.1\n\tfor ; Wed, 06 Sep 2017 12:01:17 -0700 (PDT)","by 10.237.61.196 with HTTP; Wed, 6 Sep 2017 12:01:16 -0700 (PDT)"],"DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=herbertland-com.20150623.gappssmtp.com; s=20150623;\n\th=mime-version:in-reply-to:references:from:date:message-id:subject:to\n\t:cc; bh=rtcu5/jlYe5mGLZgdCrTbDGziSqWgJ76LABOFkRx1ok=;\n\tb=KddqmKUEOiPGqsBklCm0vGRZ9QEQRapMKk3IFDt2xZaalc1Jvxud9pW0QK5wLioGwe\n\tC+Lswczam1ovEoygVWjm2HKDtbhOyY5W8AIWG5RO98EPxIBKMWB3hO89Osjhg3RZ3Lho\n\tYemPVhM6fQDEDUHD8zKGsJl6e83KEMUynVWe/DiVcxM+OHo7VPJ9nIxF+k+LnebUi56P\n\tqqUmTmZ8CT9PrVp8+IC+5NtGqpXhiB53pCVPDhfGgbX8jVOqxQSsIQcfP2bSM0LSEwGP\n\tNy17VLQ5QpfzU8ZRtdKkG9wVcd4swlqANT4Xh0IDWnMsdQ+AVr82r2oSQT9kojNGldby\n\tLUbw==","X-Google-DKIM-Signature":"v=1; a=rsa-sha256; c=relaxed/relaxed;\n\td=1e100.net; s=20161025;\n\th=x-gm-message-state:mime-version:in-reply-to:references:from:date\n\t:message-id:subject:to:cc;\n\tbh=rtcu5/jlYe5mGLZgdCrTbDGziSqWgJ76LABOFkRx1ok=;\n\tb=tw/Un/D2D+kEFU46M8o1vuM9MmHtbZ1TkzqmHyRM+3Uqm2rMPMYIRsxNboENWzjYXK\n\tOWB/2J4yNNqFTtXo55cDPSNj0hWSiwJUvoLn1DvU2mvdSS6pNvDV5/cVyaiy1bM3PonN\n\tRYcV9RCJY2h71RUhN9muxeGyGmV6Mhl6fx5J4Jkk5co2NLlTUmaR5M1bWA8bB6LujUI+\n\t3L9RgWoypPk0Dg1pFLEFh8TxNkN/08u4QTTvtrdZFYd1Zg534xBcua8kn8A1xthpefi5\n\tVVUSQwKJUYsHYkG+3ty1boMiNE9PVpn9LIynzf0uBCXdtOk+G4MreobgrURwSjtdY1Fq\n\tiKKw==","X-Gm-Message-State":"AHPjjUjWpmLMsK6oi0I2K8lKN7kwkZvMLhHYJhHREg0uCDbFm4BaV1KB\n\tIGf9/EIaP/g8ZSd1F2XVqkkkmx0eBbyK","X-Google-Smtp-Source":"AOwi7QDYJ83IYsA+3jRlkwurkNyXyZhqmAzAjz2xKDsH8W0nxCeK8v8O7YxRk7W4kS8bITtsiof+oFU3W3r42VA/N/8=","X-Received":"by 10.237.56.167 with SMTP id k36mr244378qte.286.1504724477295; \n\tWed, 06 Sep 2017 12:01:17 -0700 (PDT)","MIME-Version":"1.0","In-Reply-To":"","References":"<20170830230409.15176-1-saeedm@mellanox.com>\n\t<87fuc7ong0.fsf@stressinduktion.org>\n\t\n\t<1504402349.3299394.1093467800.72B91372@webmail.messagingengine.com>\n\t\n\t\n\t\n\t<87pob6bmjv.fsf@stressinduktion.org>\n\t\n\t<8760cytnif.fsf@stressinduktion.org>\n\t\n\t<87k21ez6r9.fsf@stressinduktion.org>\n\t\n\t<87r2vls8hp.fsf@stressinduktion.org>\n\t\n\t<87fuc1q7en.fsf@stressinduktion.org>\n\t\n\t\n\t\n\t","From":"Tom Herbert ","Date":"Wed, 6 Sep 2017 12:01:16 -0700","Message-ID":"","Subject":"Re: [pull request][net-next 0/3] Mellanox, mlx5 GRE tunnel offloads","To":"Alexander Duyck ","Cc":"Hannes Frederic Sowa ,\n\tSaeed Mahameed ,\n\tSaeed Mahameed ,\n\t\"David S. Miller\" ,\n\tLinux Netdev List ","Content-Type":"text/plain; charset=\"UTF-8\"","Sender":"netdev-owner@vger.kernel.org","Precedence":"bulk","List-ID":"","X-Mailing-List":"netdev@vger.kernel.org"}}]