| Message ID | 20170919062340.13212-1-ligs@dtdream.com |
|---|---|
| State | Superseded |
| Headers | show |
| Series | [ovs-dev] dpif-netdev: Set MAX_RECIRC_DEPTH to 10. | expand |
Distributed gateway ?
What do the kernel datapath flows look like ?
On 9/18/17, 11:29 PM, "ovs-dev-bounces@openvswitch.org on behalf of Guoshuai Li" <ovs-dev-bounces@openvswitch.org on behalf of ligs@dtdream.com> wrote:
In an ovn gateway node with DPDK, the RECIRC_DEPTH may be greater than 5.
Scenes:
VM ping self floating IP, or
VM ping Floating IP of VMs with the same network.
It need process UNDNAT SNAT in LRouter egress and
UNSNAT DNAT in LRouter ingress, and
output to geneve tunnel also need recirc.
This has an WARN:
dpif_netdev(pmd36)|WARN|Packet dropped. Max recirculation depth exceeded.
---
lib/dpif-netdev.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
index ca74df8c7..2ad8c3143 100644
--- a/lib/dpif-netdev.c
+++ b/lib/dpif-netdev.c
@@ -82,7 +82,7 @@ VLOG_DEFINE_THIS_MODULE(dpif_netdev);
#define FLOW_DUMP_MAX_BATCH 50
/* Use per thread recirc_depth to prevent recirculation loop. */
-#define MAX_RECIRC_DEPTH 5
+#define MAX_RECIRC_DEPTH 10
I guess the MAX_RECIRC_DEPTH should be as small as possible for valid use cases – 6 ?
The check is ‘less than’.
DEFINE_STATIC_PER_THREAD_DATA(uint32_t, recirc_depth, 0)
/* Configuration parameters. */
--
This is my flows by ovs-appctl dpctl/dump-flows:
tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({class=0x102,type=0x80,len=4,0x30005/0x7fffffff}),flags(-df+csum+key)),recirc_id(0),in_port(7),packet_type(ns=0,id=0),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:3, bytes:294, used:0.242s, actions:ct(zone=10,nat),recirc(0x10)
tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({}),flags(-df+csum+key)),recirc_id(0x10),in_port(7),packet_type(ns=0,id=0),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:3, bytes:294, used:0.242s, actions:ct(commit,zone=3,nat(src=10.157.13.4)),recirc(0x11)
tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({}),flags(-df+csum+key)),recirc_id(0x11),in_port(7),packet_type(ns=0,id=0),eth(src=fa:16:3e:d4:cb:56/01:00:00:00:00:00,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(src=10.157.13.4/255.255.255.128,dst=10.157.13.7,proto=1,ttl=63,frag=no), packets:3, bytes:294, used:0.242s, actions:ct(zone=3,nat),recirc(0x12)
tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({}),flags(-df+csum+key)),recirc_id(0x12),in_port(7),packet_type(ns=0,id=0),eth_type(0x0800),ipv4(dst=10.157.13.7,frag=no), packets:3, bytes:294, used:0.242s, actions:ct(commit,zone=10,nat(dst=192.168.0.10)),recirc(0x13)
tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({class=0x102,type=0x80,len=4,0x30005}),flags(-df+csum+key)),ct_state(-new+est-rel-rpl-inv+trk),ct_label(0/0x1),recirc_id(0x13),in_port(7),packet_type(ns=0,id=0),eth(src=fa:16:3e:d4:cb:56,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(dst=192.168.0.10,tos=0/0x3,ttl=63,frag=no), packets:2, bytes:196, used:0.241s, actions:set(eth(src=fa:16:3e:e4:ec:cd,dst=fa:16:3e:ed:4b:8a)),set(ipv4(dst=192.168.0.10,tos=0/0x3,ttl=62)),clone(tnl_push(tnl_port(7),header(size=58,type=5,eth(dst=6c:92:bf:13:be:0a,src=6c:92:bf:04:4c:b2,dl_type=0x0800),ipv4(src=10.157.145.214,dst=10.157.145.212,proto=17,tos=0,ttl=64,frag=0x4000),udp(src=0,dst=6081,csum=0xffff),geneve(crit,vni=0x1,options({class=0x102,type=0x80,len=4,0x10002}))),out_port(4)),push_vlan(vid=2515,pcp=0),hash(hash_l4(0)),recirc(0x1))
recirc_id(0x1),dp_hash(0x79f1a62d/0xff),in_port(4),packet_type(ns=0,id=0),eth_type(0x8100),vlan(vid=2515,pcp=0),encap(eth_type(0x0800),ipv4(frag=no)), packets:2, bytes:320, used:0.242s, actions:5
2.13.2.windows.1
_______________________________________________
dev mailing list
dev@openvswitch.org
https://urldefense.proofpoint.com/v2/url?u=https-3A__mail.openvswitch.org_mailman_listinfo_ovs-2Ddev&d=DwICAg&c=uilaK90D4TOVoH58JNXRgQ&r=BVhFA09CGX7JQ5Ih-uZnsw&m=1O8U7S-DkEKfm_1rUi8wJWAG6DdtrvTUn3WqjRZ44jg&s=y3uThLIiu-kTEYgJZtkU7dfmyJ1BmhfSP0xFgYuBawo&e=
yes, Distributed gateway.
I config my gateway router port with 'redirect-chassis'option in ovn-nb.
Logical_Router_Port table
_uuid enabled external_ids gateway_chassis mac name networks options peer
------------------------------------ ------- ------------ --------------- ------------------- ------------------------------------------ -------------------- -------------------------------------------------------------------------------------- ----
85c04f4e-6109-4049-b651-3a1495a7901c [] {} [] "fa:16:3e:5d:65:b2" "lrp-8712a984-c019-4362-befe-8969e5fd0f11" ["192.168.2.254/24"] {} []
93324dc3-6863-4831-b990-86645f4b234b [] {} [] "fa:16:3e:9f:10:13" "lrp-107688b6-07e5-4a5b-9fa7-a1b25f56b6de" ["10.157.13.0/24"] {physical-network="physnet2", redirect-chassis="f0e5148b-b6a8-46a1-9561-de76c22245bd"} []
32de97cf-1de6-4b92-b8dc-05c95b920587 [] {} [] "fa:16:3e:d4:cb:56" "lrp-6c2f6700-cc1e-44c6-ada7-da2889aa9278" ["10.157.13.0/24"] {physical-network="physnet2", redirect-chassis="f0e5148b-b6a8-46a1-9561-de76c22245bd"} []
55824043-c7d9-40f7-8530-0872efc47edc [] {} [] "fa:16:3e:e4:ec:cd" "lrp-e84181ad-fa35-4b34-81e8-1e44218fc504" ["192.168.0.254/24"] {} []
My kernel datapath flows, less than DPDK:
recirc_id(0),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({class=0x102,type=0x80,len=4,0x30005/0x7fffffff}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:21, bytes:2058, used:0.637s, actions:ct(zone=10,nat),recirc(0x24)
recirc_id(0x24),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(commit,zone=3,nat(src=10.157.13.4)),recirc(0x25)
recirc_id(0x25),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth(src=00:00:00:00:00:00/01:00:00:00:00:00,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(src=10.157.13.0/255.255.255.128,dst=10.157.13.7,proto=1,ttl=63,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(zone=3,nat),recirc(0x26)
recirc_id(0x26),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(dst=10.157.13.7,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(commit,zone=10,nat(dst=192.168.0.10)),recirc(0x27)
recirc_id(0x27),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({class=0x102,type=0x80,len=4,0x30005}),flags(-df+csum+key)),in_port(2),ct_state(-new+est-rel-rpl-inv+trk),ct_label(0/0x1),eth(src=fa:16:3e:d4:cb:56,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(dst=192.168.0.10,tos=0/0x3,ttl=63,frag=no), packets:20, bytes:1960, used:0.638s, actions:set(tunnel(tun_id=0x1,dst=10.157.145.212,ttl=64,tp_src=37366,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(df|csum|key))),set(eth(src=fa:16:3e:e4:ec:cd,dst=fa:16:3e:ed:4b:8a)),set(ipv4(dst=192.168.0.10,tos=0/0x3,ttl=62)),2
在 2017/9/20 9:23, Darrell Ball :
> Distributed gateway ?
> What do the kernel datapath flows look like ?
>
>
> On 9/18/17, 11:29 PM, "ovs-dev-bounces@openvswitch.org on behalf of Guoshuai Li" <ovs-dev-bounces@openvswitch.org on behalf of ligs@dtdream.com> wrote:
>
> In an ovn gateway node with DPDK, the RECIRC_DEPTH may be greater than 5.
>
> Scenes:
> VM ping self floating IP, or
> VM ping Floating IP of VMs with the same network.
>
> It need process UNDNAT SNAT in LRouter egress and
> UNSNAT DNAT in LRouter ingress, and
> output to geneve tunnel also need recirc.
>
> This has an WARN:
> dpif_netdev(pmd36)|WARN|Packet dropped. Max recirculation depth exceeded.
> ---
> lib/dpif-netdev.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
> index ca74df8c7..2ad8c3143 100644
> --- a/lib/dpif-netdev.c
> +++ b/lib/dpif-netdev.c
> @@ -82,7 +82,7 @@ VLOG_DEFINE_THIS_MODULE(dpif_netdev);
>
> #define FLOW_DUMP_MAX_BATCH 50
> /* Use per thread recirc_depth to prevent recirculation loop. */
> -#define MAX_RECIRC_DEPTH 5
> +#define MAX_RECIRC_DEPTH 10
>
> I guess the MAX_RECIRC_DEPTH should be as small as possible for valid use cases – 6 ?
> The check is ‘less than’.
>
> DEFINE_STATIC_PER_THREAD_DATA(uint32_t, recirc_depth, 0)
>
> /* Configuration parameters. */
> --
>
> This is my flows by ovs-appctl dpctl/dump-flows:
> tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({class=0x102,type=0x80,len=4,0x30005/0x7fffffff}),flags(-df+csum+key)),recirc_id(0),in_port(7),packet_type(ns=0,id=0),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:3, bytes:294, used:0.242s, actions:ct(zone=10,nat),recirc(0x10)
> tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({}),flags(-df+csum+key)),recirc_id(0x10),in_port(7),packet_type(ns=0,id=0),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:3, bytes:294, used:0.242s, actions:ct(commit,zone=3,nat(src=10.157.13.4)),recirc(0x11)
> tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({}),flags(-df+csum+key)),recirc_id(0x11),in_port(7),packet_type(ns=0,id=0),eth(src=fa:16:3e:d4:cb:56/01:00:00:00:00:00,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(src=10.157.13.4/255.255.255.128,dst=10.157.13.7,proto=1,ttl=63,frag=no), packets:3, bytes:294, used:0.242s, actions:ct(zone=3,nat),recirc(0x12)
> tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({}),flags(-df+csum+key)),recirc_id(0x12),in_port(7),packet_type(ns=0,id=0),eth_type(0x0800),ipv4(dst=10.157.13.7,frag=no), packets:3, bytes:294, used:0.242s, actions:ct(commit,zone=10,nat(dst=192.168.0.10)),recirc(0x13)
> tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({class=0x102,type=0x80,len=4,0x30005}),flags(-df+csum+key)),ct_state(-new+est-rel-rpl-inv+trk),ct_label(0/0x1),recirc_id(0x13),in_port(7),packet_type(ns=0,id=0),eth(src=fa:16:3e:d4:cb:56,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(dst=192.168.0.10,tos=0/0x3,ttl=63,frag=no), packets:2, bytes:196, used:0.241s, actions:set(eth(src=fa:16:3e:e4:ec:cd,dst=fa:16:3e:ed:4b:8a)),set(ipv4(dst=192.168.0.10,tos=0/0x3,ttl=62)),clone(tnl_push(tnl_port(7),header(size=58,type=5,eth(dst=6c:92:bf:13:be:0a,src=6c:92:bf:04:4c:b2,dl_type=0x0800),ipv4(src=10.157.145.214,dst=10.157.145.212,proto=17,tos=0,ttl=64,frag=0x4000),udp(src=0,dst=6081,csum=0xffff),geneve(crit,vni=0x1,options({class=0x102,type=0x80,len=4,0x10002}))),out_port(4)),push_vlan(vid=2515,pcp=0),hash(hash_l4(0)),recirc(0x1))
> recirc_id(0x1),dp_hash(0x79f1a62d/0xff),in_port(4),packet_type(ns=0,id=0),eth_type(0x8100),vlan(vid=2515,pcp=0),encap(eth_type(0x0800),ipv4(frag=no)), packets:2, bytes:320, used:0.242s, actions:5
>
>
> 2.13.2.windows.1
>
> _______________________________________________
> dev mailing list
> dev@openvswitch.org
> https://urldefense.proofpoint.com/v2/url?u=https-3A__mail.openvswitch.org_mailman_listinfo_ovs-2Ddev&d=DwICAg&c=uilaK90D4TOVoH58JNXRgQ&r=BVhFA09CGX7JQ5Ih-uZnsw&m=1O8U7S-DkEKfm_1rUi8wJWAG6DdtrvTUn3WqjRZ44jg&s=y3uThLIiu-kTEYgJZtkU7dfmyJ1BmhfSP0xFgYuBawo&e=
>
>
>
>
>
>
On 9/20/17, 4:45 AM, "Guoshuai Li" <ligs@dtdream.com> wrote:
yes, Distributed gateway.
I config my gateway router port with 'redirect-chassis'option in ovn-nb.
Logical_Router_Port table
_uuid enabled external_ids gateway_chassis mac name networks options peer
------------------------------------ ------- ------------ --------------- ------------------- ------------------------------------------ -------------------- -------------------------------------------------------------------------------------- ----
85c04f4e-6109-4049-b651-3a1495a7901c [] {} [] "fa:16:3e:5d:65:b2" "lrp-8712a984-c019-4362-befe-8969e5fd0f11" ["192.168.2.254/24"] {} []
93324dc3-6863-4831-b990-86645f4b234b [] {} [] "fa:16:3e:9f:10:13" "lrp-107688b6-07e5-4a5b-9fa7-a1b25f56b6de" ["10.157.13.0/24"] {physical-network="physnet2", redirect-chassis="f0e5148b-b6a8-46a1-9561-de76c22245bd"} []
32de97cf-1de6-4b92-b8dc-05c95b920587 [] {} [] "fa:16:3e:d4:cb:56" "lrp-6c2f6700-cc1e-44c6-ada7-da2889aa9278" ["10.157.13.0/24"] {physical-network="physnet2", redirect-chassis="f0e5148b-b6a8-46a1-9561-de76c22245bd"} []
55824043-c7d9-40f7-8530-0872efc47edc [] {} [] "fa:16:3e:e4:ec:cd" "lrp-e84181ad-fa35-4b34-81e8-1e44218fc504" ["192.168.0.254/24"] {} []
My kernel datapath flows, less than DPDK:
recirc_id(0),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({class=0x102,type=0x80,len=4,0x30005/0x7fffffff}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:21, bytes:2058, used:0.637s, actions:ct(zone=10,nat),recirc(0x24)
recirc_id(0x24),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(commit,zone=3,nat(src=10.157.13.4)),recirc(0x25)
recirc_id(0x25),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth(src=00:00:00:00:00:00/01:00:00:00:00:00,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(src=10.157.13.0/255.255.255.128,dst=10.157.13.7,proto=1,ttl=63,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(zone=3,nat),recirc(0x26)
recirc_id(0x26),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(dst=10.157.13.7,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(commit,zone=10,nat(dst=192.168.0.10)),recirc(0x27)
recirc_id(0x27),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({class=0x102,type=0x80,len=4,0x30005}),flags(-df+csum+key)),in_port(2),ct_state(-new+est-rel-rpl-inv+trk),ct_label(0/0x1),eth(src=fa:16:3e:d4:cb:56,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(dst=192.168.0.10,tos=0/0x3,ttl=63,frag=no), packets:20, bytes:1960, used:0.638s, actions:set(tunnel(tun_id=0x1,dst=10.157.145.212,ttl=64,tp_src=37366,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(df|csum|key))),set(eth(src=fa:16:3e:e4:ec:cd,dst=fa:16:3e:ed:4b:8a)),set(ipv4(dst=192.168.0.10,tos=0/0x3,ttl=62)),2
[Darrell]
The difference is the native tunneling, so it seems to make sense.
Does #define MAX_RECIRC_DEPTH 6 work in all cases that you checked ?
Maybe setting to 7 leaves a margin of error and can also can alert if there is a bug or inefficiency ?
在 2017/9/20 9:23, Darrell Ball :
> Distributed gateway ?
> What do the kernel datapath flows look like ?
>
>
> On 9/18/17, 11:29 PM, "ovs-dev-bounces@openvswitch.org on behalf of Guoshuai Li" <ovs-dev-bounces@openvswitch.org on behalf of ligs@dtdream.com> wrote:
>
> In an ovn gateway node with DPDK, the RECIRC_DEPTH may be greater than 5.
>
> Scenes:
> VM ping self floating IP, or
> VM ping Floating IP of VMs with the same network.
>
> It need process UNDNAT SNAT in LRouter egress and
> UNSNAT DNAT in LRouter ingress, and
> output to geneve tunnel also need recirc.
>
> This has an WARN:
> dpif_netdev(pmd36)|WARN|Packet dropped. Max recirculation depth exceeded.
> ---
> lib/dpif-netdev.c | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
> index ca74df8c7..2ad8c3143 100644
> --- a/lib/dpif-netdev.c
> +++ b/lib/dpif-netdev.c
> @@ -82,7 +82,7 @@ VLOG_DEFINE_THIS_MODULE(dpif_netdev);
>
> #define FLOW_DUMP_MAX_BATCH 50
> /* Use per thread recirc_depth to prevent recirculation loop. */
> -#define MAX_RECIRC_DEPTH 5
> +#define MAX_RECIRC_DEPTH 10
>
> I guess the MAX_RECIRC_DEPTH should be as small as possible for valid use cases – 6 ?
> The check is ‘less than’.
>
> DEFINE_STATIC_PER_THREAD_DATA(uint32_t, recirc_depth, 0)
>
> /* Configuration parameters. */
> --
>
> This is my flows by ovs-appctl dpctl/dump-flows:
> tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({class=0x102,type=0x80,len=4,0x30005/0x7fffffff}),flags(-df+csum+key)),recirc_id(0),in_port(7),packet_type(ns=0,id=0),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:3, bytes:294, used:0.242s, actions:ct(zone=10,nat),recirc(0x10)
> tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({}),flags(-df+csum+key)),recirc_id(0x10),in_port(7),packet_type(ns=0,id=0),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:3, bytes:294, used:0.242s, actions:ct(commit,zone=3,nat(src=10.157.13.4)),recirc(0x11)
> tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({}),flags(-df+csum+key)),recirc_id(0x11),in_port(7),packet_type(ns=0,id=0),eth(src=fa:16:3e:d4:cb:56/01:00:00:00:00:00,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(src=10.157.13.4/255.255.255.128,dst=10.157.13.7,proto=1,ttl=63,frag=no), packets:3, bytes:294, used:0.242s, actions:ct(zone=3,nat),recirc(0x12)
> tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({}),flags(-df+csum+key)),recirc_id(0x12),in_port(7),packet_type(ns=0,id=0),eth_type(0x0800),ipv4(dst=10.157.13.7,frag=no), packets:3, bytes:294, used:0.242s, actions:ct(commit,zone=10,nat(dst=192.168.0.10)),recirc(0x13)
> tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.145.214,geneve({class=0x102,type=0x80,len=4,0x30005}),flags(-df+csum+key)),ct_state(-new+est-rel-rpl-inv+trk),ct_label(0/0x1),recirc_id(0x13),in_port(7),packet_type(ns=0,id=0),eth(src=fa:16:3e:d4:cb:56,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(dst=192.168.0.10,tos=0/0x3,ttl=63,frag=no), packets:2, bytes:196, used:0.241s, actions:set(eth(src=fa:16:3e:e4:ec:cd,dst=fa:16:3e:ed:4b:8a)),set(ipv4(dst=192.168.0.10,tos=0/0x3,ttl=62)),clone(tnl_push(tnl_port(7),header(size=58,type=5,eth(dst=6c:92:bf:13:be:0a,src=6c:92:bf:04:4c:b2,dl_type=0x0800),ipv4(src=10.157.145.214,dst=10.157.145.212,proto=17,tos=0,ttl=64,frag=0x4000),udp(src=0,dst=6081,csum=0xffff),geneve(crit,vni=0x1,options({class=0x102,type=0x80,len=4,0x10002}))),out_port(4)),push_vlan(vid=2515,pcp=0),hash(hash_l4(0)),recirc(0x1))
> recirc_id(0x1),dp_hash(0x79f1a62d/0xff),in_port(4),packet_type(ns=0,id=0),eth_type(0x8100),vlan(vid=2515,pcp=0),encap(eth_type(0x0800),ipv4(frag=no)), packets:2, bytes:320, used:0.242s, actions:5
>
>
> 2.13.2.windows.1
>
> _______________________________________________
> dev mailing list
> dev@openvswitch.org
> https://urldefense.proofpoint.com/v2/url?u=https-3A__mail.openvswitch.org_mailman_listinfo_ovs-2Ddev&d=DwICAg&c=uilaK90D4TOVoH58JNXRgQ&r=BVhFA09CGX7JQ5Ih-uZnsw&m=1O8U7S-DkEKfm_1rUi8wJWAG6DdtrvTUn3WqjRZ44jg&s=y3uThLIiu-kTEYgJZtkU7dfmyJ1BmhfSP0xFgYuBawo&e=
>
>
>
>
>
>
I send a patch to change DEPTH 6, It is Ok for my cases. https://mail.openvswitch.org/pipermail/ovs-dev/2017-September/339034.html It will warning log for 7 recirc, "dpif_netdev(pmd36)|WARN|Packet dropped. Max recirculation depth exceeded." on 2017/9/20 23:57, Darrell Ball write: > > On 9/20/17, 4:45 AM, "Guoshuai Li" <ligs@dtdream.com> wrote: > > yes, Distributed gateway. > I config my gateway router port with 'redirect-chassis'option in ovn-nb. > > Logical_Router_Port table > _uuid enabled external_ids gateway_chassis mac name networks options peer > ------------------------------------ ------- ------------ --------------- ------------------- ------------------------------------------ -------------------- -------------------------------------------------------------------------------------- ---- > 85c04f4e-6109-4049-b651-3a1495a7901c [] {} [] "fa:16:3e:5d:65:b2" "lrp-8712a984-c019-4362-befe-8969e5fd0f11" ["192.168.2.254/24"] {} [] > 93324dc3-6863-4831-b990-86645f4b234b [] {} [] "fa:16:3e:9f:10:13" "lrp-107688b6-07e5-4a5b-9fa7-a1b25f56b6de" ["10.157.13.0/24"] {physical-network="physnet2", redirect-chassis="f0e5148b-b6a8-46a1-9561-de76c22245bd"} [] > 32de97cf-1de6-4b92-b8dc-05c95b920587 [] {} [] "fa:16:3e:d4:cb:56" "lrp-6c2f6700-cc1e-44c6-ada7-da2889aa9278" ["10.157.13.0/24"] {physical-network="physnet2", redirect-chassis="f0e5148b-b6a8-46a1-9561-de76c22245bd"} [] > 55824043-c7d9-40f7-8530-0872efc47edc [] {} [] "fa:16:3e:e4:ec:cd" "lrp-e84181ad-fa35-4b34-81e8-1e44218fc504" ["192.168.0.254/24"] {} [] > > > My kernel datapath flows, less than DPDK: > recirc_id(0),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({class=0x102,type=0x80,len=4,0x30005/0x7fffffff}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:21, bytes:2058, used:0.637s, actions:ct(zone=10,nat),recirc(0x24) > recirc_id(0x24),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(commit,zone=3,nat(src=10.157.13.4)),recirc(0x25) > recirc_id(0x25),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth(src=00:00:00:00:00:00/01:00:00:00:00:00,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(src=10.157.13.0/255.255.255.128,dst=10.157.13.7,proto=1,ttl=63,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(zone=3,nat),recirc(0x26) > recirc_id(0x26),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(dst=10.157.13.7,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(commit,zone=10,nat(dst=192.168.0.10)),recirc(0x27) > recirc_id(0x27),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({class=0x102,type=0x80,len=4,0x30005}),flags(-df+csum+key)),in_port(2),ct_state(-new+est-rel-rpl-inv+trk),ct_label(0/0x1),eth(src=fa:16:3e:d4:cb:56,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(dst=192.168.0.10,tos=0/0x3,ttl=63,frag=no), packets:20, bytes:1960, used:0.638s, actions:set(tunnel(tun_id=0x1,dst=10.157.145.212,ttl=64,tp_src=37366,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(df|csum|key))),set(eth(src=fa:16:3e:e4:ec:cd,dst=fa:16:3e:ed:4b:8a)),set(ipv4(dst=192.168.0.10,tos=0/0x3,ttl=62)),2 > > [Darrell] > The difference is the native tunneling, so it seems to make sense. > Does #define MAX_RECIRC_DEPTH 6 work in all cases that you checked ? > Maybe setting to 7 leaves a margin of error and can also can alert if there is a bug or inefficiency ? > > > >
Hi Guoshuai
I committed on the Version 2 patch format and requested Ben to take a look.
Thanks Darrell
On 9/21/17, 7:55 PM, "Guoshuai Li" <ligs@dtdream.com> wrote:
I send a patch to change DEPTH 6, It is Ok for my cases.
https://urldefense.proofpoint.com/v2/url?u=https-3A__mail.openvswitch.org_pipermail_ovs-2Ddev_2017-2DSeptember_339034.html&d=DwIDaQ&c=uilaK90D4TOVoH58JNXRgQ&r=BVhFA09CGX7JQ5Ih-uZnsw&m=Am8f35FAI35QVJ1jdH2neTSqxFLmZqCeNiLutjdSe94&s=7NIxt2Mb7JtWyp5oVSZljx7Namf2Ei-moD39ShR-7Dg&e=
It will warning log for 7 recirc, "dpif_netdev(pmd36)|WARN|Packet
dropped. Max recirculation depth exceeded."
on 2017/9/20 23:57, Darrell Ball write:
>
> On 9/20/17, 4:45 AM, "Guoshuai Li" <ligs@dtdream.com> wrote:
>
> yes, Distributed gateway.
> I config my gateway router port with 'redirect-chassis'option in ovn-nb.
>
> Logical_Router_Port table
> _uuid enabled external_ids gateway_chassis mac name networks options peer
> ------------------------------------ ------- ------------ --------------- ------------------- ------------------------------------------ -------------------- -------------------------------------------------------------------------------------- ----
> 85c04f4e-6109-4049-b651-3a1495a7901c [] {} [] "fa:16:3e:5d:65:b2" "lrp-8712a984-c019-4362-befe-8969e5fd0f11" ["192.168.2.254/24"] {} []
> 93324dc3-6863-4831-b990-86645f4b234b [] {} [] "fa:16:3e:9f:10:13" "lrp-107688b6-07e5-4a5b-9fa7-a1b25f56b6de" ["10.157.13.0/24"] {physical-network="physnet2", redirect-chassis="f0e5148b-b6a8-46a1-9561-de76c22245bd"} []
> 32de97cf-1de6-4b92-b8dc-05c95b920587 [] {} [] "fa:16:3e:d4:cb:56" "lrp-6c2f6700-cc1e-44c6-ada7-da2889aa9278" ["10.157.13.0/24"] {physical-network="physnet2", redirect-chassis="f0e5148b-b6a8-46a1-9561-de76c22245bd"} []
> 55824043-c7d9-40f7-8530-0872efc47edc [] {} [] "fa:16:3e:e4:ec:cd" "lrp-e84181ad-fa35-4b34-81e8-1e44218fc504" ["192.168.0.254/24"] {} []
>
>
> My kernel datapath flows, less than DPDK:
> recirc_id(0),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({class=0x102,type=0x80,len=4,0x30005/0x7fffffff}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:21, bytes:2058, used:0.637s, actions:ct(zone=10,nat),recirc(0x24)
> recirc_id(0x24),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(commit,zone=3,nat(src=10.157.13.4)),recirc(0x25)
> recirc_id(0x25),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth(src=00:00:00:00:00:00/01:00:00:00:00:00,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(src=10.157.13.0/255.255.255.128,dst=10.157.13.7,proto=1,ttl=63,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(zone=3,nat),recirc(0x26)
> recirc_id(0x26),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(dst=10.157.13.7,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(commit,zone=10,nat(dst=192.168.0.10)),recirc(0x27)
> recirc_id(0x27),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({class=0x102,type=0x80,len=4,0x30005}),flags(-df+csum+key)),in_port(2),ct_state(-new+est-rel-rpl-inv+trk),ct_label(0/0x1),eth(src=fa:16:3e:d4:cb:56,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(dst=192.168.0.10,tos=0/0x3,ttl=63,frag=no), packets:20, bytes:1960, used:0.638s, actions:set(tunnel(tun_id=0x1,dst=10.157.145.212,ttl=64,tp_src=37366,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(df|csum|key))),set(eth(src=fa:16:3e:e4:ec:cd,dst=fa:16:3e:ed:4b:8a)),set(ipv4(dst=192.168.0.10,tos=0/0x3,ttl=62)),2
>
> [Darrell]
> The difference is the native tunneling, so it seems to make sense.
> Does #define MAX_RECIRC_DEPTH 6 work in all cases that you checked ?
> Maybe setting to 7 leaves a margin of error and can also can alert if there is a bug or inefficiency ?
>
>
>
>
Correction inline:
s/committed/commented
On 9/21/17, 9:16 PM, "Darrell Ball" <dball@vmware.com> wrote:
Hi Guoshuai
I commented on the Version 2 patch format and requested Ben to take a look.
Thanks Darrell
On 9/21/17, 7:55 PM, "Guoshuai Li" <ligs@dtdream.com> wrote:
I send a patch to change DEPTH 6, It is Ok for my cases.
https://urldefense.proofpoint.com/v2/url?u=https-3A__mail.openvswitch.org_pipermail_ovs-2Ddev_2017-2DSeptember_339034.html&d=DwIDaQ&c=uilaK90D4TOVoH58JNXRgQ&r=BVhFA09CGX7JQ5Ih-uZnsw&m=Am8f35FAI35QVJ1jdH2neTSqxFLmZqCeNiLutjdSe94&s=7NIxt2Mb7JtWyp5oVSZljx7Namf2Ei-moD39ShR-7Dg&e=
It will warning log for 7 recirc, "dpif_netdev(pmd36)|WARN|Packet
dropped. Max recirculation depth exceeded."
on 2017/9/20 23:57, Darrell Ball write:
>
> On 9/20/17, 4:45 AM, "Guoshuai Li" <ligs@dtdream.com> wrote:
>
> yes, Distributed gateway.
> I config my gateway router port with 'redirect-chassis'option in ovn-nb.
>
> Logical_Router_Port table
> _uuid enabled external_ids gateway_chassis mac name networks options peer
> ------------------------------------ ------- ------------ --------------- ------------------- ------------------------------------------ -------------------- -------------------------------------------------------------------------------------- ----
> 85c04f4e-6109-4049-b651-3a1495a7901c [] {} [] "fa:16:3e:5d:65:b2" "lrp-8712a984-c019-4362-befe-8969e5fd0f11" ["192.168.2.254/24"] {} []
> 93324dc3-6863-4831-b990-86645f4b234b [] {} [] "fa:16:3e:9f:10:13" "lrp-107688b6-07e5-4a5b-9fa7-a1b25f56b6de" ["10.157.13.0/24"] {physical-network="physnet2", redirect-chassis="f0e5148b-b6a8-46a1-9561-de76c22245bd"} []
> 32de97cf-1de6-4b92-b8dc-05c95b920587 [] {} [] "fa:16:3e:d4:cb:56" "lrp-6c2f6700-cc1e-44c6-ada7-da2889aa9278" ["10.157.13.0/24"] {physical-network="physnet2", redirect-chassis="f0e5148b-b6a8-46a1-9561-de76c22245bd"} []
> 55824043-c7d9-40f7-8530-0872efc47edc [] {} [] "fa:16:3e:e4:ec:cd" "lrp-e84181ad-fa35-4b34-81e8-1e44218fc504" ["192.168.0.254/24"] {} []
>
>
> My kernel datapath flows, less than DPDK:
> recirc_id(0),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({class=0x102,type=0x80,len=4,0x30005/0x7fffffff}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:21, bytes:2058, used:0.637s, actions:ct(zone=10,nat),recirc(0x24)
> recirc_id(0x24),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(src=192.168.0.6,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(commit,zone=3,nat(src=10.157.13.4)),recirc(0x25)
> recirc_id(0x25),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth(src=00:00:00:00:00:00/01:00:00:00:00:00,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(src=10.157.13.0/255.255.255.128,dst=10.157.13.7,proto=1,ttl=63,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(zone=3,nat),recirc(0x26)
> recirc_id(0x26),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({}{}),flags(-df+csum+key)),in_port(2),eth_type(0x0800),ipv4(dst=10.157.13.7,frag=no), packets:21, bytes:2058, used:0.638s, actions:ct(commit,zone=10,nat(dst=192.168.0.10)),recirc(0x27)
> recirc_id(0x27),tunnel(tun_id=0x2,src=10.157.145.211,dst=10.157.12.111,geneve({class=0x102,type=0x80,len=4,0x30005}),flags(-df+csum+key)),in_port(2),ct_state(-new+est-rel-rpl-inv+trk),ct_label(0/0x1),eth(src=fa:16:3e:d4:cb:56,dst=fa:16:3e:d4:cb:56),eth_type(0x0800),ipv4(dst=192.168.0.10,tos=0/0x3,ttl=63,frag=no), packets:20, bytes:1960, used:0.638s, actions:set(tunnel(tun_id=0x1,dst=10.157.145.212,ttl=64,tp_src=37366,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10002}),flags(df|csum|key))),set(eth(src=fa:16:3e:e4:ec:cd,dst=fa:16:3e:ed:4b:8a)),set(ipv4(dst=192.168.0.10,tos=0/0x3,ttl=62)),2
>
> [Darrell]
> The difference is the native tunneling, so it seems to make sense.
> Does #define MAX_RECIRC_DEPTH 6 work in all cases that you checked ?
> Maybe setting to 7 leaves a margin of error and can also can alert if there is a bug or inefficiency ?
>
>
>
>
diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c index ca74df8c7..2ad8c3143 100644 --- a/lib/dpif-netdev.c +++ b/lib/dpif-netdev.c @@ -82,7 +82,7 @@ VLOG_DEFINE_THIS_MODULE(dpif_netdev); #define FLOW_DUMP_MAX_BATCH 50 /* Use per thread recirc_depth to prevent recirculation loop. */ -#define MAX_RECIRC_DEPTH 5 +#define MAX_RECIRC_DEPTH 10 DEFINE_STATIC_PER_THREAD_DATA(uint32_t, recirc_depth, 0) /* Configuration parameters. */