diff mbox

[RFC,V10,7/7] colo-compare: add TCP, UDP, ICMP packet comparison

Message ID 1469497794-16976-8-git-send-email-zhangchen.fnst@cn.fujitsu.com
State New
Headers show

Commit Message

Zhang Chen July 26, 2016, 1:49 a.m. UTC
We add TCP,UDP,ICMP packet comparison to replace
IP packet comparison. This can increase the
accuracy of the package comparison.
less checkpoint more efficiency.

Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
---
 net/colo-compare.c | 174 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 trace-events       |   4 ++
 2 files changed, 174 insertions(+), 4 deletions(-)

Comments

Jason Wang Aug. 2, 2016, 8:04 a.m. UTC | #1
On 2016年07月26日 09:49, Zhang Chen wrote:
> We add TCP,UDP,ICMP packet comparison to replace
> IP packet comparison. This can increase the
> accuracy of the package comparison.
> less checkpoint more efficiency.

s/less/Less/

>
> Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>
> Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
> ---
>   net/colo-compare.c | 174 +++++++++++++++++++++++++++++++++++++++++++++++++++--
>   trace-events       |   4 ++
>   2 files changed, 174 insertions(+), 4 deletions(-)
>
> diff --git a/net/colo-compare.c b/net/colo-compare.c
> index e020edc..c7bb5f7 100644
> --- a/net/colo-compare.c
> +++ b/net/colo-compare.c
> @@ -18,6 +18,7 @@
>   #include "qapi/qmp/qerror.h"
>   #include "qapi/error.h"
>   #include "net/net.h"
> +#include "net/eth.h"
>   #include "net/vhost_net.h"
>   #include "qom/object_interfaces.h"
>   #include "qemu/iov.h"
> @@ -197,9 +198,158 @@ static int colo_packet_compare(Packet *ppkt, Packet *spkt)
>       }
>   }
>   
> -static int colo_packet_compare_all(Packet *spkt, Packet *ppkt)
> +/*
> + * called from the compare thread on the primary
> + * for compare tcp packet
> + * compare_tcp copied from Dr. David Alan Gilbert's branch
> + */
> +static int colo_packet_compare_tcp(Packet *spkt, Packet *ppkt)
> +{
> +    struct tcphdr *ptcp, *stcp;
> +    int res;
> +    char *sdebug, *ddebug;
> +
> +    trace_colo_compare_main("compare tcp");
> +    if (ppkt->size != spkt->size) {
> +        if (trace_event_get_state(TRACE_COLO_COMPARE_MISCOMPARE)) {
> +            trace_colo_compare_main("pkt size not same");
> +        }
> +        return -1;
> +    }
> +
> +    ptcp = (struct tcphdr *)ppkt->transport_layer;
> +    stcp = (struct tcphdr *)spkt->transport_layer;
> +
> +    if (ptcp->th_seq != stcp->th_seq) {
> +        if (trace_event_get_state(TRACE_COLO_COMPARE_MISCOMPARE)) {
> +            trace_colo_compare_main("pkt tcp seq not same");
> +        }
> +        return -1;
> +    }
> +
> +    /*
> +     * The 'identification' field in the IP header is *very* random
> +     * it almost never matches.  Fudge this by ignoring differences in
> +     * unfragmented packets; they'll normally sort themselves out if different
> +     * anyway, and it should recover at the TCP level.
> +     * An alternative would be to get both the primary and secondary to rewrite
> +     * somehow; but that would need some sync traffic to sync the state
> +     */
> +    if (ntohs(ppkt->ip->ip_off) & IP_DF) {
> +        spkt->ip->ip_id = ppkt->ip->ip_id;
> +        /* and the sum will be different if the IDs were different */
> +        spkt->ip->ip_sum = ppkt->ip->ip_sum;

This looks dangerous. If packet were not logical same, ip cusm of 
secondary were changed.

> +    }
> +
> +    res = memcmp(ppkt->data + ETH_HLEN, spkt->data + ETH_HLEN,
> +                (spkt->size - ETH_HLEN));
> +
> +    if (res != 0 && trace_event_get_state(TRACE_COLO_COMPARE_MISCOMPARE)) {
> +        sdebug = strdup(inet_ntoa(ppkt->ip->ip_src));
> +        ddebug = strdup(inet_ntoa(ppkt->ip->ip_dst));
> +        fprintf(stderr, "%s: src/dst: %s/%s p: seq/ack=%u/%u"
> +        " s: seq/ack=%u/%u res=%d flags=%x/%x\n", __func__,
> +                   sdebug, ddebug,
> +                   ntohl(ptcp->th_seq), ntohl(ptcp->th_ack),
> +                   ntohl(stcp->th_seq), ntohl(stcp->th_ack),
> +                   res, ptcp->th_flags, stcp->th_flags);
> +
> +        trace_colo_compare_tcp_miscompare("Primary len", ppkt->size);
> +        qemu_hexdump((char *)ppkt->data, stderr, "colo-compare", ppkt->size);
> +        trace_colo_compare_tcp_miscompare("Secondary len", spkt->size);
> +        qemu_hexdump((char *)spkt->data, stderr, "colo-compare", spkt->size);
> +
> +        g_free(sdebug);
> +        g_free(ddebug);
> +    }
> +
> +    return res;
> +}
> +
> +/*
> + * called from the compare thread on the primary
> + * for compare udp packet
> + */
> +static int colo_packet_compare_udp(Packet *spkt, Packet *ppkt)
> +{
> +    int ret;
> +
> +    trace_colo_compare_main("compare udp");
> +    ret = colo_packet_compare(ppkt, spkt);
> +
> +    if (ret) {
> +        trace_colo_compare_udp_miscompare("primary pkt size", ppkt->size);
> +        qemu_hexdump((char *)ppkt->data, stderr, "colo-compare", ppkt->size);
> +        trace_colo_compare_udp_miscompare("Secondary pkt size", spkt->size);
> +        qemu_hexdump((char *)spkt->data, stderr, "colo-compare", spkt->size);
> +    }
> +
> +    return ret;
> +}
> +
> +/*
> + * called from the compare thread on the primary
> + * for compare icmp packet
> + */
> +static int colo_packet_compare_icmp(Packet *spkt, Packet *ppkt)
>   {

Consider icmp packet were usually not big, why not simply use memcpy? (I 
think I asked the same question in the past).

> -    trace_colo_compare_main("compare all");
> +    int network_length;
> +    struct icmp *icmp_ppkt, *icmp_spkt;
> +
> +    trace_colo_compare_main("compare icmp");
> +    network_length = ppkt->ip->ip_hl * 4;
> +    if (ppkt->size != spkt->size ||
> +        ppkt->size < network_length + ETH_HLEN) {
> +        return -1;
> +    }
> +    icmp_ppkt = (struct icmp *)(ppkt->data + network_length + ETH_HLEN);
> +    icmp_spkt = (struct icmp *)(spkt->data + network_length + ETH_HLEN);
> +
> +    if ((icmp_ppkt->icmp_type == icmp_spkt->icmp_type) &&
> +        (icmp_ppkt->icmp_code == icmp_spkt->icmp_code)) {
> +        if (icmp_ppkt->icmp_type == ICMP_REDIRECT) {
> +            if (icmp_ppkt->icmp_gwaddr.s_addr !=
> +                icmp_spkt->icmp_gwaddr.s_addr) {
> +                trace_colo_compare_main("icmp_gwaddr.s_addr not same");
> +                return -1;
> +            }
> +        } else if ((icmp_ppkt->icmp_type == ICMP_UNREACH) &&
> +                   (icmp_ppkt->icmp_type == ICMP_UNREACH_NEEDFRAG)) {
> +            if (icmp_ppkt->icmp_nextmtu != icmp_spkt->icmp_nextmtu) {
> +                trace_colo_compare_main("icmp_nextmtu not same");
> +                return -1;
> +            }
> +        }
> +
> +        if (colo_packet_compare(ppkt, spkt)) {
> +            trace_colo_compare_icmp_miscompare("primary pkt size",
> +                                               ppkt->size);
> +            qemu_hexdump((char *)ppkt->data, stderr, "colo-compare",
> +                         ppkt->size);
> +            trace_colo_compare_icmp_miscompare("Secondary pkt size",
> +                                               spkt->size);
> +            qemu_hexdump((char *)spkt->data, stderr, "colo-compare",
> +                         spkt->size);
> +            return -1;
> +        }
> +    } else {
> +        return -1;
> +    }
> +
> +    return 0;
> +}
> +
> +/*
> + * called from the compare thread on the primary
> + * for compare other packet
> + */
> +static int colo_packet_compare_other(Packet *spkt, Packet *ppkt)
> +{
> +    trace_colo_compare_main("compare other");
> +    trace_colo_compare_ip_info(ppkt->size, inet_ntoa(ppkt->ip->ip_src),
> +                               inet_ntoa(ppkt->ip->ip_dst), spkt->size,
> +                               inet_ntoa(spkt->ip->ip_src),
> +                               inet_ntoa(spkt->ip->ip_dst));
>       return colo_packet_compare(ppkt, spkt);
>   }
>   
> @@ -263,8 +413,24 @@ static void colo_compare_connection(void *opaque, void *user_data)
>       while (!g_queue_is_empty(&conn->primary_list) &&
>              !g_queue_is_empty(&conn->secondary_list)) {
>           pkt = g_queue_pop_tail(&conn->primary_list);
> -        result = g_queue_find_custom(&conn->secondary_list,
> -                              pkt, (GCompareFunc)colo_packet_compare_all);
> +        switch (conn->ip_proto) {
> +        case IPPROTO_TCP:
> +            result = g_queue_find_custom(&conn->secondary_list,
> +                     pkt, (GCompareFunc)colo_packet_compare_tcp);
> +            break;
> +        case IPPROTO_UDP:
> +            result = g_queue_find_custom(&conn->secondary_list,
> +                     pkt, (GCompareFunc)colo_packet_compare_udp);
> +            break;
> +        case IPPROTO_ICMP:
> +            result = g_queue_find_custom(&conn->secondary_list,
> +                     pkt, (GCompareFunc)colo_packet_compare_icmp);
> +            break;
> +        default:
> +            result = g_queue_find_custom(&conn->secondary_list,
> +                     pkt, (GCompareFunc)colo_packet_compare_other);
> +            break;
> +        }
>   
>           if (result) {
>               ret = compare_chr_send(s->chr_out, pkt->data, pkt->size);
> diff --git a/trace-events b/trace-events
> index 1537e91..ab22eb2 100644
> --- a/trace-events
> +++ b/trace-events
> @@ -1919,5 +1919,9 @@ aspeed_vic_write(uint64_t offset, unsigned size, uint32_t data) "To 0x%" PRIx64
>   
>   # net/colo-compare.c
>   colo_compare_main(const char *chr) ": %s"
> +colo_compare_tcp_miscompare(const char *sta, int size) ": %s = %d"
> +colo_compare_udp_miscompare(const char *sta, int size) ": %s = %d"
> +colo_compare_icmp_miscompare(const char *sta, int size) ": %s = %d"
>   colo_compare_ip_info(int psize, const char *sta, const char *stb, int ssize, const char *stc, const char *std) "ppkt size = %d, ip_src = %s, ip_dst = %s, spkt size = %d, ip_src = %s, ip_dst = %s"
>   colo_old_packet_check_found(int64_t old_time) "%" PRId64
> +colo_compare_miscompare(void) ""
Zhang Chen Aug. 5, 2016, 8:55 a.m. UTC | #2
On 08/02/2016 04:04 PM, Jason Wang wrote:
>
>
> On 2016年07月26日 09:49, Zhang Chen wrote:
>> We add TCP,UDP,ICMP packet comparison to replace
>> IP packet comparison. This can increase the
>> accuracy of the package comparison.
>> less checkpoint more efficiency.
>
> s/less/Less/

OK.

>
>>
>> Signed-off-by: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>
>> Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
>> Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
>> ---
>>   net/colo-compare.c | 174 
>> +++++++++++++++++++++++++++++++++++++++++++++++++++--
>>   trace-events       |   4 ++
>>   2 files changed, 174 insertions(+), 4 deletions(-)
>>
>> diff --git a/net/colo-compare.c b/net/colo-compare.c
>> index e020edc..c7bb5f7 100644
>> --- a/net/colo-compare.c
>> +++ b/net/colo-compare.c
>> @@ -18,6 +18,7 @@
>>   #include "qapi/qmp/qerror.h"
>>   #include "qapi/error.h"
>>   #include "net/net.h"
>> +#include "net/eth.h"
>>   #include "net/vhost_net.h"
>>   #include "qom/object_interfaces.h"
>>   #include "qemu/iov.h"
>> @@ -197,9 +198,158 @@ static int colo_packet_compare(Packet *ppkt, 
>> Packet *spkt)
>>       }
>>   }
>>   -static int colo_packet_compare_all(Packet *spkt, Packet *ppkt)
>> +/*
>> + * called from the compare thread on the primary
>> + * for compare tcp packet
>> + * compare_tcp copied from Dr. David Alan Gilbert's branch
>> + */
>> +static int colo_packet_compare_tcp(Packet *spkt, Packet *ppkt)
>> +{
>> +    struct tcphdr *ptcp, *stcp;
>> +    int res;
>> +    char *sdebug, *ddebug;
>> +
>> +    trace_colo_compare_main("compare tcp");
>> +    if (ppkt->size != spkt->size) {
>> +        if (trace_event_get_state(TRACE_COLO_COMPARE_MISCOMPARE)) {
>> +            trace_colo_compare_main("pkt size not same");
>> +        }
>> +        return -1;
>> +    }
>> +
>> +    ptcp = (struct tcphdr *)ppkt->transport_layer;
>> +    stcp = (struct tcphdr *)spkt->transport_layer;
>> +
>> +    if (ptcp->th_seq != stcp->th_seq) {
>> +        if (trace_event_get_state(TRACE_COLO_COMPARE_MISCOMPARE)) {
>> +            trace_colo_compare_main("pkt tcp seq not same");
>> +        }
>> +        return -1;
>> +    }
>> +
>> +    /*
>> +     * The 'identification' field in the IP header is *very* random
>> +     * it almost never matches.  Fudge this by ignoring differences in
>> +     * unfragmented packets; they'll normally sort themselves out if 
>> different
>> +     * anyway, and it should recover at the TCP level.
>> +     * An alternative would be to get both the primary and secondary 
>> to rewrite
>> +     * somehow; but that would need some sync traffic to sync the state
>> +     */
>> +    if (ntohs(ppkt->ip->ip_off) & IP_DF) {
>> +        spkt->ip->ip_id = ppkt->ip->ip_id;
>> +        /* and the sum will be different if the IDs were different */
>> +        spkt->ip->ip_sum = ppkt->ip->ip_sum;
>
> This looks dangerous. If packet were not logical same, ip cusm of 
> secondary were changed.

If packet not same, we will do checkpoint.


>
>> +    }
>> +
>> +    res = memcmp(ppkt->data + ETH_HLEN, spkt->data + ETH_HLEN,
>> +                (spkt->size - ETH_HLEN));
>> +
>> +    if (res != 0 && 
>> trace_event_get_state(TRACE_COLO_COMPARE_MISCOMPARE)) {
>> +        sdebug = strdup(inet_ntoa(ppkt->ip->ip_src));
>> +        ddebug = strdup(inet_ntoa(ppkt->ip->ip_dst));
>> +        fprintf(stderr, "%s: src/dst: %s/%s p: seq/ack=%u/%u"
>> +        " s: seq/ack=%u/%u res=%d flags=%x/%x\n", __func__,
>> +                   sdebug, ddebug,
>> +                   ntohl(ptcp->th_seq), ntohl(ptcp->th_ack),
>> +                   ntohl(stcp->th_seq), ntohl(stcp->th_ack),
>> +                   res, ptcp->th_flags, stcp->th_flags);
>> +
>> +        trace_colo_compare_tcp_miscompare("Primary len", ppkt->size);
>> +        qemu_hexdump((char *)ppkt->data, stderr, "colo-compare", 
>> ppkt->size);
>> +        trace_colo_compare_tcp_miscompare("Secondary len", spkt->size);
>> +        qemu_hexdump((char *)spkt->data, stderr, "colo-compare", 
>> spkt->size);
>> +
>> +        g_free(sdebug);
>> +        g_free(ddebug);
>> +    }
>> +
>> +    return res;
>> +}
>> +
>> +/*
>> + * called from the compare thread on the primary
>> + * for compare udp packet
>> + */
>> +static int colo_packet_compare_udp(Packet *spkt, Packet *ppkt)
>> +{
>> +    int ret;
>> +
>> +    trace_colo_compare_main("compare udp");
>> +    ret = colo_packet_compare(ppkt, spkt);
>> +
>> +    if (ret) {
>> +        trace_colo_compare_udp_miscompare("primary pkt size", 
>> ppkt->size);
>> +        qemu_hexdump((char *)ppkt->data, stderr, "colo-compare", 
>> ppkt->size);
>> +        trace_colo_compare_udp_miscompare("Secondary pkt size", 
>> spkt->size);
>> +        qemu_hexdump((char *)spkt->data, stderr, "colo-compare", 
>> spkt->size);
>> +    }
>> +
>> +    return ret;
>> +}
>> +
>> +/*
>> + * called from the compare thread on the primary
>> + * for compare icmp packet
>> + */
>> +static int colo_packet_compare_icmp(Packet *spkt, Packet *ppkt)
>>   {
>
> Consider icmp packet were usually not big, why not simply use memcpy? 
> (I think I asked the same question in the past).
>

In past we add the colo_packet_compare(ppkt, spkt) in tail to compare 
all packet,
by the head of this function we can know what part of icmp pkt 
difference when packet not same.
So, for this comments I will remove head of this function in next version.

Thanks
Zhang Chen

>> -    trace_colo_compare_main("compare all");
>> +    int network_length;
>> +    struct icmp *icmp_ppkt, *icmp_spkt;
>> +
>> +    trace_colo_compare_main("compare icmp");
>> +    network_length = ppkt->ip->ip_hl * 4;
>> +    if (ppkt->size != spkt->size ||
>> +        ppkt->size < network_length + ETH_HLEN) {
>> +        return -1;
>> +    }
>> +    icmp_ppkt = (struct icmp *)(ppkt->data + network_length + 
>> ETH_HLEN);
>> +    icmp_spkt = (struct icmp *)(spkt->data + network_length + 
>> ETH_HLEN);
>> +
>> +    if ((icmp_ppkt->icmp_type == icmp_spkt->icmp_type) &&
>> +        (icmp_ppkt->icmp_code == icmp_spkt->icmp_code)) {
>> +        if (icmp_ppkt->icmp_type == ICMP_REDIRECT) {
>> +            if (icmp_ppkt->icmp_gwaddr.s_addr !=
>> +                icmp_spkt->icmp_gwaddr.s_addr) {
>> +                trace_colo_compare_main("icmp_gwaddr.s_addr not same");
>> +                return -1;
>> +            }
>> +        } else if ((icmp_ppkt->icmp_type == ICMP_UNREACH) &&
>> +                   (icmp_ppkt->icmp_type == ICMP_UNREACH_NEEDFRAG)) {
>> +            if (icmp_ppkt->icmp_nextmtu != icmp_spkt->icmp_nextmtu) {
>> +                trace_colo_compare_main("icmp_nextmtu not same");
>> +                return -1;
>> +            }
>> +        }
>> +
>> +        if (colo_packet_compare(ppkt, spkt)) {
>> +            trace_colo_compare_icmp_miscompare("primary pkt size",
>> +                                               ppkt->size);
>> +            qemu_hexdump((char *)ppkt->data, stderr, "colo-compare",
>> +                         ppkt->size);
>> +            trace_colo_compare_icmp_miscompare("Secondary pkt size",
>> +                                               spkt->size);
>> +            qemu_hexdump((char *)spkt->data, stderr, "colo-compare",
>> +                         spkt->size);
>> +            return -1;
>> +        }
>> +    } else {
>> +        return -1;
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>> +/*
>> + * called from the compare thread on the primary
>> + * for compare other packet
>> + */
>> +static int colo_packet_compare_other(Packet *spkt, Packet *ppkt)
>> +{
>> +    trace_colo_compare_main("compare other");
>> +    trace_colo_compare_ip_info(ppkt->size, inet_ntoa(ppkt->ip->ip_src),
>> + inet_ntoa(ppkt->ip->ip_dst), spkt->size,
>> + inet_ntoa(spkt->ip->ip_src),
>> + inet_ntoa(spkt->ip->ip_dst));
>>       return colo_packet_compare(ppkt, spkt);
>>   }
>>   @@ -263,8 +413,24 @@ static void colo_compare_connection(void 
>> *opaque, void *user_data)
>>       while (!g_queue_is_empty(&conn->primary_list) &&
>>              !g_queue_is_empty(&conn->secondary_list)) {
>>           pkt = g_queue_pop_tail(&conn->primary_list);
>> -        result = g_queue_find_custom(&conn->secondary_list,
>> -                              pkt, 
>> (GCompareFunc)colo_packet_compare_all);
>> +        switch (conn->ip_proto) {
>> +        case IPPROTO_TCP:
>> +            result = g_queue_find_custom(&conn->secondary_list,
>> +                     pkt, (GCompareFunc)colo_packet_compare_tcp);
>> +            break;
>> +        case IPPROTO_UDP:
>> +            result = g_queue_find_custom(&conn->secondary_list,
>> +                     pkt, (GCompareFunc)colo_packet_compare_udp);
>> +            break;
>> +        case IPPROTO_ICMP:
>> +            result = g_queue_find_custom(&conn->secondary_list,
>> +                     pkt, (GCompareFunc)colo_packet_compare_icmp);
>> +            break;
>> +        default:
>> +            result = g_queue_find_custom(&conn->secondary_list,
>> +                     pkt, (GCompareFunc)colo_packet_compare_other);
>> +            break;
>> +        }
>>             if (result) {
>>               ret = compare_chr_send(s->chr_out, pkt->data, pkt->size);
>> diff --git a/trace-events b/trace-events
>> index 1537e91..ab22eb2 100644
>> --- a/trace-events
>> +++ b/trace-events
>> @@ -1919,5 +1919,9 @@ aspeed_vic_write(uint64_t offset, unsigned 
>> size, uint32_t data) "To 0x%" PRIx64
>>     # net/colo-compare.c
>>   colo_compare_main(const char *chr) ": %s"
>> +colo_compare_tcp_miscompare(const char *sta, int size) ": %s = %d"
>> +colo_compare_udp_miscompare(const char *sta, int size) ": %s = %d"
>> +colo_compare_icmp_miscompare(const char *sta, int size) ": %s = %d"
>>   colo_compare_ip_info(int psize, const char *sta, const char *stb, 
>> int ssize, const char *stc, const char *std) "ppkt size = %d, ip_src 
>> = %s, ip_dst = %s, spkt size = %d, ip_src = %s, ip_dst = %s"
>>   colo_old_packet_check_found(int64_t old_time) "%" PRId64
>> +colo_compare_miscompare(void) ""
>
>
>
> .
>
diff mbox

Patch

diff --git a/net/colo-compare.c b/net/colo-compare.c
index e020edc..c7bb5f7 100644
--- a/net/colo-compare.c
+++ b/net/colo-compare.c
@@ -18,6 +18,7 @@ 
 #include "qapi/qmp/qerror.h"
 #include "qapi/error.h"
 #include "net/net.h"
+#include "net/eth.h"
 #include "net/vhost_net.h"
 #include "qom/object_interfaces.h"
 #include "qemu/iov.h"
@@ -197,9 +198,158 @@  static int colo_packet_compare(Packet *ppkt, Packet *spkt)
     }
 }
 
-static int colo_packet_compare_all(Packet *spkt, Packet *ppkt)
+/*
+ * called from the compare thread on the primary
+ * for compare tcp packet
+ * compare_tcp copied from Dr. David Alan Gilbert's branch
+ */
+static int colo_packet_compare_tcp(Packet *spkt, Packet *ppkt)
+{
+    struct tcphdr *ptcp, *stcp;
+    int res;
+    char *sdebug, *ddebug;
+
+    trace_colo_compare_main("compare tcp");
+    if (ppkt->size != spkt->size) {
+        if (trace_event_get_state(TRACE_COLO_COMPARE_MISCOMPARE)) {
+            trace_colo_compare_main("pkt size not same");
+        }
+        return -1;
+    }
+
+    ptcp = (struct tcphdr *)ppkt->transport_layer;
+    stcp = (struct tcphdr *)spkt->transport_layer;
+
+    if (ptcp->th_seq != stcp->th_seq) {
+        if (trace_event_get_state(TRACE_COLO_COMPARE_MISCOMPARE)) {
+            trace_colo_compare_main("pkt tcp seq not same");
+        }
+        return -1;
+    }
+
+    /*
+     * The 'identification' field in the IP header is *very* random
+     * it almost never matches.  Fudge this by ignoring differences in
+     * unfragmented packets; they'll normally sort themselves out if different
+     * anyway, and it should recover at the TCP level.
+     * An alternative would be to get both the primary and secondary to rewrite
+     * somehow; but that would need some sync traffic to sync the state
+     */
+    if (ntohs(ppkt->ip->ip_off) & IP_DF) {
+        spkt->ip->ip_id = ppkt->ip->ip_id;
+        /* and the sum will be different if the IDs were different */
+        spkt->ip->ip_sum = ppkt->ip->ip_sum;
+    }
+
+    res = memcmp(ppkt->data + ETH_HLEN, spkt->data + ETH_HLEN,
+                (spkt->size - ETH_HLEN));
+
+    if (res != 0 && trace_event_get_state(TRACE_COLO_COMPARE_MISCOMPARE)) {
+        sdebug = strdup(inet_ntoa(ppkt->ip->ip_src));
+        ddebug = strdup(inet_ntoa(ppkt->ip->ip_dst));
+        fprintf(stderr, "%s: src/dst: %s/%s p: seq/ack=%u/%u"
+        " s: seq/ack=%u/%u res=%d flags=%x/%x\n", __func__,
+                   sdebug, ddebug,
+                   ntohl(ptcp->th_seq), ntohl(ptcp->th_ack),
+                   ntohl(stcp->th_seq), ntohl(stcp->th_ack),
+                   res, ptcp->th_flags, stcp->th_flags);
+
+        trace_colo_compare_tcp_miscompare("Primary len", ppkt->size);
+        qemu_hexdump((char *)ppkt->data, stderr, "colo-compare", ppkt->size);
+        trace_colo_compare_tcp_miscompare("Secondary len", spkt->size);
+        qemu_hexdump((char *)spkt->data, stderr, "colo-compare", spkt->size);
+
+        g_free(sdebug);
+        g_free(ddebug);
+    }
+
+    return res;
+}
+
+/*
+ * called from the compare thread on the primary
+ * for compare udp packet
+ */
+static int colo_packet_compare_udp(Packet *spkt, Packet *ppkt)
+{
+    int ret;
+
+    trace_colo_compare_main("compare udp");
+    ret = colo_packet_compare(ppkt, spkt);
+
+    if (ret) {
+        trace_colo_compare_udp_miscompare("primary pkt size", ppkt->size);
+        qemu_hexdump((char *)ppkt->data, stderr, "colo-compare", ppkt->size);
+        trace_colo_compare_udp_miscompare("Secondary pkt size", spkt->size);
+        qemu_hexdump((char *)spkt->data, stderr, "colo-compare", spkt->size);
+    }
+
+    return ret;
+}
+
+/*
+ * called from the compare thread on the primary
+ * for compare icmp packet
+ */
+static int colo_packet_compare_icmp(Packet *spkt, Packet *ppkt)
 {
-    trace_colo_compare_main("compare all");
+    int network_length;
+    struct icmp *icmp_ppkt, *icmp_spkt;
+
+    trace_colo_compare_main("compare icmp");
+    network_length = ppkt->ip->ip_hl * 4;
+    if (ppkt->size != spkt->size ||
+        ppkt->size < network_length + ETH_HLEN) {
+        return -1;
+    }
+    icmp_ppkt = (struct icmp *)(ppkt->data + network_length + ETH_HLEN);
+    icmp_spkt = (struct icmp *)(spkt->data + network_length + ETH_HLEN);
+
+    if ((icmp_ppkt->icmp_type == icmp_spkt->icmp_type) &&
+        (icmp_ppkt->icmp_code == icmp_spkt->icmp_code)) {
+        if (icmp_ppkt->icmp_type == ICMP_REDIRECT) {
+            if (icmp_ppkt->icmp_gwaddr.s_addr !=
+                icmp_spkt->icmp_gwaddr.s_addr) {
+                trace_colo_compare_main("icmp_gwaddr.s_addr not same");
+                return -1;
+            }
+        } else if ((icmp_ppkt->icmp_type == ICMP_UNREACH) &&
+                   (icmp_ppkt->icmp_type == ICMP_UNREACH_NEEDFRAG)) {
+            if (icmp_ppkt->icmp_nextmtu != icmp_spkt->icmp_nextmtu) {
+                trace_colo_compare_main("icmp_nextmtu not same");
+                return -1;
+            }
+        }
+
+        if (colo_packet_compare(ppkt, spkt)) {
+            trace_colo_compare_icmp_miscompare("primary pkt size",
+                                               ppkt->size);
+            qemu_hexdump((char *)ppkt->data, stderr, "colo-compare",
+                         ppkt->size);
+            trace_colo_compare_icmp_miscompare("Secondary pkt size",
+                                               spkt->size);
+            qemu_hexdump((char *)spkt->data, stderr, "colo-compare",
+                         spkt->size);
+            return -1;
+        }
+    } else {
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * called from the compare thread on the primary
+ * for compare other packet
+ */
+static int colo_packet_compare_other(Packet *spkt, Packet *ppkt)
+{
+    trace_colo_compare_main("compare other");
+    trace_colo_compare_ip_info(ppkt->size, inet_ntoa(ppkt->ip->ip_src),
+                               inet_ntoa(ppkt->ip->ip_dst), spkt->size,
+                               inet_ntoa(spkt->ip->ip_src),
+                               inet_ntoa(spkt->ip->ip_dst));
     return colo_packet_compare(ppkt, spkt);
 }
 
@@ -263,8 +413,24 @@  static void colo_compare_connection(void *opaque, void *user_data)
     while (!g_queue_is_empty(&conn->primary_list) &&
            !g_queue_is_empty(&conn->secondary_list)) {
         pkt = g_queue_pop_tail(&conn->primary_list);
-        result = g_queue_find_custom(&conn->secondary_list,
-                              pkt, (GCompareFunc)colo_packet_compare_all);
+        switch (conn->ip_proto) {
+        case IPPROTO_TCP:
+            result = g_queue_find_custom(&conn->secondary_list,
+                     pkt, (GCompareFunc)colo_packet_compare_tcp);
+            break;
+        case IPPROTO_UDP:
+            result = g_queue_find_custom(&conn->secondary_list,
+                     pkt, (GCompareFunc)colo_packet_compare_udp);
+            break;
+        case IPPROTO_ICMP:
+            result = g_queue_find_custom(&conn->secondary_list,
+                     pkt, (GCompareFunc)colo_packet_compare_icmp);
+            break;
+        default:
+            result = g_queue_find_custom(&conn->secondary_list,
+                     pkt, (GCompareFunc)colo_packet_compare_other);
+            break;
+        }
 
         if (result) {
             ret = compare_chr_send(s->chr_out, pkt->data, pkt->size);
diff --git a/trace-events b/trace-events
index 1537e91..ab22eb2 100644
--- a/trace-events
+++ b/trace-events
@@ -1919,5 +1919,9 @@  aspeed_vic_write(uint64_t offset, unsigned size, uint32_t data) "To 0x%" PRIx64
 
 # net/colo-compare.c
 colo_compare_main(const char *chr) ": %s"
+colo_compare_tcp_miscompare(const char *sta, int size) ": %s = %d"
+colo_compare_udp_miscompare(const char *sta, int size) ": %s = %d"
+colo_compare_icmp_miscompare(const char *sta, int size) ": %s = %d"
 colo_compare_ip_info(int psize, const char *sta, const char *stb, int ssize, const char *stc, const char *std) "ppkt size = %d, ip_src = %s, ip_dst = %s, spkt size = %d, ip_src = %s, ip_dst = %s"
 colo_old_packet_check_found(int64_t old_time) "%" PRId64
+colo_compare_miscompare(void) ""