From 968fc24d843c9e9b24231ca1960b47ef2fc724ea Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Fri, 28 Mar 2014 15:12:57 +0000 Subject: [PATCH] xbzrle.c: Avoid undefined behaviour with signed arithmetic Use unsigned types for doing bitwise arithmetic in the xzbrle calculations, to avoid undefined behaviour: xbzrle.c:99:49: runtime error: left shift of 72340172838076673 by 7 places cannot be represented in type 'long' Signed-off-by: Peter Maydell Signed-off-by: Michael Tokarev --- xbzrle.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/xbzrle.c b/xbzrle.c index fbcb35d0e3..8e220bf25b 100644 --- a/xbzrle.c +++ b/xbzrle.c @@ -28,7 +28,7 @@ int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen, { uint32_t zrun_len = 0, nzrun_len = 0; int d = 0, i = 0; - long res, xor; + long res; uint8_t *nzrun_start = NULL; g_assert(!(((uintptr_t)old_buf | (uintptr_t)new_buf | slen) % @@ -93,9 +93,11 @@ int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen, /* word at a time for speed, use of 32-bit long okay */ if (!res) { /* truncation to 32-bit long okay */ - long mask = (long)0x0101010101010101ULL; + unsigned long mask = (unsigned long)0x0101010101010101ULL; while (i < slen) { - xor = *(long *)(old_buf + i) ^ *(long *)(new_buf + i); + unsigned long xor; + xor = *(unsigned long *)(old_buf + i) + ^ *(unsigned long *)(new_buf + i); if ((xor - mask) & ~xor & (mask << 7)) { /* found the end of an nzrun within the current long */ while (old_buf[i] != new_buf[i]) {