unpack: Fix negative signed integer unpacking calculation

Currently the calculation is just wrong since it effectively relies on
the widths of the signed and unsigned integer types and on
wrapping-on-assignment behavior. I have no idea why I wrote it like
this, it seems obviously wrong in retrospect.

The new behavior ensure that this succeeds in situations where type
widths are not correct.

The new code sets vals to the minimum possible two's complement signed
value (I just realised this code now assumes two's complement) and then
offsets it with the value part of the signed integer.

1 files changed, 13 insertions, 3 deletions

diff --git a/unpack.c b/unpack.c
index 17d9c45..011aebf 100644
--- a/unpack.c
+++ b/unpack.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020 Tomasz Kramkowski <tk@the-tk.com>
+ * Copyright (C) 2020-2021 Tomasz Kramkowski <tk@the-tk.com>
  * SPDX-License-Identifier: MIT
  */
 #include <ctype.h>
@@ -60,6 +60,16 @@ static uintmax_t read_val(const unsigned char *buf, size_t size, enum endian e)
 	return val;
 }
 
+static intmax_t minval(size_t s)
+{
+	switch (s) {
+	case 1: return INTMAX_C(-128);
+	case 2: return INTMAX_C(-32768);
+	case 4: return INTMAX_C(-2147483648);
+	default: return -INTMAX_C(9223372036854775807) - 1;
+	}
+}
+
 enum pack_status unpack(const void *buf_, size_t size, const char *fmt, ...)
 {
 	enum endian endianness = BIG;
@@ -151,8 +161,8 @@ enum pack_status unpack(const void *buf_, size_t size, const char *fmt, ...)
 				if (!(val.u & (UINTMAX_C(1) << (s * 8 - 1)))) {
 					vals = val.u;
 				} else {
-					uintmax_t offt = BITMASK(s * 8);
-					vals = val.u - offt - 1;
+					vals = minval(s);
+					vals += val.u ^ (UINTMAX_C(1) << (s * 8 - 1));
 				}
 				val.s = vals;
 				tr_debug("val.s: %" PRIdMAX, val.s);