Add signed bn2bin and bin2bn functions

This adds the functions BN_signed_bin2bn(), BN_signed_bn2bin(),
BN_signed_lebin2bn(), BN_signed_bn2lebin(), BN_signed_native2bn(),
and BN_signed_bn2native(), all essentially doing the same job as
BN_bin2bn(), BN_bn2binpad(), BN_lebin2bn(), BN_bn2lebinpad(),
BN_native2bn(), and BN_bn2nativepad(), except that the 'signed'
ones operate on signed number bins in 2's complement form.

Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/17139)

crypto/bn/bn_lib.c

@@ -431,13 +431,15 @@ int BN_set_word(BIGNUM *a, BN_ULONG w)
 }
 
 typedef enum {BIG, LITTLE} endianess_t;
+typedef enum {SIGNED, UNSIGNED} signedness_t;
 
 static BIGNUM *bin2bn(const unsigned char *s, int len, BIGNUM *ret,
-                      endianess_t endianess)
+                      endianess_t endianess, signedness_t signedness)
 {
     int inc;
     const unsigned char *s2;
     int inc2;
+    int neg = 0, xor = 0, carry = 0;
     unsigned int i;
     unsigned int n;
     BIGNUM *bn = NULL;
@@ -467,13 +469,30 @@ static BIGNUM *bin2bn(const unsigned char *s, int len, BIGNUM *ret,
         break;
     }
 
+    /* Take note of the signedness of the input bytes*/
+    if (signedness == SIGNED) {
+        neg = !!(*s2 & 0x80);
+        xor = neg ? 0xff : 0x00;
+        carry = neg;
+    }
+
     /*
-     * Skip leading sign extensions (zero for unsigned numbers).
+     * Skip leading sign extensions (the value of |xor|).
      * This is the only spot where |s2| and |inc2| are used.
      */
     for ( ; len > 0 && *s2 == xor; s2 += inc2, len--)
         continue;
 
+    /*
+     * If there was a set of 0xff, we backtrack one byte unless the next
+     * one has a sign bit, as the last 0xff is then part of the actual
+     * number, rather then a mere sign extension.
+     */
+    if (xor == 0xff) {
+        if (len == 0 || !(*s2 & 0x80))
+            len++;
+    }
+    /* If it was all zeros, we're done */
     if (len == 0) {
         ret->top = 0;
         return ret;
@@ -484,14 +503,16 @@ static BIGNUM *bin2bn(const unsigned char *s, int len, BIGNUM *ret,
         return NULL;
     }
     ret->top = n;
-    ret->neg = 0;
+    ret->neg = neg;
     for (i = 0; n-- > 0; i++) {
         BN_ULONG l = 0;        /* Accumulator */
         unsigned int m = 0;    /* Offset in a bignum chunk, in bits */
 
         for (; len > 0 && m < BN_BYTES * 8; len--, s += inc, m += 8) {
-            BN_ULONG byte = *s;
+            BN_ULONG byte_xored = *s ^ xor;
+            BN_ULONG byte = (byte_xored + carry) & 0xff;
 
+            carry = byte_xored > byte; /* Implicit 1 or 0 */
             l |= (byte << m);
         }
         ret->d[i] = l;
@@ -506,33 +527,56 @@ static BIGNUM *bin2bn(const unsigned char *s, int len, BIGNUM *ret,
 
 BIGNUM *BN_bin2bn(const unsigned char *s, int len, BIGNUM *ret)
 {
-    return bin2bn(s, len, ret, BIG);
+    return bin2bn(s, len, ret, BIG, UNSIGNED);
 }
 
-/* ignore negative */
-static
-int bn2binpad(const BIGNUM *a, unsigned char *to, int tolen,
-              endianess_t endianess)
+BIGNUM *BN_signed_bin2bn(const unsigned char *s, int len, BIGNUM *ret)
+{
+    return bin2bn(s, len, ret, BIG, SIGNED);
+}
+
+static int bn2binpad(const BIGNUM *a, unsigned char *to, int tolen,
+                     endianess_t endianess, signedness_t signedness)
 {
     int inc;
-    int n;
+    int n, n8;
+    int xor = 0, carry = 0, ext = 0;
     size_t i, lasti, j, atop, mask;
     BN_ULONG l;
 
     /*
-     * In case |a| is fixed-top, BN_num_bytes can return bogus length,
+     * In case |a| is fixed-top, BN_num_bits can return bogus length,
      * but it's assumed that fixed-top inputs ought to be "nominated"
      * even for padded output, so it works out...
      */
-    n = BN_num_bytes(a);
+    n8 = BN_num_bits(a);
+    n = (n8 + 7) / 8;           /* This is what BN_num_bytes() does */
+
+    /* Take note of the signedness of the bignum */
+    if (signedness == SIGNED) {
+        xor = a->neg ? 0xff : 0x00;
+        carry = a->neg;
+
+        /*
+         * if |n * 8 == n|, then the MSbit is set, otherwise unset.
+         * We must compensate with one extra byte if that doesn't
+         * correspond to the signedness of the bignum with regards
+         * to 2's complement.
+         */
+        ext = (n * 8 == n8)
+            ? !a->neg            /* MSbit set on nonnegative bignum */
+            : a->neg;            /* MSbit unset on negative bignum */
+    }
+
     if (tolen == -1) {
-        tolen = n;
-    } else if (tolen < n) {     /* uncommon/unlike case */
+        tolen = n + ext;
+    } else if (tolen < n + ext) { /* uncommon/unlike case */
         BIGNUM temp = *a;
 
         bn_correct_top(&temp);
-        n = BN_num_bytes(&temp);
-        if (tolen < n)
+        n8 = BN_num_bits(&temp);
+        n = (n8 + 7) / 8;       /* This is what BN_num_bytes() does */
+        if (tolen < n + ext)
             return -1;
     }
 
@@ -562,9 +606,14 @@ int bn2binpad(const BIGNUM *a, unsigned char *to, int tolen,
     lasti = atop - 1;
     atop = a->top * BN_BYTES;
     for (i = 0, j = 0; j < (size_t)tolen; j++) {
+        unsigned char byte, byte_xored;
+
         l = a->d[i / BN_BYTES];
         mask = 0 - ((j - atop) >> (8 * sizeof(i) - 1));
-        *to = (unsigned char)(l >> (8 * (i % BN_BYTES)) & mask);
+        byte = (unsigned char)(l >> (8 * (i % BN_BYTES)) & mask);
+        byte_xored = byte ^ xor;
+        *to = (unsigned char)(byte_xored + carry);
+        carry = byte_xored > *to; /* Implicit 1 or 0 */
         to += inc;
         i += (i - lasti) >> (8 * sizeof(i) - 1); /* stay on last limb */
     }
@@ -576,24 +625,43 @@ int BN_bn2binpad(const BIGNUM *a, unsigned char *to, int tolen)
 {
     if (tolen < 0)
         return -1;
-    return bn2binpad(a, to, tolen, BIG);
+    return bn2binpad(a, to, tolen, BIG, UNSIGNED);
+}
+
+int BN_signed_bn2bin(const BIGNUM *a, unsigned char *to, int tolen)
+{
+    if (tolen < 0)
+        return -1;
+    return bn2binpad(a, to, tolen, BIG, SIGNED);
 }
 
 int BN_bn2bin(const BIGNUM *a, unsigned char *to)
 {
-    return bn2binpad(a, to, -1, BIG);
+    return bn2binpad(a, to, -1, BIG, UNSIGNED);
 }
 
 BIGNUM *BN_lebin2bn(const unsigned char *s, int len, BIGNUM *ret)
 {
-    return bin2bn(s, len, ret, LITTLE);
+    return bin2bn(s, len, ret, LITTLE, UNSIGNED);
+}
+
+BIGNUM *BN_signed_lebin2bn(const unsigned char *s, int len, BIGNUM *ret)
+{
+    return bin2bn(s, len, ret, LITTLE, SIGNED);
 }
 
 int BN_bn2lebinpad(const BIGNUM *a, unsigned char *to, int tolen)
 {
     if (tolen < 0)
         return -1;
-    return bn2binpad(a, to, tolen, LITTLE);
+    return bn2binpad(a, to, tolen, LITTLE, UNSIGNED);
+}
+
+int BN_signed_bn2lebin(const BIGNUM *a, unsigned char *to, int tolen)
+{
+    if (tolen < 0)
+        return -1;
+    return bn2binpad(a, to, tolen, LITTLE, SIGNED);
 }
 
 BIGNUM *BN_native2bn(const unsigned char *s, int len, BIGNUM *ret)
@@ -605,6 +673,15 @@ BIGNUM *BN_native2bn(const unsigned char *s, int len, BIGNUM *ret)
     return BN_bin2bn(s, len, ret);
 }
 
+BIGNUM *BN_signed_native2bn(const unsigned char *s, int len, BIGNUM *ret)
+{
+    DECLARE_IS_ENDIAN;
+
+    if (IS_LITTLE_ENDIAN)
+        return BN_signed_lebin2bn(s, len, ret);
+    return BN_signed_bin2bn(s, len, ret);
+}
+
 int BN_bn2nativepad(const BIGNUM *a, unsigned char *to, int tolen)
 {
     DECLARE_IS_ENDIAN;
@@ -614,6 +691,15 @@ int BN_bn2nativepad(const BIGNUM *a, unsigned char *to, int tolen)
     return BN_bn2binpad(a, to, tolen);
 }
 
+int BN_signed_bn2native(const BIGNUM *a, unsigned char *to, int tolen)
+{
+    DECLARE_IS_ENDIAN;
+
+    if (IS_LITTLE_ENDIAN)
+        return BN_signed_bn2lebin(a, to, tolen);
+    return BN_signed_bn2bin(a, to, tolen);
+}
+
 int BN_ucmp(const BIGNUM *a, const BIGNUM *b)
 {
     int i;

doc/man3/BN_bn2bin.pod

@@ -2,9 +2,10 @@
 
 =head1 NAME
 
-BN_bn2binpad,
-BN_bn2bin, BN_bin2bn, BN_bn2lebinpad, BN_lebin2bn,
-BN_bn2nativepad, BN_native2bn, BN_bn2hex, BN_bn2dec, BN_hex2bn, BN_dec2bn,
+BN_bn2binpad, BN_signed_bn2bin, BN_bn2bin, BN_bin2bn, BN_signed_bin2bn,
+BN_bn2lebinpad, BN_signed_bn2lebin, BN_lebin2bn, BN_signed_lebin2bn,
+BN_bn2nativepad, BN_signed_bn2native, BN_native2bn, BN_signed_native2bn,
+BN_bn2hex, BN_bn2dec, BN_hex2bn, BN_dec2bn,
 BN_print, BN_print_fp, BN_bn2mpi, BN_mpi2bn - format conversions
 
 =head1 SYNOPSIS
@@ -13,13 +14,19 @@ BN_print, BN_print_fp, BN_bn2mpi, BN_mpi2bn - format conversions
 
  int BN_bn2bin(const BIGNUM *a, unsigned char *to);
  int BN_bn2binpad(const BIGNUM *a, unsigned char *to, int tolen);
+ int BN_signed_bn2bin(const BIGNUM *a, unsigned char *to, int tolen);
  BIGNUM *BN_bin2bn(const unsigned char *s, int len, BIGNUM *ret);
+ BIGNUM *BN_signed_bin2bn(const unsigned char *s, int len, BIGNUM *ret);
 
  int BN_bn2lebinpad(const BIGNUM *a, unsigned char *to, int tolen);
+ int BN_signed_bn2lebin(const BIGNUM *a, unsigned char *to, int tolen);
  BIGNUM *BN_lebin2bn(const unsigned char *s, int len, BIGNUM *ret);
+ BIGNUM *BN_signed_lebin2bn(const unsigned char *s, int len, BIGNUM *ret);
 
  int BN_bn2nativepad(const BIGNUM *a, unsigned char *to, int tolen);
+ int BN_signed_bn2native(const BIGNUM *a, unsigned char *to, int tolen);
  BIGNUM *BN_native2bn(const unsigned char *s, int len, BIGNUM *ret);
+ BIGNUM *BN_signed_native2bn(const unsigned char *s, int len, BIGNUM *ret);
 
  char *BN_bn2hex(const BIGNUM *a);
  char *BN_bn2dec(const BIGNUM *a);
@@ -43,17 +50,29 @@ and stores it at B<to>. B<tolen> indicates the length of the output buffer
 B<to>. The result is padded with zeros if necessary. If B<tolen> is less than
 BN_num_bytes(B<a>) an error is returned.
 
+BN_signed_bn2bin() converts the value of B<a> into big-endian signed 2's
+complements form and stores it at B<to>. B<tolen> indicates the length of
+the output buffer B<to>. The result is signed extended (padded with 0x00
+for positive numbers or with 0xff for negative numbers) if necessary.
+If B<tolen> is smaller than the necessary size (which may be
+C<<BN_num_bytes(B<a>) + 1>>), an error is returned.
+
 BN_bin2bn() converts the positive integer in big-endian form of length
 B<len> at B<s> into a B<BIGNUM> and places it in B<ret>. If B<ret> is
 NULL, a new B<BIGNUM> is created.
 
-BN_bn2lebinpad() and BN_lebin2bn() are identical to BN_bn2binpad() and
-BN_bin2bn() except the buffer is in little-endian format.
+BN_signed_bin2bn() converts the integer in big-endian signed 2's complement
+form of length B<len> at B<s> into a B<BIGNUM> and places it in B<ret>. If
+B<ret> is NULL, a new B<BIGNUM> is created.
+
+BN_bn2lebinpad(), BN_signed_bn2lebin() and BN_lebin2bn() are identical to
+BN_bn2binpad(), BN_signed_bn2bin() and BN_bin2bn() except the buffer is in
+little-endian format.
 
-BN_bn2nativepad() and BN_native2bn() are identical to BN_bn2binpad() and
-BN_bin2bn() except the buffer is in native format, i.e. most significant
-byte first on big-endian platforms, and least significant byte first on
-little-endian platforms.
+BN_bn2nativepad(), BN_signed_bn2native() and BN_native2bn() are identical
+to BN_bn2binpad(), BN_signed_bn2bin() and BN_bin2bn() except the buffer is
+in native format, i.e. most significant byte first on big-endian platforms,
+and least significant byte first on little-endian platforms.
 
 BN_bn2hex() and BN_bn2dec() return printable strings containing the
 hexadecimal and decimal encoding of B<a> respectively. For negative
@@ -91,8 +110,9 @@ if B<ret> is NULL.
 BN_bn2bin() returns the length of the big-endian number placed at B<to>.
 BN_bin2bn() returns the B<BIGNUM>, NULL on error.
 
-BN_bn2binpad(), BN_bn2lebinpad(), and BN_bn2nativepad() return the number of bytes written or -1 if the supplied
-buffer is too small.
+BN_bn2binpad(), BN_signed_bn2bin(), BN_bn2lebinpad(), BN_signed_bn2lebin(),
+BN_bn2nativepad(), and_signed BN_bn2native() return the number of bytes
+written or -1 if the supplied buffer is too small.
 
 BN_bn2hex() and BN_bn2dec() return a NUL-terminated string, or NULL
 on error. BN_hex2bn() and BN_dec2bn() return the number of characters

include/openssl/bn.h

@@ -241,12 +241,18 @@ void BN_clear_free(BIGNUM *a);
 BIGNUM *BN_copy(BIGNUM *a, const BIGNUM *b);
 void BN_swap(BIGNUM *a, BIGNUM *b);
 BIGNUM *BN_bin2bn(const unsigned char *s, int len, BIGNUM *ret);
+BIGNUM *BN_signed_bin2bn(const unsigned char *s, int len, BIGNUM *ret);
 int BN_bn2bin(const BIGNUM *a, unsigned char *to);
 int BN_bn2binpad(const BIGNUM *a, unsigned char *to, int tolen);
+int BN_signed_bn2bin(const BIGNUM *a, unsigned char *to, int tolen);
 BIGNUM *BN_lebin2bn(const unsigned char *s, int len, BIGNUM *ret);
+BIGNUM *BN_signed_lebin2bn(const unsigned char *s, int len, BIGNUM *ret);
 int BN_bn2lebinpad(const BIGNUM *a, unsigned char *to, int tolen);
+int BN_signed_bn2lebin(const BIGNUM *a, unsigned char *to, int tolen);
 BIGNUM *BN_native2bn(const unsigned char *s, int len, BIGNUM *ret);
+BIGNUM *BN_signed_native2bn(const unsigned char *s, int len, BIGNUM *ret);
 int BN_bn2nativepad(const BIGNUM *a, unsigned char *to, int tolen);
+int BN_signed_bn2native(const BIGNUM *a, unsigned char *to, int tolen);
 BIGNUM *BN_mpi2bn(const unsigned char *s, int len, BIGNUM *ret);
 int BN_bn2mpi(const BIGNUM *a, unsigned char *to);
 int BN_sub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b);

util/libcrypto.num

@@ -5428,3 +5428,9 @@ EVP_PKEY_CTX_get0_provider              5555	3_0_0	EXIST::FUNCTION:
 OSSL_STACK_OF_X509_free                 ?	3_1_0	EXIST::FUNCTION:
 EVP_MD_CTX_dup                          ?	3_1_0	EXIST::FUNCTION:
 EVP_CIPHER_CTX_dup                      ?	3_1_0	EXIST::FUNCTION:
+BN_signed_bin2bn                        ?	3_1_0	EXIST::FUNCTION:
+BN_signed_bn2bin                        ?	3_1_0	EXIST::FUNCTION:
+BN_signed_lebin2bn                      ?	3_1_0	EXIST::FUNCTION:
+BN_signed_bn2lebin                      ?	3_1_0	EXIST::FUNCTION:
+BN_signed_native2bn                     ?	3_1_0	EXIST::FUNCTION:
+BN_signed_bn2native                     ?	3_1_0	EXIST::FUNCTION: