OpenSSL breaks factor(6)

[Steve Kargl]

On Fri, Dec 27, 2019 at 01:47:17PM -0800, Steve Kargl wrote:
> On Fri, Dec 27, 2019 at 01:25:30PM -0800, Steve Kargl wrote:
> > The use of OpenSSL in factor(6) breaks factor(6) with respect to
> > its documentation.
> > 
> > % man factor
> >   ...
> >   Numbers may be preceded by a single '+'.
> >   ...
> > 
> > % factor +125
> > factor: +125: illegal numeric format.
> > 
> 
> This fixes factor(6) for the above issue.  The issue with
> hexadecimal is not easily fixed.
>
 
This patch now includes a fix for hexadecimal conversion.  It
simple scans the string for a hex digit in [a,...,f] and assumes
that a hexadecimal string has been entered.  A string that includes
character from the decimal digits is assumed to by a decimal
representation.  

Index: factor.c
===================================================================
--- factor.c	(revision 355983)
+++ factor.c	(working copy)
@@ -71,6 +71,7 @@
 #include <errno.h>
 #include <inttypes.h>
 #include <limits.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -104,6 +105,7 @@
 
 #endif
 
+static bool	is_hex(char *str);
 static void	BN_print_dec_fp(FILE *, const BIGNUM *);
 
 static void	pr_fact(BIGNUM *);	/* print factors of a value */
@@ -148,21 +150,25 @@
 			for (p = buf; isblank(*p); ++p);
 			if (*p == '\n' || *p == '\0')
 				continue;
+			if (*p == '+') p++;
 			if (*p == '-')
 				errx(1, "negative numbers aren't permitted.");
-			if (BN_dec2bn(&val, buf) == 0 &&
-			    BN_hex2bn(&val, buf) == 0)
-				errx(1, "%s: illegal numeric format.", buf);
+			ch = is_hex(p) ? BN_hex2bn(&val, p) :
+			    BN_dec2bn(&val, p);
+			if (ch == 0)
+				errx(1, "%s: illegal numeric format.", p);
 			pr_fact(val);
 		}
 	/* Factor the arguments. */
 	else
-		for (; *argv != NULL; ++argv) {
-			if (argv[0][0] == '-')
+		for (p = *argv; p != NULL; p = *++argv) {
+			if (*p == '-')
 				errx(1, "negative numbers aren't permitted.");
-			if (BN_dec2bn(&val, argv[0]) == 0 &&
-			    BN_hex2bn(&val, argv[0]) == 0)
-				errx(1, "%s: illegal numeric format.", argv[0]);
+			if (*p == '+') p++;
+			ch = is_hex(p) ? BN_hex2bn(&val, p) :
+			    BN_dec2bn(&val, p);
+			if (ch == 0)
+				errx(1, "%s: illegal numeric format.", p);
 			pr_fact(val);
 		}
 	exit(0);
@@ -343,10 +349,9 @@
 BN_dec2bn(BIGNUM **a, const char *str)
 {
 	char *p;
-
 	errno = 0;
 	**a = strtoul(str, &p, 10);
-	return (errno == 0 && (*p == '\n' || *p == '\0'));
+	return (errno == 0 ? 1 : 0);	/* OpenSSL returns 0 on error! */
 }
 
 static int
@@ -356,7 +361,7 @@
 
 	errno = 0;
 	**a = strtoul(str, &p, 16);
-	return (errno == 0 && (*p == '\n' || *p == '\0'));
+	return (errno == 0 ? 1 : 0);	/* OpenSSL returns 0 on error! */
 }
 
 static BN_ULONG
@@ -370,3 +375,17 @@
 }
 
 #endif
+
+/* Check if the string contains a hexadecimal digit. */
+static bool
+is_hex(char *str)
+{
+	char c, *p;
+	for (p = str; *p; p++) {
+		c = tolower(*p);
+		if (c == 'a' || c == 'b' || c == 'c' || c == 'd' ||
+		    c == 'e' || c == 'f')
+			return true;
+	}
+	return false;
+}

-- 
Steve