git: 2300a22c9712 - main - libc: iconv: add mb_cur_min for encoder traits
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Thu, 11 Aug 2022 16:43:48 UTC
The branch main has been updated by kevans:
URL: https://cgit.FreeBSD.org/src/commit/?id=2300a22c971240e813e73515acfa3970e0871b20
commit 2300a22c971240e813e73515acfa3970e0871b20
Author: Kyle Evans <kevans@FreeBSD.org>
AuthorDate: 2022-02-22 07:12:27 +0000
Commit: Kyle Evans <kevans@FreeBSD.org>
CommitDate: 2022-08-11 16:42:17 +0000
libc: iconv: add mb_cur_min for encoder traits
A future commit will actually implement //IGNORE so that applications
using base iconv can, e.g., sanitize UTF-8 strings. To do this, the
iconv_std module needs to be able to determine the minimum width for any
given encoding so that it can skip that many bytes in the input buffer.
This is mainly an issue for UTF-16 and UTF-32.
This commit bumps shlib versions to 5 for libiconv modules to reflect
the ABI change. It also fixes OptionalObsoleteFiles to remove the
libiconv modules if WITHOUT_ICONV is in use.
re: _ENCODING_MB_CUR_MIN, note that this file (citrus_stdenc_template.h)
is included at the bottom of an encoding *implementation*, so the
implementation is free to #define it prior. UTF1632 is a good example,
as it redefines the minimum to be a property on the encodinginfo, and
the minimum is set to 2 or 4 bytes for UTF-16 and UTF-32 respectively.
Sponsored by: Klara, Inc.
Differential Revision: https://reviews.freebsd.org/D34344
---
ObsoleteFiles.inc | 25 ++++++++++++++
lib/libc/iconv/citrus_module.c | 2 +-
lib/libc/iconv/citrus_namespace.h | 1 +
lib/libc/iconv/citrus_stdenc.h | 14 ++++++++
lib/libc/iconv/citrus_stdenc_local.h | 2 ++
lib/libc/iconv/citrus_stdenc_template.h | 6 ++++
lib/libiconv_modules/Makefile.inc | 2 +-
lib/libiconv_modules/UTF1632/citrus_utf1632.c | 3 ++
tools/build/mk/OptionalObsoleteFiles.inc | 47 +++++++++++++++++++++++++++
9 files changed, 100 insertions(+), 2 deletions(-)
diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc
index ab55b981106c..37ec8ddf8546 100644
--- a/ObsoleteFiles.inc
+++ b/ObsoleteFiles.inc
@@ -52,6 +52,31 @@
# xargs -n1 | sort | uniq -d;
# done
+# 20220811: new iconv encoder trait added
+OLD_LIBS+=usr/lib/i18n/libBIG5.so.4
+OLD_LIBS+=usr/lib/i18n/libDECHanyu.so.4
+OLD_LIBS+=usr/lib/i18n/libEUC.so.4
+OLD_LIBS+=usr/lib/i18n/libEUCTW.so.4
+OLD_LIBS+=usr/lib/i18n/libGBK2K.so.4
+OLD_LIBS+=usr/lib/i18n/libHZ.so.4
+OLD_LIBS+=usr/lib/i18n/libISO2022.so.4
+OLD_LIBS+=usr/lib/i18n/libJOHAB.so.4
+OLD_LIBS+=usr/lib/i18n/libMSKanji.so.4
+OLD_LIBS+=usr/lib/i18n/libUES.so.4
+OLD_LIBS+=usr/lib/i18n/libUTF1632.so.4
+OLD_LIBS+=usr/lib/i18n/libUTF7.so.4
+OLD_LIBS+=usr/lib/i18n/libUTF8.so.4
+OLD_LIBS+=usr/lib/i18n/libVIQR.so.4
+OLD_LIBS+=usr/lib/i18n/libZW.so.4
+OLD_LIBS+=usr/lib/i18n/libiconv_none.so.4
+OLD_LIBS+=usr/lib/i18n/libiconv_std.so.4
+OLD_LIBS+=usr/lib/i18n/libmapper_646.so.4
+OLD_LIBS+=usr/lib/i18n/libmapper_none.so.4
+OLD_LIBS+=usr/lib/i18n/libmapper_parallel.so.4
+OLD_LIBS+=usr/lib/i18n/libmapper_serial.so.4
+OLD_LIBS+=usr/lib/i18n/libmapper_std.so.4
+OLD_LIBS+=usr/lib/i18n/libmapper_zone.so.4
+
# 202208XX: raw socket layer removed
OLD_FILES+=usr/include/net/raw_cb.h
diff --git a/lib/libc/iconv/citrus_module.c b/lib/libc/iconv/citrus_module.c
index 64ef0271ea18..bd173b41bb04 100644
--- a/lib/libc/iconv/citrus_module.c
+++ b/lib/libc/iconv/citrus_module.c
@@ -106,7 +106,7 @@
#include <string.h>
#include <unistd.h>
-#define I18NMODULE_MAJOR 4
+#define I18NMODULE_MAJOR 5
#include "citrus_namespace.h"
#include "citrus_bcs.h"
diff --git a/lib/libc/iconv/citrus_namespace.h b/lib/libc/iconv/citrus_namespace.h
index 4244c70d701e..ef2c01b97f06 100644
--- a/lib/libc/iconv/citrus_namespace.h
+++ b/lib/libc/iconv/citrus_namespace.h
@@ -223,6 +223,7 @@
#define _stdenc_wctomb _citrus_stdenc_wctomb
#define _stdenc_put_state_reset _citrus_stdenc_put_state_reset
#define _stdenc_get_state_size _citrus_stdenc_get_state_size
+#define _stdenc_get_mb_cur_min _citrus_stdenc_get_mb_cur_min
#define _stdenc_get_mb_cur_max _citrus_stdenc_get_mb_cur_max
#define _stdenc_get_state_desc _citrus_stdenc_get_state_desc
#define _STDENC_SDID_GENERIC _CITRUS_STDENC_SDID_GENERIC
diff --git a/lib/libc/iconv/citrus_stdenc.h b/lib/libc/iconv/citrus_stdenc.h
index a643b77a27aa..da48093c15da 100644
--- a/lib/libc/iconv/citrus_stdenc.h
+++ b/lib/libc/iconv/citrus_stdenc.h
@@ -115,6 +115,20 @@ _citrus_stdenc_get_state_size(struct _citrus_stdenc *ce)
return (ce->ce_traits->et_state_size);
}
+static __inline size_t
+_citrus_stdenc_get_mb_cur_min(struct _citrus_stdenc *ce)
+{
+
+ return (ce->ce_traits->et_mb_cur_min);
+}
+
+static __inline size_t
+_citrus_stdenc_get_mb_cur_max(struct _citrus_stdenc *ce)
+{
+
+ return (ce->ce_traits->et_mb_cur_max);
+}
+
static __inline int
_citrus_stdenc_get_state_desc(struct _citrus_stdenc * __restrict ce,
void * __restrict ps, int id,
diff --git a/lib/libc/iconv/citrus_stdenc_local.h b/lib/libc/iconv/citrus_stdenc_local.h
index 2af3b8a992b7..9e4b827e830b 100644
--- a/lib/libc/iconv/citrus_stdenc_local.h
+++ b/lib/libc/iconv/citrus_stdenc_local.h
@@ -149,6 +149,8 @@ struct _citrus_stdenc_traits {
/* version 0x00000001 */
size_t et_state_size;
size_t et_mb_cur_max;
+ /* version 0x00000005 */
+ size_t et_mb_cur_min;
};
struct _citrus_stdenc {
diff --git a/lib/libc/iconv/citrus_stdenc_template.h b/lib/libc/iconv/citrus_stdenc_template.h
index 06fca844392f..ad19787bc1bc 100644
--- a/lib/libc/iconv/citrus_stdenc_template.h
+++ b/lib/libc/iconv/citrus_stdenc_template.h
@@ -49,6 +49,11 @@
#define _CE_TO_EI(_ce_) (_TO_EI((_ce_)->ce_closure))
#define _TO_STATE(_ps_) ((_ENCODING_STATE*)(_ps_))
+#ifndef _ENCODING_MB_CUR_MIN
+/* Assume one byte minimum unless otherwise specified. */
+#define _ENCODING_MB_CUR_MIN(_ei_) 1
+#endif
+
/* ----------------------------------------------------------------------
* templates for public functions
*/
@@ -87,6 +92,7 @@ _FUNCNAME(stdenc_init)(struct _citrus_stdenc * __restrict ce,
ce->ce_closure = ei;
et->et_state_size = sizeof(_ENCODING_STATE);
et->et_mb_cur_max = _ENCODING_MB_CUR_MAX(_CE_TO_EI(ce));
+ et->et_mb_cur_min = _ENCODING_MB_CUR_MIN(_CE_TO_EI(ce));
return (0);
}
diff --git a/lib/libiconv_modules/Makefile.inc b/lib/libiconv_modules/Makefile.inc
index 556997404f99..364e78c9bf85 100644
--- a/lib/libiconv_modules/Makefile.inc
+++ b/lib/libiconv_modules/Makefile.inc
@@ -2,7 +2,7 @@
.PATH: ${SRCTOP}/lib/libc/iconv
-SHLIB_MAJOR= 4
+SHLIB_MAJOR= 5
CFLAGS+= -I${SRCTOP}/lib/libc/iconv
CFLAGS+= -Dbool=_Bool
diff --git a/lib/libiconv_modules/UTF1632/citrus_utf1632.c b/lib/libiconv_modules/UTF1632/citrus_utf1632.c
index b9625165051c..27a810597c7f 100644
--- a/lib/libiconv_modules/UTF1632/citrus_utf1632.c
+++ b/lib/libiconv_modules/UTF1632/citrus_utf1632.c
@@ -77,6 +77,7 @@ typedef struct {
typedef struct {
int preffered_endian;
unsigned int cur_max;
+ unsigned int cur_min;
uint32_t mode;
} _UTF1632EncodingInfo;
@@ -84,6 +85,7 @@ typedef struct {
#define _ENCODING_INFO _UTF1632EncodingInfo
#define _ENCODING_STATE _UTF1632State
#define _ENCODING_MB_CUR_MAX(_ei_) ((_ei_)->cur_max)
+#define _ENCODING_MB_CUR_MIN(_ei_) ((_ei_)->cur_min)
#define _ENCODING_IS_STATE_DEPENDENT 0
#define _STATE_NEEDS_EXPLICIT_INIT(_ps_) 0
@@ -390,6 +392,7 @@ _citrus_UTF1632_encoding_module_init(_UTF1632EncodingInfo * __restrict ei,
parse_variable(ei, var, lenvar);
+ ei->cur_min = ((ei->mode&_MODE_UTF32) == 0) ? 2 : 4;
ei->cur_max = ((ei->mode&_MODE_UTF32) == 0) ? 6 : 8;
/* 6: endian + surrogate */
/* 8: endian + normal */
diff --git a/tools/build/mk/OptionalObsoleteFiles.inc b/tools/build/mk/OptionalObsoleteFiles.inc
index 1b26bfa7473d..9a5726ab0f4d 100644
--- a/tools/build/mk/OptionalObsoleteFiles.inc
+++ b/tools/build/mk/OptionalObsoleteFiles.inc
@@ -2455,6 +2455,52 @@ OLD_FILES+=usr/bin/mkcsmapper
OLD_FILES+=usr/bin/mkesdb
OLD_FILES+=usr/include/_libiconv_compat.h
OLD_FILES+=usr/include/iconv.h
+OLD_LIBS+=usr/lib/i18n/libBIG5.so.5
+OLD_FILES+=usr/lib/i18n/libBIG5.so
+OLD_LIBS+=usr/lib/i18n/libDECHanyu.so.5
+OLD_FILES+=usr/lib/i18n/libDECHanyu.so
+OLD_LIBS+=usr/lib/i18n/libEUC.so.5
+OLD_FILES+=usr/lib/i18n/libEUC.so
+OLD_LIBS+=usr/lib/i18n/libEUCTW.so.5
+OLD_FILES+=usr/lib/i18n/libEUCTW.so
+OLD_LIBS+=usr/lib/i18n/libGBK2K.so.5
+OLD_FILES+=usr/lib/i18n/libGBK2K.so
+OLD_LIBS+=usr/lib/i18n/libHZ.so.5
+OLD_FILES+=usr/lib/i18n/libHZ.so
+OLD_LIBS+=usr/lib/i18n/libISO2022.so.5
+OLD_FILES+=usr/lib/i18n/libISO2022.so
+OLD_LIBS+=usr/lib/i18n/libJOHAB.so.5
+OLD_FILES+=usr/lib/i18n/libJOHAB.so
+OLD_LIBS+=usr/lib/i18n/libMSKanji.so.5
+OLD_FILES+=usr/lib/i18n/libMSKanji.so
+OLD_LIBS+=usr/lib/i18n/libUES.so.5
+OLD_FILES+=usr/lib/i18n/libUES.so
+OLD_LIBS+=usr/lib/i18n/libUTF1632.so.5
+OLD_FILES+=usr/lib/i18n/libUTF1632.so
+OLD_LIBS+=usr/lib/i18n/libUTF7.so.5
+OLD_FILES+=usr/lib/i18n/libUTF7.so
+OLD_LIBS+=usr/lib/i18n/libUTF8.so.5
+OLD_FILES+=usr/lib/i18n/libUTF8.so
+OLD_LIBS+=usr/lib/i18n/libVIQR.so.5
+OLD_FILES+=usr/lib/i18n/libVIQR.so
+OLD_LIBS+=usr/lib/i18n/libZW.so.5
+OLD_FILES+=usr/lib/i18n/libZW.so
+OLD_LIBS+=usr/lib/i18n/libiconv_none.so.5
+OLD_FILES+=usr/lib/i18n/libiconv_none.so
+OLD_LIBS+=usr/lib/i18n/libiconv_std.so.5
+OLD_FILES+=usr/lib/i18n/libiconv_std.so
+OLD_LIBS+=usr/lib/i18n/libmapper_646.so.5
+OLD_FILES+=usr/lib/i18n/libmapper_646.so
+OLD_LIBS+=usr/lib/i18n/libmapper_none.so.5
+OLD_FILES+=usr/lib/i18n/libmapper_none.so
+OLD_LIBS+=usr/lib/i18n/libmapper_parallel.so.5
+OLD_FILES+=usr/lib/i18n/libmapper_parallel.so
+OLD_LIBS+=usr/lib/i18n/libmapper_serial.so.5
+OLD_FILES+=usr/lib/i18n/libmapper_serial.so
+OLD_LIBS+=usr/lib/i18n/libmapper_std.so.5
+OLD_FILES+=usr/lib/i18n/libmapper_std.so
+OLD_LIBS+=usr/lib/i18n/libmapper_zone.so.5
+OLD_FILES+=usr/lib/i18n/libmapper_zone.so
OLD_FILES+=usr/share/man/man1/iconv.1.gz
OLD_FILES+=usr/share/man/man1/mkcsmapper.1.gz
OLD_FILES+=usr/share/man/man1/mkesdb.1.gz
@@ -2468,6 +2514,7 @@ OLD_FILES+=usr/share/man/man3/iconv_open.3.gz
OLD_FILES+=usr/share/man/man3/iconv_open_into.3.gz
OLD_FILES+=usr/share/man/man3/iconvctl.3.gz
OLD_FILES+=usr/share/man/man3/iconvlist.3.gz
+OLD_DIRS+=usr/lib/i18n
OLD_DIRS+=usr/share/i18n
OLD_DIRS+=usr/share/i18n/esdb
OLD_DIRS+=usr/share/i18n/esdb/ISO-2022