git: d70609a8b3e1 - main - devel/signal-fts5-extension: Add new port

Go to: [ bottom of page ] [ top of archives ] [ this month ]
From: Mikael Urankar <mikael_at_FreeBSD.org>
Date: Tue, 14 Feb 2023 17:32:45 UTC
The branch main has been updated by mikael:

URL: https://cgit.FreeBSD.org/ports/commit/?id=d70609a8b3e183359c8a8660612c01550d9d5ec0

commit d70609a8b3e183359c8a8660612c01550d9d5ec0
Author:     Mikael Urankar <mikael@FreeBSD.org>
AuthorDate: 2023-02-14 17:30:04 +0000
Commit:     Mikael Urankar <mikael@FreeBSD.org>
CommitDate: 2023-02-14 17:32:31 +0000

    devel/signal-fts5-extension: Add new port
    
    Signal-FTS5-Extension is a C ABI library which exposes a FTS5 tokenizer function
    named signal_tokenizer that:
    
      Segments UTF-8 strings into words according to Unicode standard
      Normalizes and removes diacritics from words
      Converts words to lower case
    
    When used as a custom FTS5 tokenizer this enables application to support CJK
    symbols in full-text search.
---
 devel/Makefile                        |   1 +
 devel/signal-fts5-extension/Makefile  |  93 ++++++++++++++++++++++
 devel/signal-fts5-extension/distinfo  | 143 ++++++++++++++++++++++++++++++++++
 devel/signal-fts5-extension/pkg-descr |   9 +++
 4 files changed, 246 insertions(+)

diff --git a/devel/Makefile b/devel/Makefile
index 51839dec375a..090df0bcbfba 100644
--- a/devel/Makefile
+++ b/devel/Makefile
@@ -7359,6 +7359,7 @@
     SUBDIR += shtool
     SUBDIR += shunit2
     SUBDIR += sigar
+    SUBDIR += signal-fts5-extension
     SUBDIR += sigslot
     SUBDIR += silc-toolkit
     SUBDIR += silentbob
diff --git a/devel/signal-fts5-extension/Makefile b/devel/signal-fts5-extension/Makefile
new file mode 100644
index 000000000000..d21a38bdec3e
--- /dev/null
+++ b/devel/signal-fts5-extension/Makefile
@@ -0,0 +1,93 @@
+PORTNAME=	signal-fts5-extension
+DISTVERSIONPREFIX=	v
+DISTVERSION=	0.2.1
+CATEGORIES=	devel
+
+MAINTAINER=	mikael@FreeBSD.org
+COMMENT=	FTS5 extension for signal_tokenizer
+WWW=		https://github.com/signalapp/Signal-FTS5-Extension
+
+LICENSE=	AGPLv3
+LICENSE_FILE=	${WRKSRC}/LICENSE
+
+USES=		cargo
+USE_GITHUB=	yes
+GH_ACCOUNT=	signalapp
+GH_PROJECT=	Signal-FTS5-Extension
+CARGO_CRATES=	atty-0.2.14 \
+		autocfg-1.1.0 \
+		bitflags-1.3.2 \
+		bstr-0.2.17 \
+		bumpalo-3.12.0 \
+		cast-0.3.0 \
+		cfg-if-1.0.0 \
+		clap-2.34.0 \
+		criterion-0.3.6 \
+		criterion-plot-0.4.5 \
+		crossbeam-channel-0.5.6 \
+		crossbeam-deque-0.8.2 \
+		crossbeam-epoch-0.9.13 \
+		crossbeam-utils-0.8.14 \
+		csv-1.1.6 \
+		csv-core-0.1.10 \
+		either-1.8.1 \
+		half-1.8.2 \
+		hermit-abi-0.1.19 \
+		hermit-abi-0.2.6 \
+		itertools-0.10.5 \
+		itoa-0.4.8 \
+		itoa-1.0.5 \
+		js-sys-0.3.60 \
+		lazy_static-1.4.0 \
+		libc-0.2.139 \
+		log-0.4.17 \
+		memchr-2.5.0 \
+		memoffset-0.7.1 \
+		num-traits-0.2.15 \
+		num_cpus-1.15.0 \
+		once_cell-1.17.0 \
+		oorandom-11.1.3 \
+		plotters-0.3.4 \
+		plotters-backend-0.3.4 \
+		plotters-svg-0.3.3 \
+		proc-macro2-1.0.50 \
+		quote-1.0.23 \
+		rayon-1.6.1 \
+		rayon-core-1.10.2 \
+		regex-1.7.1 \
+		regex-automata-0.1.10 \
+		regex-syntax-0.6.28 \
+		ryu-1.0.12 \
+		same-file-1.0.6 \
+		scopeguard-1.1.0 \
+		serde-1.0.152 \
+		serde_cbor-0.11.2 \
+		serde_derive-1.0.152 \
+		serde_json-1.0.91 \
+		syn-1.0.107 \
+		textwrap-0.11.0 \
+		tinytemplate-1.2.1 \
+		tinyvec-1.6.0 \
+		tinyvec_macros-0.1.0 \
+		unicode-ident-1.0.6 \
+		unicode-normalization-0.1.22 \
+		unicode-segmentation-1.10.1 \
+		unicode-width-0.1.10 \
+		walkdir-2.3.2 \
+		wasm-bindgen-0.2.83 \
+		wasm-bindgen-backend-0.2.83 \
+		wasm-bindgen-macro-0.2.83 \
+		wasm-bindgen-macro-support-0.2.83 \
+		wasm-bindgen-shared-0.2.83 \
+		web-sys-0.3.60 \
+		winapi-0.3.9 \
+		winapi-i686-pc-windows-gnu-0.4.0 \
+		winapi-util-0.1.5 \
+		winapi-x86_64-pc-windows-gnu-0.4.0
+
+PLIST_FILES=	lib/libsignal_tokenizer.a
+
+do-install:
+	${INSTALL_DATA} ${CARGO_TARGET_DIR}/${CARGO_BUILD_TARGET}/*/libsignal_tokenizer.a ${STAGEDIR}${LOCALBASE}/lib/
+
+.include <bsd.port.mk>
diff --git a/devel/signal-fts5-extension/distinfo b/devel/signal-fts5-extension/distinfo
new file mode 100644
index 000000000000..63ad6c489411
--- /dev/null
+++ b/devel/signal-fts5-extension/distinfo
@@ -0,0 +1,143 @@
+TIMESTAMP = 1676363331
+SHA256 (rust/crates/atty-0.2.14.crate) = d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8
+SIZE (rust/crates/atty-0.2.14.crate) = 5470
+SHA256 (rust/crates/autocfg-1.1.0.crate) = d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa
+SIZE (rust/crates/autocfg-1.1.0.crate) = 13272
+SHA256 (rust/crates/bitflags-1.3.2.crate) = bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a
+SIZE (rust/crates/bitflags-1.3.2.crate) = 23021
+SHA256 (rust/crates/bstr-0.2.17.crate) = ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223
+SIZE (rust/crates/bstr-0.2.17.crate) = 330350
+SHA256 (rust/crates/bumpalo-3.12.0.crate) = 0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535
+SIZE (rust/crates/bumpalo-3.12.0.crate) = 81604
+SHA256 (rust/crates/cast-0.3.0.crate) = 37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5
+SIZE (rust/crates/cast-0.3.0.crate) = 11452
+SHA256 (rust/crates/cfg-if-1.0.0.crate) = baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd
+SIZE (rust/crates/cfg-if-1.0.0.crate) = 7934
+SHA256 (rust/crates/clap-2.34.0.crate) = a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c
+SIZE (rust/crates/clap-2.34.0.crate) = 202210
+SHA256 (rust/crates/criterion-0.3.6.crate) = b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f
+SIZE (rust/crates/criterion-0.3.6.crate) = 110967
+SHA256 (rust/crates/criterion-plot-0.4.5.crate) = 2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876
+SIZE (rust/crates/criterion-plot-0.4.5.crate) = 22721
+SHA256 (rust/crates/crossbeam-channel-0.5.6.crate) = c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521
+SIZE (rust/crates/crossbeam-channel-0.5.6.crate) = 90292
+SHA256 (rust/crates/crossbeam-deque-0.8.2.crate) = 715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc
+SIZE (rust/crates/crossbeam-deque-0.8.2.crate) = 21237
+SHA256 (rust/crates/crossbeam-epoch-0.9.13.crate) = 01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a
+SIZE (rust/crates/crossbeam-epoch-0.9.13.crate) = 48313
+SHA256 (rust/crates/crossbeam-utils-0.8.14.crate) = 4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f
+SIZE (rust/crates/crossbeam-utils-0.8.14.crate) = 42127
+SHA256 (rust/crates/csv-1.1.6.crate) = 22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1
+SIZE (rust/crates/csv-1.1.6.crate) = 886999
+SHA256 (rust/crates/csv-core-0.1.10.crate) = 2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90
+SIZE (rust/crates/csv-core-0.1.10.crate) = 25668
+SHA256 (rust/crates/either-1.8.1.crate) = 7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91
+SIZE (rust/crates/either-1.8.1.crate) = 16027
+SHA256 (rust/crates/half-1.8.2.crate) = eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7
+SIZE (rust/crates/half-1.8.2.crate) = 41874
+SHA256 (rust/crates/hermit-abi-0.1.19.crate) = 62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33
+SIZE (rust/crates/hermit-abi-0.1.19.crate) = 9979
+SHA256 (rust/crates/hermit-abi-0.2.6.crate) = ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7
+SIZE (rust/crates/hermit-abi-0.2.6.crate) = 13027
+SHA256 (rust/crates/itertools-0.10.5.crate) = b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473
+SIZE (rust/crates/itertools-0.10.5.crate) = 115354
+SHA256 (rust/crates/itoa-0.4.8.crate) = b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4
+SIZE (rust/crates/itoa-0.4.8.crate) = 11926
+SHA256 (rust/crates/itoa-1.0.5.crate) = fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440
+SIZE (rust/crates/itoa-1.0.5.crate) = 10622
+SHA256 (rust/crates/js-sys-0.3.60.crate) = 49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47
+SIZE (rust/crates/js-sys-0.3.60.crate) = 79257
+SHA256 (rust/crates/lazy_static-1.4.0.crate) = e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646
+SIZE (rust/crates/lazy_static-1.4.0.crate) = 10443
+SHA256 (rust/crates/libc-0.2.139.crate) = 201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79
+SIZE (rust/crates/libc-0.2.139.crate) = 638983
+SHA256 (rust/crates/log-0.4.17.crate) = abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e
+SIZE (rust/crates/log-0.4.17.crate) = 38028
+SHA256 (rust/crates/memchr-2.5.0.crate) = 2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d
+SIZE (rust/crates/memchr-2.5.0.crate) = 65812
+SHA256 (rust/crates/memoffset-0.7.1.crate) = 5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4
+SIZE (rust/crates/memoffset-0.7.1.crate) = 8556
+SHA256 (rust/crates/num-traits-0.2.15.crate) = 578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd
+SIZE (rust/crates/num-traits-0.2.15.crate) = 49262
+SHA256 (rust/crates/num_cpus-1.15.0.crate) = 0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b
+SIZE (rust/crates/num_cpus-1.15.0.crate) = 15680
+SHA256 (rust/crates/once_cell-1.17.0.crate) = 6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66
+SIZE (rust/crates/once_cell-1.17.0.crate) = 32736
+SHA256 (rust/crates/oorandom-11.1.3.crate) = 0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575
+SIZE (rust/crates/oorandom-11.1.3.crate) = 10068
+SHA256 (rust/crates/plotters-0.3.4.crate) = 2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97
+SIZE (rust/crates/plotters-0.3.4.crate) = 143219
+SHA256 (rust/crates/plotters-backend-0.3.4.crate) = 193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142
+SIZE (rust/crates/plotters-backend-0.3.4.crate) = 13416
+SHA256 (rust/crates/plotters-svg-0.3.3.crate) = f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f
+SIZE (rust/crates/plotters-svg-0.3.3.crate) = 6916
+SHA256 (rust/crates/proc-macro2-1.0.50.crate) = 6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2
+SIZE (rust/crates/proc-macro2-1.0.50.crate) = 41667
+SHA256 (rust/crates/quote-1.0.23.crate) = 8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b
+SIZE (rust/crates/quote-1.0.23.crate) = 28058
+SHA256 (rust/crates/rayon-1.6.1.crate) = 6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7
+SIZE (rust/crates/rayon-1.6.1.crate) = 166330
+SHA256 (rust/crates/rayon-core-1.10.2.crate) = 356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b
+SIZE (rust/crates/rayon-core-1.10.2.crate) = 70599
+SHA256 (rust/crates/regex-1.7.1.crate) = 48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733
+SIZE (rust/crates/regex-1.7.1.crate) = 239648
+SHA256 (rust/crates/regex-automata-0.1.10.crate) = 6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132
+SIZE (rust/crates/regex-automata-0.1.10.crate) = 114533
+SHA256 (rust/crates/regex-syntax-0.6.28.crate) = 456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848
+SIZE (rust/crates/regex-syntax-0.6.28.crate) = 299288
+SHA256 (rust/crates/ryu-1.0.12.crate) = 7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde
+SIZE (rust/crates/ryu-1.0.12.crate) = 47070
+SHA256 (rust/crates/same-file-1.0.6.crate) = 93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502
+SIZE (rust/crates/same-file-1.0.6.crate) = 10183
+SHA256 (rust/crates/scopeguard-1.1.0.crate) = d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd
+SIZE (rust/crates/scopeguard-1.1.0.crate) = 11470
+SHA256 (rust/crates/serde-1.0.152.crate) = bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb
+SIZE (rust/crates/serde-1.0.152.crate) = 77091
+SHA256 (rust/crates/serde_cbor-0.11.2.crate) = 2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5
+SIZE (rust/crates/serde_cbor-0.11.2.crate) = 44570
+SHA256 (rust/crates/serde_derive-1.0.152.crate) = af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e
+SIZE (rust/crates/serde_derive-1.0.152.crate) = 55586
+SHA256 (rust/crates/serde_json-1.0.91.crate) = 877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883
+SIZE (rust/crates/serde_json-1.0.91.crate) = 144654
+SHA256 (rust/crates/syn-1.0.107.crate) = 1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5
+SIZE (rust/crates/syn-1.0.107.crate) = 237539
+SHA256 (rust/crates/textwrap-0.11.0.crate) = d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060
+SIZE (rust/crates/textwrap-0.11.0.crate) = 17322
+SHA256 (rust/crates/tinytemplate-1.2.1.crate) = be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc
+SIZE (rust/crates/tinytemplate-1.2.1.crate) = 26490
+SHA256 (rust/crates/tinyvec-1.6.0.crate) = 87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50
+SIZE (rust/crates/tinyvec-1.6.0.crate) = 45991
+SHA256 (rust/crates/tinyvec_macros-0.1.0.crate) = cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c
+SIZE (rust/crates/tinyvec_macros-0.1.0.crate) = 1817
+SHA256 (rust/crates/unicode-ident-1.0.6.crate) = 84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc
+SIZE (rust/crates/unicode-ident-1.0.6.crate) = 42158
+SHA256 (rust/crates/unicode-normalization-0.1.22.crate) = 5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921
+SIZE (rust/crates/unicode-normalization-0.1.22.crate) = 122604
+SHA256 (rust/crates/unicode-segmentation-1.10.1.crate) = 1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36
+SIZE (rust/crates/unicode-segmentation-1.10.1.crate) = 98416
+SHA256 (rust/crates/unicode-width-0.1.10.crate) = c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b
+SIZE (rust/crates/unicode-width-0.1.10.crate) = 18968
+SHA256 (rust/crates/walkdir-2.3.2.crate) = 808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56
+SIZE (rust/crates/walkdir-2.3.2.crate) = 23516
+SHA256 (rust/crates/wasm-bindgen-0.2.83.crate) = eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268
+SIZE (rust/crates/wasm-bindgen-0.2.83.crate) = 169618
+SHA256 (rust/crates/wasm-bindgen-backend-0.2.83.crate) = 4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142
+SIZE (rust/crates/wasm-bindgen-backend-0.2.83.crate) = 25620
+SHA256 (rust/crates/wasm-bindgen-macro-0.2.83.crate) = 052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810
+SIZE (rust/crates/wasm-bindgen-macro-0.2.83.crate) = 12075
+SHA256 (rust/crates/wasm-bindgen-macro-support-0.2.83.crate) = 07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c
+SIZE (rust/crates/wasm-bindgen-macro-support-0.2.83.crate) = 18530
+SHA256 (rust/crates/wasm-bindgen-shared-0.2.83.crate) = 1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f
+SIZE (rust/crates/wasm-bindgen-shared-0.2.83.crate) = 7194
+SHA256 (rust/crates/web-sys-0.3.60.crate) = bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f
+SIZE (rust/crates/web-sys-0.3.60.crate) = 700485
+SHA256 (rust/crates/winapi-0.3.9.crate) = 5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419
+SIZE (rust/crates/winapi-0.3.9.crate) = 1200382
+SHA256 (rust/crates/winapi-i686-pc-windows-gnu-0.4.0.crate) = ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6
+SIZE (rust/crates/winapi-i686-pc-windows-gnu-0.4.0.crate) = 2918815
+SHA256 (rust/crates/winapi-util-0.1.5.crate) = 70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178
+SIZE (rust/crates/winapi-util-0.1.5.crate) = 10164
+SHA256 (rust/crates/winapi-x86_64-pc-windows-gnu-0.4.0.crate) = 712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f
+SIZE (rust/crates/winapi-x86_64-pc-windows-gnu-0.4.0.crate) = 2947998
+SHA256 (signalapp-Signal-FTS5-Extension-v0.2.1_GH0.tar.gz) = 2a5da0640b71230e05656ae06b857a73d4cc1828ab43d084ce1473da5692004d
+SIZE (signalapp-Signal-FTS5-Extension-v0.2.1_GH0.tar.gz) = 22433
diff --git a/devel/signal-fts5-extension/pkg-descr b/devel/signal-fts5-extension/pkg-descr
new file mode 100644
index 000000000000..5dda9721dbd2
--- /dev/null
+++ b/devel/signal-fts5-extension/pkg-descr
@@ -0,0 +1,9 @@
+Signal-FTS5-Extension is a C ABI library which exposes a FTS5 tokenizer function
+named signal_tokenizer that:
+
+  Segments UTF-8 strings into words according to Unicode standard
+  Normalizes and removes diacritics from words
+  Converts words to lower case
+
+When used as a custom FTS5 tokenizer this enables application to support CJK
+symbols in full-text search.