summaryrefslogtreecommitdiff
path: root/libbb/unicode.c
diff options
context:
space:
mode:
authorDenys Vlasenko2010-01-24 07:44:03 +0100
committerDenys Vlasenko2010-01-24 07:44:03 +0100
commit9f93d621925966c22ee51fdcb5def8e131596f9b (patch)
treec1024fa92d6f422df6cf0991d5c9c8a4977b8625 /libbb/unicode.c
parent5da9f96ad85a2d9119d92c7a3d89deca7d904210 (diff)
downloadbusybox-9f93d621925966c22ee51fdcb5def8e131596f9b.zip
busybox-9f93d621925966c22ee51fdcb5def8e131596f9b.tar.gz
libbb: better unicode width support. Hopefully fixes bug 839.
Also opens up a possibility to make other unicode stuff smaller and more correct later. but: function old new delta static.combining - 516 +516 bb_wcwidth - 328 +328 unicode_cut_nchars - 141 +141 mbstowc_internal - 93 +93 in_table - 78 +78 cal_main 899 961 +62 static.combining0x10000 - 40 +40 unicode_strlen - 31 +31 bb_mbstrlen 31 - -31 bb_mbstowcs 173 102 -71 ------------------------------------------------------------------------------ (add/remove: 7/1 grow/shrink: 1/1 up/down: 1289/-102) Total: 1187 bytes Uses code of Markus Kuhn, which is in public domain: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c "Permission to use, copy, modify, and distribute this software for any purpose and without fee is hereby granted. The author disclaims all warranties with regard to this software." Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com>
Diffstat (limited to 'libbb/unicode.c')
-rw-r--r--libbb/unicode.c210
1 files changed, 153 insertions, 57 deletions
diff --git a/libbb/unicode.c b/libbb/unicode.c
index 8004295..844c154 100644
--- a/libbb/unicode.c
+++ b/libbb/unicode.c
@@ -9,22 +9,19 @@
#include "libbb.h"
#include "unicode.h"
-/* If it's not a constant... */
+/* If it's not #defined as a constant in unicode.h... */
#ifndef unicode_status
uint8_t unicode_status;
#endif
-size_t FAST_FUNC bb_mbstrlen(const char *string)
-{
- size_t width = mbstowcs(NULL, string, INT_MAX);
- if (width == (size_t)-1L)
- return strlen(string);
- return width;
-}
+/* This file is compiled only if FEATURE_ASSUME_UNICODE is on.
+ * We check other options and decide whether to use libc support
+ * via locale, or use our own logic:
+ */
#if ENABLE_LOCALE_SUPPORT
-/* Unicode support using libc */
+/* Unicode support using libc locale support. */
void FAST_FUNC init_unicode(void)
{
@@ -34,12 +31,12 @@ void FAST_FUNC init_unicode(void)
if (unicode_status != UNICODE_UNKNOWN)
return;
- unicode_status = bb_mbstrlen(unicode_0x394) == 1 ? UNICODE_ON : UNICODE_OFF;
+ unicode_status = unicode_strlen(unicode_0x394) == 1 ? UNICODE_ON : UNICODE_OFF;
}
#else
-/* Crude "locale support" which knows only C and Unicode locales */
+/* Homegrown Unicode support. It knows only C and Unicode locales. */
# if ENABLE_FEATURE_CHECK_UNICODE_IN_ENV
void FAST_FUNC init_unicode(void)
@@ -93,7 +90,6 @@ static size_t wcrtomb_internal(char *s, wchar_t wc)
s[0] = wc | (uint8_t)(0x3f00 >> n);
return n;
}
-
size_t FAST_FUNC wcrtomb(char *s, wchar_t wc, mbstate_t *ps UNUSED_PARAM)
{
if (unicode_status != UNICODE_ON) {
@@ -103,7 +99,6 @@ size_t FAST_FUNC wcrtomb(char *s, wchar_t wc, mbstate_t *ps UNUSED_PARAM)
return wcrtomb_internal(s, wc);
}
-
size_t FAST_FUNC wcstombs(char *dest, const wchar_t *src, size_t n)
{
size_t org_n = n;
@@ -144,6 +139,51 @@ size_t FAST_FUNC wcstombs(char *dest, const wchar_t *src, size_t n)
return org_n - n;
}
+static const char *mbstowc_internal(wchar_t *res, const char *src)
+{
+ int bytes;
+ unsigned c = (unsigned char) *src++;
+
+ if (c <= 0x7f) {
+ *res = c;
+ return src;
+ }
+
+ /* 80-7FF -> 110yyyxx 10xxxxxx */
+ /* 800-FFFF -> 1110yyyy 10yyyyxx 10xxxxxx */
+ /* 10000-1FFFFF -> 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx */
+ /* 200000-3FFFFFF -> 111110tt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */
+ /* 4000000-FFFFFFFF -> 111111tt 10tttttt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */
+ bytes = 0;
+ do {
+ c <<= 1;
+ bytes++;
+ } while ((c & 0x80) && bytes < 6);
+ if (bytes == 1)
+ return NULL;
+ c = (uint8_t)(c) >> bytes;
+
+ while (--bytes) {
+ unsigned ch = (unsigned char) *src++;
+ if ((ch & 0xc0) != 0x80) {
+ return NULL;
+ }
+ c = (c << 6) + (ch & 0x3f);
+ }
+
+ /* TODO */
+ /* Need to check that c isn't produced by overlong encoding */
+ /* Example: 11000000 10000000 converts to NUL */
+ /* 11110000 10000000 10000100 10000000 converts to 0x100 */
+ /* correct encoding: 11000100 10000000 */
+ if (c <= 0x7f) { /* crude check */
+ return NULL;
+ //or maybe 0xfffd; /* replacement character */
+ }
+
+ *res = c;
+ return src;
+}
size_t FAST_FUNC mbstowcs(wchar_t *dest, const char *src, size_t n)
{
size_t org_n = n;
@@ -162,58 +202,20 @@ size_t FAST_FUNC mbstowcs(wchar_t *dest, const char *src, size_t n)
}
while (n) {
- int bytes;
- unsigned c = (unsigned char) *src++;
-
- if (c <= 0x7f) {
- if (dest)
- *dest++ = c;
- if (c == '\0')
- break;
- n--;
- continue;
- }
-
- /* 80-7FF -> 110yyyxx 10xxxxxx */
- /* 800-FFFF -> 1110yyyy 10yyyyxx 10xxxxxx */
- /* 10000-1FFFFF -> 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx */
- /* 200000-3FFFFFF -> 111110tt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */
- /* 4000000-FFFFFFFF -> 111111tt 10tttttt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */
- bytes = 0;
- do {
- c <<= 1;
- bytes++;
- } while ((c & 0x80) && bytes < 6);
- if (bytes == 1)
+ wchar_t wc;
+ const char *rc = mbstowc_internal(dest ? dest : &wc, src);
+ if (rc == NULL) /* error */
return (size_t) -1L;
- c = (uint8_t)(c) >> bytes;
-
- while (--bytes) {
- unsigned ch = (unsigned char) *src++;
- if ((ch & 0xc0) != 0x80) {
- return (size_t) -1L;
- }
- c = (c << 6) + (ch & 0x3f);
- }
-
- /* TODO */
- /* Need to check that c isn't produced by overlong encoding */
- /* Example: 11000000 10000000 converts to NUL */
- /* 11110000 10000000 10000100 10000000 converts to 0x100 */
- /* correct encoding: 11000100 10000000 */
- if (c <= 0x7f) { /* crude check */
- return (size_t) -1L;
- //or maybe: c = 0xfffd; /* replacement character */
- }
-
if (dest)
- *dest++ = c;
+ dest++;
n--;
}
return org_n - n;
}
+#include "unicode_wcwidth.c"
+
int FAST_FUNC iswspace(wint_t wc)
{
return (unsigned)wc <= 0x7f && isspace(wc);
@@ -229,4 +231,98 @@ int FAST_FUNC iswpunct(wint_t wc)
return (unsigned)wc <= 0x7f && ispunct(wc);
}
+#endif /* Homegrown Unicode support */
+
+
+/* The rest is mostly same for libc and for "homegrown" support */
+
+size_t FAST_FUNC unicode_strlen(const char *string)
+{
+ size_t width = mbstowcs(NULL, string, INT_MAX);
+ if (width == (size_t)-1L)
+ return strlen(string);
+ return width;
+}
+
+char* FAST_FUNC unicode_cut_nchars(unsigned width, const char *src)
+{
+ char *dst;
+ unsigned dst_len;
+
+ if (unicode_status != UNICODE_ON)
+ return xasprintf("%-*.*s", width, width, src);
+
+ dst = NULL;
+ dst_len = 0;
+ while (1) {
+ int w;
+ wchar_t wc;
+
+ dst = xrealloc(dst, dst_len + 2 * MB_CUR_MAX);
+#if ENABLE_LOCALE_SUPPORT
+ {
+ mbstate_t mbst = { 0 };
+ ssize_t rc = mbsrtowcs(&wc, &src, 1, &mbst);
+ if (rc <= 0) /* error, or end-of-string */
+ break;
+ }
+#else
+ src = mbstowc_internal(&wc, src);
+ if (!src || wc == 0) /* error, or end-of-string */
+ break;
#endif
+ w = wcwidth(wc);
+ if (w < 0) /* non-printable wchar */
+ break;
+ width -= w;
+ if ((int)width < 0) { /* string is longer than width */
+ width += w;
+ while (width) {
+ dst[dst_len++] = ' ';
+ width--;
+ }
+ break;
+ }
+#if ENABLE_LOCALE_SUPPORT
+ {
+ mbstate_t mbst = { 0 };
+ dst_len += wcrtomb(&dst[dst_len], wc, &mbst);
+ }
+#else
+ dst_len += wcrtomb_internal(&dst[dst_len], wc);
+#endif
+ }
+ dst[dst_len] = '\0';
+ return dst;
+}
+
+unsigned FAST_FUNC unicode_padding_to_width(unsigned width, const char *src)
+{
+ if (unicode_status != UNICODE_ON) {
+ return width - strnlen(src, width);
+ }
+
+ while (1) {
+ int w;
+ wchar_t wc;
+
+#if ENABLE_LOCALE_SUPPORT
+ {
+ mbstate_t mbst = { 0 };
+ ssize_t rc = mbsrtowcs(&wc, &src, 1, &mbst);
+ if (rc <= 0) /* error, or end-of-string */
+ return width;
+ }
+#else
+ src = mbstowc_internal(&wc, src);
+ if (!src || wc == 0) /* error, or end-of-string */
+ return width;
+#endif
+ w = wcwidth(wc);
+ if (w < 0) /* non-printable wchar */
+ return width;
+ width -= w;
+ if ((int)width <= 0) /* string is longer than width */
+ return 0;
+ }
+}