diff options
Diffstat (limited to 'libbb/unicode.c')
| -rw-r--r-- | libbb/unicode.c | 210 |
1 files changed, 153 insertions, 57 deletions
diff --git a/libbb/unicode.c b/libbb/unicode.c index 8004295..844c154 100644 --- a/libbb/unicode.c +++ b/libbb/unicode.c @@ -9,22 +9,19 @@ #include "libbb.h" #include "unicode.h" -/* If it's not a constant... */ +/* If it's not #defined as a constant in unicode.h... */ #ifndef unicode_status uint8_t unicode_status; #endif -size_t FAST_FUNC bb_mbstrlen(const char *string) -{ - size_t width = mbstowcs(NULL, string, INT_MAX); - if (width == (size_t)-1L) - return strlen(string); - return width; -} +/* This file is compiled only if FEATURE_ASSUME_UNICODE is on. + * We check other options and decide whether to use libc support + * via locale, or use our own logic: + */ #if ENABLE_LOCALE_SUPPORT -/* Unicode support using libc */ +/* Unicode support using libc locale support. */ void FAST_FUNC init_unicode(void) { @@ -34,12 +31,12 @@ void FAST_FUNC init_unicode(void) if (unicode_status != UNICODE_UNKNOWN) return; - unicode_status = bb_mbstrlen(unicode_0x394) == 1 ? UNICODE_ON : UNICODE_OFF; + unicode_status = unicode_strlen(unicode_0x394) == 1 ? UNICODE_ON : UNICODE_OFF; } #else -/* Crude "locale support" which knows only C and Unicode locales */ +/* Homegrown Unicode support. It knows only C and Unicode locales. */ # if ENABLE_FEATURE_CHECK_UNICODE_IN_ENV void FAST_FUNC init_unicode(void) @@ -93,7 +90,6 @@ static size_t wcrtomb_internal(char *s, wchar_t wc) s[0] = wc | (uint8_t)(0x3f00 >> n); return n; } - size_t FAST_FUNC wcrtomb(char *s, wchar_t wc, mbstate_t *ps UNUSED_PARAM) { if (unicode_status != UNICODE_ON) { @@ -103,7 +99,6 @@ size_t FAST_FUNC wcrtomb(char *s, wchar_t wc, mbstate_t *ps UNUSED_PARAM) return wcrtomb_internal(s, wc); } - size_t FAST_FUNC wcstombs(char *dest, const wchar_t *src, size_t n) { size_t org_n = n; @@ -144,6 +139,51 @@ size_t FAST_FUNC wcstombs(char *dest, const wchar_t *src, size_t n) return org_n - n; } +static const char *mbstowc_internal(wchar_t *res, const char *src) +{ + int bytes; + unsigned c = (unsigned char) *src++; + + if (c <= 0x7f) { + *res = c; + return src; + } + + /* 80-7FF -> 110yyyxx 10xxxxxx */ + /* 800-FFFF -> 1110yyyy 10yyyyxx 10xxxxxx */ + /* 10000-1FFFFF -> 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx */ + /* 200000-3FFFFFF -> 111110tt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */ + /* 4000000-FFFFFFFF -> 111111tt 10tttttt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */ + bytes = 0; + do { + c <<= 1; + bytes++; + } while ((c & 0x80) && bytes < 6); + if (bytes == 1) + return NULL; + c = (uint8_t)(c) >> bytes; + + while (--bytes) { + unsigned ch = (unsigned char) *src++; + if ((ch & 0xc0) != 0x80) { + return NULL; + } + c = (c << 6) + (ch & 0x3f); + } + + /* TODO */ + /* Need to check that c isn't produced by overlong encoding */ + /* Example: 11000000 10000000 converts to NUL */ + /* 11110000 10000000 10000100 10000000 converts to 0x100 */ + /* correct encoding: 11000100 10000000 */ + if (c <= 0x7f) { /* crude check */ + return NULL; + //or maybe 0xfffd; /* replacement character */ + } + + *res = c; + return src; +} size_t FAST_FUNC mbstowcs(wchar_t *dest, const char *src, size_t n) { size_t org_n = n; @@ -162,58 +202,20 @@ size_t FAST_FUNC mbstowcs(wchar_t *dest, const char *src, size_t n) } while (n) { - int bytes; - unsigned c = (unsigned char) *src++; - - if (c <= 0x7f) { - if (dest) - *dest++ = c; - if (c == '\0') - break; - n--; - continue; - } - - /* 80-7FF -> 110yyyxx 10xxxxxx */ - /* 800-FFFF -> 1110yyyy 10yyyyxx 10xxxxxx */ - /* 10000-1FFFFF -> 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx */ - /* 200000-3FFFFFF -> 111110tt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */ - /* 4000000-FFFFFFFF -> 111111tt 10tttttt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */ - bytes = 0; - do { - c <<= 1; - bytes++; - } while ((c & 0x80) && bytes < 6); - if (bytes == 1) + wchar_t wc; + const char *rc = mbstowc_internal(dest ? dest : &wc, src); + if (rc == NULL) /* error */ return (size_t) -1L; - c = (uint8_t)(c) >> bytes; - - while (--bytes) { - unsigned ch = (unsigned char) *src++; - if ((ch & 0xc0) != 0x80) { - return (size_t) -1L; - } - c = (c << 6) + (ch & 0x3f); - } - - /* TODO */ - /* Need to check that c isn't produced by overlong encoding */ - /* Example: 11000000 10000000 converts to NUL */ - /* 11110000 10000000 10000100 10000000 converts to 0x100 */ - /* correct encoding: 11000100 10000000 */ - if (c <= 0x7f) { /* crude check */ - return (size_t) -1L; - //or maybe: c = 0xfffd; /* replacement character */ - } - if (dest) - *dest++ = c; + dest++; n--; } return org_n - n; } +#include "unicode_wcwidth.c" + int FAST_FUNC iswspace(wint_t wc) { return (unsigned)wc <= 0x7f && isspace(wc); @@ -229,4 +231,98 @@ int FAST_FUNC iswpunct(wint_t wc) return (unsigned)wc <= 0x7f && ispunct(wc); } +#endif /* Homegrown Unicode support */ + + +/* The rest is mostly same for libc and for "homegrown" support */ + +size_t FAST_FUNC unicode_strlen(const char *string) +{ + size_t width = mbstowcs(NULL, string, INT_MAX); + if (width == (size_t)-1L) + return strlen(string); + return width; +} + +char* FAST_FUNC unicode_cut_nchars(unsigned width, const char *src) +{ + char *dst; + unsigned dst_len; + + if (unicode_status != UNICODE_ON) + return xasprintf("%-*.*s", width, width, src); + + dst = NULL; + dst_len = 0; + while (1) { + int w; + wchar_t wc; + + dst = xrealloc(dst, dst_len + 2 * MB_CUR_MAX); +#if ENABLE_LOCALE_SUPPORT + { + mbstate_t mbst = { 0 }; + ssize_t rc = mbsrtowcs(&wc, &src, 1, &mbst); + if (rc <= 0) /* error, or end-of-string */ + break; + } +#else + src = mbstowc_internal(&wc, src); + if (!src || wc == 0) /* error, or end-of-string */ + break; #endif + w = wcwidth(wc); + if (w < 0) /* non-printable wchar */ + break; + width -= w; + if ((int)width < 0) { /* string is longer than width */ + width += w; + while (width) { + dst[dst_len++] = ' '; + width--; + } + break; + } +#if ENABLE_LOCALE_SUPPORT + { + mbstate_t mbst = { 0 }; + dst_len += wcrtomb(&dst[dst_len], wc, &mbst); + } +#else + dst_len += wcrtomb_internal(&dst[dst_len], wc); +#endif + } + dst[dst_len] = '\0'; + return dst; +} + +unsigned FAST_FUNC unicode_padding_to_width(unsigned width, const char *src) +{ + if (unicode_status != UNICODE_ON) { + return width - strnlen(src, width); + } + + while (1) { + int w; + wchar_t wc; + +#if ENABLE_LOCALE_SUPPORT + { + mbstate_t mbst = { 0 }; + ssize_t rc = mbsrtowcs(&wc, &src, 1, &mbst); + if (rc <= 0) /* error, or end-of-string */ + return width; + } +#else + src = mbstowc_internal(&wc, src); + if (!src || wc == 0) /* error, or end-of-string */ + return width; +#endif + w = wcwidth(wc); + if (w < 0) /* non-printable wchar */ + return width; + width -= w; + if ((int)width <= 0) /* string is longer than width */ + return 0; + } +} |