Cr8-xyz/blender
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

replace own unicode functions with versions from glib which support more unicode characters.

Browse Source 
added BLI_str_utf8_as_unicode(), BLI_str_utf8_from_unicode()
This commit is contained in:
Campbell Barton
2011-10-21 00:01:22 +00:00
parent 874c96a2b9
commit 2e549e0241
2 changed files with 203 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
source/blender/blenlib/BLI_string_utf8.h
View File

@@ -33,20 +33,27 @@
extern "C" {
#endif
char *BLI_strncpy_utf8(char *dst, const char *src, size_t maxncpy);
int BLI_utf8_invalid_byte(const char *str, int length);
int BLI_utf8_invalid_strip(char *str, int length);
char *BLI_strncpy_utf8(char *dst, const char *src, size_t maxncpy);
int BLI_utf8_invalid_byte(const char *str, int length);
int BLI_utf8_invalid_strip(char *str, int length);
/* copied from glib */
char *BLI_str_find_prev_char_utf8(const char *str, const char *p);
char *BLI_str_find_next_char_utf8(const char *p, const char *end);
char *BLI_str_prev_char_utf8(const char *p);
unsigned int BLI_str_utf8_as_unicode(const char *p);
unsigned int BLI_str_utf8_as_unicode_and_size(const char *p, size_t *index);
unsigned int BLI_str_utf8_as_unicode_step(const char *p, size_t *index);
size_t BLI_str_utf8_from_unicode(unsigned int c, char *outbuf);
char *BLI_str_find_prev_char_utf8(const char *str, const char *p);
char *BLI_str_find_next_char_utf8(const char *p, const char *end);
char *BLI_str_prev_char_utf8(const char *p);
/* wchar_t functions, copied from blenders own font.c originally */
size_t BLI_wstrlen_utf8(const wchar_t *src);
size_t BLI_strlen_utf8(const char *strc);
size_t BLI_strncpy_wchar_as_utf8(char *dst, const wchar_t *src, const size_t maxcpy);
size_t BLI_strncpy_wchar_from_utf8(wchar_t *dst, const char *src, const size_t maxcpy);
size_t BLI_wstrlen_utf8(const wchar_t *src);
size_t BLI_strlen_utf8(const char *strc);
size_t BLI_strncpy_wchar_as_utf8(char *dst, const wchar_t *src, const size_t maxcpy);
size_t BLI_strncpy_wchar_from_utf8(wchar_t *dst, const char *src, const size_t maxcpy);
#define BLI_STRING_MAX_UTF8 6
#ifdef __cplusplus
}

276
source/blender/blenlib/intern/string_utf8.c
View File

@@ -34,7 +34,7 @@
#include <wchar.h>
#include <wctype.h>
#include "BLI_string.h"
#include "BLI_string_utf8.h"
/* from libswish3, originally called u8_isvalid(),
* modified to return the index of the bad character (byte index not utf).
@@ -189,45 +189,11 @@ char *BLI_strncpy_utf8(char *dst, const char *src, size_t maxncpy)
/* --------------------------------------------------------------------------*/
/* wchar_t / utf8 functions */
/* UTF-8 <-> wchar transformations */
static size_t chtoutf8(const unsigned long c, char o[4])
{
// Variables and initialization
/* memset(o, 0, 4); */
// Create the utf-8 string
if (c < 0x80) {
o[0] = (char) c;
return 1;
}
else if (c < 0x800) {
o[0] = (0xC0 | (c>>6));
o[1] = (0x80 | (c & 0x3f));
return 2;
}
else if (c < 0x10000) {
o[0] = (0xe0 | (c >> 12));
o[1] = (0x80 | (c >>6 & 0x3f));
o[2] = (0x80 | (c & 0x3f));
return 3;
}
else if (c < 0x200000) {
o[0] = (0xf0 | (c>>18));
o[1] = (0x80 | (c >>12 & 0x3f));
o[2] = (0x80 | (c >> 6 & 0x3f));
o[3] = (0x80 | (c & 0x3f));
return 4;
}
/* should we assert here? */
return 0;
}
size_t BLI_strncpy_wchar_as_utf8(char *dst, const wchar_t *src, const size_t maxcpy)
{
size_t len = 0;
while(*src && len < maxcpy) { /* XXX can still run over the buffer because utf8 size isnt known :| */
len += chtoutf8(*src++, dst+len);
len += BLI_str_utf8_from_unicode(*src++, dst+len);
}
dst[len]= '\0';
@@ -238,11 +204,10 @@ size_t BLI_strncpy_wchar_as_utf8(char *dst, const wchar_t *src, const size_t max
/* wchar len in utf8 */
size_t BLI_wstrlen_utf8(const wchar_t *src)
{
char ch_dummy[4];
size_t len = 0;
while(*src) {
len += chtoutf8(*src++, ch_dummy);
len += BLI_str_utf8_from_unicode(*src++, NULL);
}
return len;
@@ -272,26 +237,6 @@ size_t BLI_strlen_utf8(const char *strc)
return len;
}
/* Converts Unicode to wchar
According to RFC 3629 "UTF-8, a transformation format of ISO 10646"
(http://tools.ietf.org/html/rfc3629), the valid UTF-8 encoding are:
Char. number range | UTF-8 octet sequence
(hexadecimal) | (binary)
--------------------+---------------------------------------------
0000 0000-0000 007F | 0xxxxxxx
0000 0080-0000 07FF | 110xxxxx 10xxxxxx
0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
If the encoding incidated by the first character is incorrect (because the
1 to 3 following characters do not match 10xxxxxx), the output is a '?' and
only a single input character is consumed.
*/
size_t BLI_strncpy_wchar_from_utf8(wchar_t *dst_w, const char *src_c, const size_t maxcpy)
{
int len=0;
@@ -299,32 +244,16 @@ size_t BLI_strncpy_wchar_from_utf8(wchar_t *dst_w, const char *src_c, const size
if(dst_w==NULL || src_c==NULL) return(0);
while(*src_c && len < maxcpy) {
if ((*src_c & 0xe0) == 0xc0) {
if((src_c[1] & 0x80) && (src_c[1] & 0x40) == 0x00) {
*dst_w=((src_c[0] &0x1f)<<6) | (src_c[1]&0x3f);
src_c++;
} else {
*dst_w = '?';
}
} else if ((*src_c & 0xf0) == 0xe0) {
if((src_c[1] & src_c[2] & 0x80) && ((src_c[1] | src_c[2]) & 0x40) == 0x00) {
*dst_w=((src_c[0] & 0x0f)<<12) | ((src_c[1]&0x3f)<<6) | (src_c[2]&0x3f);
src_c += 2;
} else {
*dst_w = '?';
}
} else if ((*src_c & 0xf8) == 0xf0) {
if((src_c[1] & src_c[2] & src_c[3] & 0x80) && ((src_c[1] | src_c[2] | src_c[3]) & 0x40) == 0x00) {
*dst_w=((src_c[0] & 0x07)<<18) | ((src_c[1]&0x1f)<<12) | ((src_c[2]&0x3f)<<6) | (src_c[3]&0x3f);
src_c += 3;
} else {
*dst_w = '?';
}
} else {
*dst_w=(src_c[0] & 0x7f);
size_t step= 0;
unsigned int unicode= BLI_str_utf8_as_unicode_and_size(src_c, &step);
if (unicode != (unsigned int)-1) {
*dst_w= (wchar_t)unicode;
src_c += step;
}
else {
*dst_w = '?';
src_c= BLI_str_find_next_char_utf8(src_c, NULL);
}
src_c++;
dst_w++;
len++;
}
@@ -334,13 +263,178 @@ size_t BLI_strncpy_wchar_from_utf8(wchar_t *dst_w, const char *src_c, const size
/* end wchar_t / utf8 functions */
/* --------------------------------------------------------------------------*/
/* copied from glib */
/* note, glib uses unsigned int for unicode, best we do the same,
* though we dont typedef it - campbell */
#define UTF8_COMPUTE(Char, Mask, Len) \
if (Char < 128) { \
Len = 1; \
Mask = 0x7f; \
} \
else if ((Char & 0xe0) == 0xc0) { \
Len = 2; \
Mask = 0x1f; \
} \
else if ((Char & 0xf0) == 0xe0) { \
Len = 3; \
Mask = 0x0f; \
} \
else if ((Char & 0xf8) == 0xf0) { \
Len = 4; \
Mask = 0x07; \
} \
else if ((Char & 0xfc) == 0xf8) { \
Len = 5; \
Mask = 0x03; \
} \
else if ((Char & 0xfe) == 0xfc) { \
Len = 6; \
Mask = 0x01; \
} \
else { \
Len = -1; \
}
#define UTF8_GET(Result, Chars, Count, Mask, Len) \
(Result) = (Chars)[0] & (Mask); \
for ((Count) = 1; (Count) < (Len); ++(Count)) { \
if (((Chars)[(Count)] & 0xc0) != 0x80) { \
(Result) = -1; \
break; \
} \
(Result) <<= 6; \
(Result) |= ((Chars)[(Count)] & 0x3f); \
}
/* was g_utf8_get_char */
/**
* g_utf8_find_prev_char:
* BLI_str_utf8_as_unicode:
* @p: a pointer to Unicode character encoded as UTF-8
*
* Converts a sequence of bytes encoded as UTF-8 to a Unicode character.
* If @p does not point to a valid UTF-8 encoded character, results are
* undefined. If you are not sure that the bytes are complete
* valid Unicode characters, you should use g_utf8_get_char_validated()
* instead.
*
* Return value: the resulting character
**/
unsigned int BLI_str_utf8_as_unicode(const char *p)
{
int i, mask = 0, len;
unsigned int result;
unsigned char c = (unsigned char) *p;
UTF8_COMPUTE (c, mask, len);
if (len == -1)
return (unsigned int)-1;
UTF8_GET (result, p, i, mask, len);
return result;
}
/* varient that increments the length */
unsigned int BLI_str_utf8_as_unicode_and_size(const char *p, size_t *index)
{
int i, mask = 0, len;
unsigned int result;
unsigned char c = (unsigned char) *p;
UTF8_COMPUTE (c, mask, len);
if (len == -1)
return (unsigned int)-1;
UTF8_GET (result, p, i, mask, len);
*index += len;
return result;
}
/* another varient that steps over the index */
unsigned int BLI_str_utf8_as_unicode_step(const char *p, size_t *index)
{
int i, mask = 0, len;
unsigned int result;
unsigned char c;
p += *index;
c= (unsigned char) *p;
UTF8_COMPUTE (c, mask, len);
if (len == -1) {
/* when called with NULL end, result will never be NULL,
* checks for a NULL character */
char *p_next= BLI_str_find_next_char_utf8(p, NULL);
/* will never return the same pointer unless '\0',
* eternal loop is prevented */
*index += (size_t)(p_next - p);
return (unsigned int)-1;
}
UTF8_GET (result, p, i, mask, len);
*index += len;
return result;
}
/* was g_unichar_to_utf8 */
/**
* BLI_str_utf8_from_unicode:
* @c: a Unicode character code
* @outbuf: output buffer, must have at least 6 bytes of space.
* If %NULL, the length will be computed and returned
* and nothing will be written to @outbuf.
*
* Converts a single character to UTF-8.
*
* Return value: number of bytes written
**/
size_t BLI_str_utf8_from_unicode(unsigned int c, char *outbuf)
{
/* If this gets modified, also update the copy in g_string_insert_unichar() */
unsigned int len = 0;
int first;
int i;
if (c < 0x80) {
first = 0;
len = 1;
}
else if (c < 0x800) {
first = 0xc0;
len = 2;
}
else if (c < 0x10000) {
first = 0xe0;
len = 3;
}
else if (c < 0x200000) {
first = 0xf0;
len = 4;
}
else if (c < 0x4000000) {
first = 0xf8;
len = 5;
}
else {
first = 0xfc;
len = 6;
}
if (outbuf) {
for (i = len - 1; i > 0; --i) {
outbuf[i] = (c & 0x3f) | 0x80;
c >>= 6;
}
outbuf[0] = c | first;
}
return len;
}
/* was g_utf8_find_prev_char */
/**
* BLI_str_find_prev_char_utf8:
* @str: pointer to the beginning of a UTF-8 encoded string
* @p: pointer to some position within @str
*
@@ -364,8 +458,9 @@ char * BLI_str_find_prev_char_utf8(const char *str, const char *p)
return NULL;
}
/* was g_utf8_find_next_char */
/**
* g_utf8_find_next_char:
* BLI_str_find_next_char_utf8:
* @p: a pointer to a position within a UTF-8 encoded string
* @end: a pointer to the byte following the end of the string,
* or %NULL to indicate that the string is nul-terminated.
@@ -395,8 +490,9 @@ char *BLI_str_find_next_char_utf8(const char *p, const char *end)
return (p == end) ? NULL : (char *)p;
}
/* was g_utf8_prev_char */
/**
* g_utf8_prev_char:
* BLI_str_prev_char_utf8:
* @p: a pointer to a position within a UTF-8 encoded string
*
* Finds the previous UTF-8 character in the string before @p.