Cr8-xyz/blender
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix T71273: Bad encoding of utf-8 for Text objects

Browse Source 
`BLI_strncpy_wchar_from_utf8` internally assumes `wchar_t` is 32 bits
which is not the case on windows.

The solution is to replace `wchar_t` with `char32_t`.

Thanks to @robbott for compatibility on macOS.

Differential Revision: https://developer.blender.org/D6198
This commit is contained in:
mano-wii
2019-11-22 12:26:54 -03:00
parent 1304cee920
commit 177dfc6384
16 changed files with 198 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
extern/wcwidth/README.blender vendored
View File

@@ -2,4 +2,6 @@ Project: WC Width
URL: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c URL: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
License: ICS License: ICS
Upstream version: 2007-05-26 Upstream version: 2007-05-26
Local modifications: None Local modifications:
* Fix T33192
Bad encoding of utf-8 on windows systems.

12
extern/wcwidth/wcwidth.c vendored
View File

@@ -59,8 +59,6 @@
* Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
*/ */
#include <wchar.h>
#include "wcwidth.h" #include "wcwidth.h"
struct interval { struct interval {
@@ -69,7 +67,7 @@ struct interval {
}; };
/* auxiliary function for binary search in interval table */ /* auxiliary function for binary search in interval table */
static int bisearch(wchar_t ucs, const struct interval *table, int max) { static int bisearch(char32_t ucs, const struct interval *table, int max) {
int min = 0; int min = 0;
int mid; int mid;
@@ -121,7 +119,7 @@ static int bisearch(wchar_t ucs, const struct interval *table, int max) {
* in ISO 10646. * in ISO 10646.
*/ */
int mk_wcwidth(wchar_t ucs) int mk_wcwidth(char32_t ucs)
{ {
/* sorted list of non-overlapping intervals of non-spacing characters */ /* sorted list of non-overlapping intervals of non-spacing characters */
/* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */ /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */
@@ -206,7 +204,7 @@ int mk_wcwidth(wchar_t ucs)
} }
int mk_wcswidth(const wchar_t *pwcs, size_t n) int mk_wcswidth(const char32_t *pwcs, size_t n)
{ {
int w, width = 0; int w, width = 0;
@@ -229,7 +227,7 @@ int mk_wcswidth(const wchar_t *pwcs, size_t n)
* the traditional terminal character-width behaviour. It is not * the traditional terminal character-width behaviour. It is not
* otherwise recommended for general use. * otherwise recommended for general use.
*/ */
int mk_wcwidth_cjk(wchar_t ucs) int mk_wcwidth_cjk(char32_t ucs)
{ {
/* sorted list of non-overlapping intervals of East Asian Ambiguous /* sorted list of non-overlapping intervals of East Asian Ambiguous
* characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */ * characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */
@@ -297,7 +295,7 @@ int mk_wcwidth_cjk(wchar_t ucs)
} }
int mk_wcswidth_cjk(const wchar_t *pwcs, size_t n) int mk_wcswidth_cjk(const char32_t *pwcs, size_t n)
{ {
int w, width = 0; int w, width = 0;

18
extern/wcwidth/wcwidth.h vendored
View File

@@ -20,11 +20,19 @@
#ifndef __WCWIDTH_H__ #ifndef __WCWIDTH_H__
#define __WCWIDTH_H__ #define __WCWIDTH_H__
#include <wchar.h> #ifndef __cplusplus
# if defined(__APPLE__)
/* The <uchar.h> standard header is missing on macOS. */
#include <stddef.h>
typedef unsigned int char32_t;
# else
# include <uchar.h>
# endif
#endif
int mk_wcwidth(wchar_t ucs); int mk_wcwidth(char32_t ucs);
int mk_wcswidth(const wchar_t *pwcs, size_t n); int mk_wcswidth(const char32_t *pwcs, size_t n);
int mk_wcwidth_cjk(wchar_t ucs); int mk_wcwidth_cjk(char32_t ucs);
int mk_wcswidth_cjk(const wchar_t *pwcs, size_t n); int mk_wcswidth_cjk(const char32_t *pwcs, size_t n);
#endif #endif

2
source/blender/blenfont/intern/blf_font.c
View File

@@ -504,7 +504,7 @@ int blf_font_draw_mono(FontBLF *font, const char *str, size_t len, int cwidth)
/* do not return this loop if clipped, we want every character tested */ /* do not return this loop if clipped, we want every character tested */
blf_glyph_render(font, gc, g, (float)pen_x, (float)pen_y); blf_glyph_render(font, gc, g, (float)pen_x, (float)pen_y);
col = BLI_wcwidth((wchar_t)c); col = BLI_wcwidth((char32_t)c);
if (col < 0) { if (col < 0) {
col = 1; col = 1;
} }

12
source/blender/blenkernel/BKE_font.h
View File

@@ -27,8 +27,6 @@
extern "C" { extern "C" {
#endif #endif
#include <wchar.h>
struct CharInfo; struct CharInfo;
struct Curve; struct Curve;
struct Main; struct Main;
@@ -48,7 +46,7 @@ typedef struct EditFontSelBox {
} EditFontSelBox; } EditFontSelBox;
typedef struct EditFont { typedef struct EditFont {
wchar_t *textbuf; char32_t *textbuf;
struct CharInfo *textbufinfo; struct CharInfo *textbufinfo;
/* array of rectangles & rotation */ /* array of rectangles & rotation */
@@ -90,7 +88,7 @@ bool BKE_vfont_to_curve_ex(struct Object *ob,
struct Curve *cu, struct Curve *cu,
int mode, int mode,
struct ListBase *r_nubase, struct ListBase *r_nubase,
const wchar_t **r_text, const char32_t **r_text,
int *r_text_len, int *r_text_len,
bool *r_text_free, bool *r_text_free,
struct CharTrans **r_chartransdata); struct CharTrans **r_chartransdata);
@@ -101,13 +99,13 @@ int BKE_vfont_select_get(struct Object *ob, int *r_start, int *r_end);
void BKE_vfont_select_clamp(struct Object *ob); void BKE_vfont_select_clamp(struct Object *ob);
void BKE_vfont_clipboard_free(void); void BKE_vfont_clipboard_free(void);
void BKE_vfont_clipboard_set(const wchar_t *text_buf, void BKE_vfont_clipboard_set(const char32_t *text_buf,
const struct CharInfo *info_buf, const struct CharInfo *info_buf,
const size_t len); const size_t len);
void BKE_vfont_clipboard_get(wchar_t **r_text_buf, void BKE_vfont_clipboard_get(char32_t **r_text_buf,
struct CharInfo **r_info_buf, struct CharInfo **r_info_buf,
size_t *r_len_utf8, size_t *r_len_utf8,
size_t *r_len_wchar); size_t *r_len_utf32);
#ifdef __cplusplus #ifdef __cplusplus
} }

41
source/blender/blenkernel/intern/font.c
View File

@@ -25,7 +25,6 @@
#include <string.h> #include <string.h>
#include <math.h> #include <math.h>
#include <stdlib.h> #include <stdlib.h>
#include <wchar.h>
#include <wctype.h> #include <wctype.h>
#include "CLG_log.h" #include "CLG_log.h"
@@ -691,7 +690,7 @@ static bool vfont_to_curve(Object *ob,
int mode, int mode,
VFontToCurveIter *iter_data, VFontToCurveIter *iter_data,
ListBase *r_nubase, ListBase *r_nubase,
const wchar_t **r_text, const char32_t **r_text,
int *r_text_len, int *r_text_len,
bool *r_text_free, bool *r_text_free,
struct CharTrans **r_chartransdata) struct CharTrans **r_chartransdata)
@@ -712,8 +711,8 @@ static bool vfont_to_curve(Object *ob,
int curbox; int curbox;
int selstart, selend; int selstart, selend;
int cnr = 0, lnr = 0, wsnr = 0; int cnr = 0, lnr = 0, wsnr = 0;
const wchar_t *mem = NULL; const char32_t *mem = NULL;
wchar_t ascii; char32_t ascii;
bool ok = false; bool ok = false;
const float font_size = cu->fsize * iter_data->scale_to_fit; const float font_size = cu->fsize * iter_data->scale_to_fit;
const float xof_scale = cu->xof / font_size; const float xof_scale = cu->xof / font_size;
@@ -759,16 +758,16 @@ static bool vfont_to_curve(Object *ob,
custrinfo = ef->textbufinfo; custrinfo = ef->textbufinfo;
} }
else { else {
wchar_t *mem_tmp; char32_t *mem_tmp;
slen = cu->len_wchar; slen = cu->len_wchar;
/* Create unicode string */ /* Create unicode string */
mem_tmp = MEM_malloc_arrayN((slen + 1), sizeof(wchar_t), "convertedmem"); mem_tmp = MEM_malloc_arrayN((slen + 1), sizeof(*mem_tmp), "convertedmem");
if (!mem_tmp) { if (!mem_tmp) {
return ok; return ok;
} }
BLI_strncpy_wchar_from_utf8(mem_tmp, cu->str, slen + 1); BLI_str_utf8_as_utf32(mem_tmp, cu->str, slen + 1);
if (cu->strinfo == NULL) { /* old file */ if (cu->strinfo == NULL) { /* old file */
cu->strinfo = MEM_calloc_arrayN((slen + 4), sizeof(CharInfo), "strinfo compat"); cu->strinfo = MEM_calloc_arrayN((slen + 4), sizeof(CharInfo), "strinfo compat");
@@ -1605,7 +1604,7 @@ bool BKE_vfont_to_curve_ex(Object *ob,
Curve *cu, Curve *cu,
int mode, int mode,
ListBase *r_nubase, ListBase *r_nubase,
const wchar_t **r_text, const char32_t **r_text,
int *r_text_len, int *r_text_len,
bool *r_text_free, bool *r_text_free,
struct CharTrans **r_chartransdata) struct CharTrans **r_chartransdata)
@@ -1649,9 +1648,9 @@ bool BKE_vfont_to_curve(Object *ob, int mode)
* \{ */ * \{ */
static struct { static struct {
wchar_t *text_buffer; char32_t *text_buffer;
CharInfo *info_buffer; CharInfo *info_buffer;
size_t len_wchar; size_t len_utf32;
size_t len_utf8; size_t len_utf8;
} g_vfont_clipboard = {NULL}; } g_vfont_clipboard = {NULL};
@@ -1659,19 +1658,19 @@ void BKE_vfont_clipboard_free(void)
{ {
MEM_SAFE_FREE(g_vfont_clipboard.text_buffer); MEM_SAFE_FREE(g_vfont_clipboard.text_buffer);
MEM_SAFE_FREE(g_vfont_clipboard.info_buffer); MEM_SAFE_FREE(g_vfont_clipboard.info_buffer);
g_vfont_clipboard.len_wchar = 0; g_vfont_clipboard.len_utf32 = 0;
g_vfont_clipboard.len_utf8 = 0; g_vfont_clipboard.len_utf8 = 0;
} }
void BKE_vfont_clipboard_set(const wchar_t *text_buf, const CharInfo *info_buf, const size_t len) void BKE_vfont_clipboard_set(const char32_t *text_buf, const CharInfo *info_buf, const size_t len)
{ {
wchar_t *text; char32_t *text;
CharInfo *info; CharInfo *info;
/* clean previous buffers*/ /* clean previous buffers*/
BKE_vfont_clipboard_free(); BKE_vfont_clipboard_free();
text = MEM_malloc_arrayN((len + 1), sizeof(wchar_t), __func__); text = MEM_malloc_arrayN((len + 1), sizeof(*text), __func__);
if (text == NULL) { if (text == NULL) {
return; return;
} }
@@ -1682,21 +1681,21 @@ void BKE_vfont_clipboard_set(const wchar_t *text_buf, const CharInfo *info_buf,
return; return;
} }
memcpy(text, text_buf, len * sizeof(wchar_t)); memcpy(text, text_buf, len * sizeof(*text));
text[len] = '\0'; text[len] = '\0';
memcpy(info, info_buf, len * sizeof(CharInfo)); memcpy(info, info_buf, len * sizeof(CharInfo));
/* store new buffers */ /* store new buffers */
g_vfont_clipboard.text_buffer = text; g_vfont_clipboard.text_buffer = text;
g_vfont_clipboard.info_buffer = info; g_vfont_clipboard.info_buffer = info;
g_vfont_clipboard.len_utf8 = BLI_wstrlen_utf8(text); g_vfont_clipboard.len_utf8 = BLI_str_utf32_as_utf8_len(text);
g_vfont_clipboard.len_wchar = len; g_vfont_clipboard.len_utf32 = len;
} }
void BKE_vfont_clipboard_get(wchar_t **r_text_buf, void BKE_vfont_clipboard_get(char32_t **r_text_buf,
CharInfo **r_info_buf, CharInfo **r_info_buf,
size_t *r_len_utf8, size_t *r_len_utf8,
size_t *r_len_wchar) size_t *r_len_utf32)
{ {
if (r_text_buf) { if (r_text_buf) {
*r_text_buf = g_vfont_clipboard.text_buffer; *r_text_buf = g_vfont_clipboard.text_buffer;
@@ -1706,8 +1705,8 @@ void BKE_vfont_clipboard_get(wchar_t **r_text_buf,
*r_info_buf = g_vfont_clipboard.info_buffer; *r_info_buf = g_vfont_clipboard.info_buffer;
} }
if (r_len_wchar) { if (r_len_utf32) {
*r_len_wchar = g_vfont_clipboard.len_wchar; *r_len_utf32 = g_vfont_clipboard.len_utf32;
} }
if (r_len_utf8) { if (r_len_utf8) {

2
source/blender/blenkernel/intern/object_dupli.c
View File

@@ -491,7 +491,7 @@ static void make_duplis_font(const DupliContext *ctx)
float vec[3], obmat[4][4], pmat[4][4], fsize, xof, yof; float vec[3], obmat[4][4], pmat[4][4], fsize, xof, yof;
int text_len, a; int text_len, a;
size_t family_len; size_t family_len;
const wchar_t *text = NULL; const char32_t *text = NULL;
bool text_free = false; bool text_free = false;
/* font dupliverts not supported inside collections */ /* font dupliverts not supported inside collections */

1
source/blender/blenkernel/intern/text.c
View File

@@ -25,7 +25,6 @@
#include <string.h> /* strstr */ #include <string.h> /* strstr */
#include <sys/types.h> #include <sys/types.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <wchar.h>
#include <wctype.h> #include <wctype.h>
#include "MEM_guardedalloc.h" #include "MEM_guardedalloc.h"

2
source/blender/blenlib/BLI_string_cursor_utf8.h
View File

@@ -45,7 +45,7 @@ void BLI_str_cursor_step_utf8(const char *str,
eStrCursorJumpType jump, eStrCursorJumpType jump,
bool use_init_step); bool use_init_step);
void BLI_str_cursor_step_wchar(const wchar_t *str, void BLI_str_cursor_step_utf32(const char32_t *str,
size_t maxlen, size_t maxlen,
int *pos, int *pos,
eStrCursorJumpDirection direction, eStrCursorJumpDirection direction,

12
source/blender/blenlib/BLI_string_utf8.h
View File

@@ -26,6 +26,7 @@ extern "C" {
#endif #endif
#include "BLI_compiler_attrs.h" #include "BLI_compiler_attrs.h"
#include "BLI_sys_types.h"
char *BLI_strncpy_utf8(char *__restrict dst, const char *__restrict src, size_t maxncpy) char *BLI_strncpy_utf8(char *__restrict dst, const char *__restrict src, size_t maxncpy)
ATTR_NONNULL(); ATTR_NONNULL();
@@ -48,6 +49,13 @@ unsigned int BLI_str_utf8_as_unicode_and_size_safe(const char *__restrict p,
unsigned int BLI_str_utf8_as_unicode_step(const char *__restrict p, size_t *__restrict index) unsigned int BLI_str_utf8_as_unicode_step(const char *__restrict p, size_t *__restrict index)
ATTR_NONNULL(); ATTR_NONNULL();
size_t BLI_str_utf8_from_unicode(unsigned int c, char *outbuf); size_t BLI_str_utf8_from_unicode(unsigned int c, char *outbuf);
size_t BLI_str_utf8_as_utf32(char32_t *__restrict dst_w,
const char *__restrict src_c,
const size_t maxncpy) ATTR_NONNULL();
size_t BLI_str_utf32_as_utf8(char *__restrict dst,
const char32_t *__restrict src,
const size_t maxncpy) ATTR_NONNULL();
size_t BLI_str_utf32_as_utf8_len(const char32_t *src) ATTR_NONNULL();
char *BLI_str_find_prev_char_utf8(const char *str, const char *p) ATTR_NONNULL(); char *BLI_str_find_prev_char_utf8(const char *str, const char *p) ATTR_NONNULL();
char *BLI_str_find_next_char_utf8(const char *p, const char *end) ATTR_NONNULL(1); char *BLI_str_find_next_char_utf8(const char *p, const char *end) ATTR_NONNULL(1);
@@ -68,8 +76,8 @@ size_t BLI_strncpy_wchar_from_utf8(wchar_t *__restrict dst,
const size_t maxcpy) ATTR_NONNULL(); const size_t maxcpy) ATTR_NONNULL();
/* count columns that character/string occupies, based on wcwidth.c */ /* count columns that character/string occupies, based on wcwidth.c */
int BLI_wcwidth(wchar_t ucs); int BLI_wcwidth(char32_t ucs);
int BLI_wcswidth(const wchar_t *pwcs, size_t n) ATTR_NONNULL(); int BLI_wcswidth(const char32_t *pwcs, size_t n) ATTR_NONNULL();
/* warning, can return -1 on bad chars */ /* warning, can return -1 on bad chars */
int BLI_str_utf8_char_width(const char *p) ATTR_NONNULL(); int BLI_str_utf8_char_width(const char *p) ATTR_NONNULL();
int BLI_str_utf8_char_width_safe(const char *p) ATTR_NONNULL(); int BLI_str_utf8_char_width_safe(const char *p) ATTR_NONNULL();

9
source/blender/blenlib/BLI_sys_types.h
View File

@@ -72,6 +72,15 @@ typedef uint64_t u_int64_t;
#include <stddef.h> /* size_t define */ #include <stddef.h> /* size_t define */
#include <stdbool.h> #include <stdbool.h>
#ifndef __cplusplus
# if defined(__APPLE__)
/* The <uchar.h> standard header is missing on macOS. */
typedef unsigned int char32_t;
# else
# include <uchar.h>
# endif
#endif
typedef unsigned int uint; typedef unsigned int uint;
typedef unsigned short ushort; typedef unsigned short ushort;
typedef unsigned long ulong; typedef unsigned long ulong;

16
source/blender/blenlib/intern/string_cursor_utf8.c
View File

@@ -211,12 +211,12 @@ void BLI_str_cursor_step_utf8(const char *str,
} }
} }
/* wchar_t version of BLI_str_cursor_step_utf8 (keep in sync!) /* UTF32 version of BLI_str_cursor_step_utf8 (keep in sync!)
* less complex since it doesn't need to do multi-byte stepping. * less complex since it doesn't need to do multi-byte stepping.
*/ */
/* helper funcs so we can match BLI_str_cursor_step_utf8 */ /* helper funcs so we can match BLI_str_cursor_step_utf8 */
static bool wchar_t_step_next(const wchar_t *UNUSED(str), size_t maxlen, int *pos) static bool cursor_step_next_utf32(const char32_t *UNUSED(str), size_t maxlen, int *pos)
{ {
if ((*pos) >= (int)maxlen) { if ((*pos) >= (int)maxlen) {
return false; return false;
@@ -225,7 +225,7 @@ static bool wchar_t_step_next(const wchar_t *UNUSED(str), size_t maxlen, int *po
return true; return true;
} }
static bool wchar_t_step_prev(const wchar_t *UNUSED(str), size_t UNUSED(maxlen), int *pos) static bool cursor_step_prev_utf32(const char32_t *UNUSED(str), size_t UNUSED(maxlen), int *pos)
{ {
if ((*pos) <= 0) { if ((*pos) <= 0) {
return false; return false;
@@ -234,7 +234,7 @@ static bool wchar_t_step_prev(const wchar_t *UNUSED(str), size_t UNUSED(maxlen),
return true; return true;
} }
void BLI_str_cursor_step_wchar(const wchar_t *str, void BLI_str_cursor_step_utf32(const char32_t *str,
size_t maxlen, size_t maxlen,
int *pos, int *pos,
eStrCursorJumpDirection direction, eStrCursorJumpDirection direction,
@@ -245,7 +245,7 @@ void BLI_str_cursor_step_wchar(const wchar_t *str,
if (direction == STRCUR_DIR_NEXT) { if (direction == STRCUR_DIR_NEXT) {
if (use_init_step) { if (use_init_step) {
wchar_t_step_next(str, maxlen, pos); cursor_step_next_utf32(str, maxlen, pos);
} }
else { else {
BLI_assert(jump == STRCUR_JUMP_DELIM); BLI_assert(jump == STRCUR_JUMP_DELIM);
@@ -259,7 +259,7 @@ void BLI_str_cursor_step_wchar(const wchar_t *str,
* look at function cursor_delim_type_unicode() for complete * look at function cursor_delim_type_unicode() for complete
* list of special character, ctr -> */ * list of special character, ctr -> */
while ((*pos) < maxlen) { while ((*pos) < maxlen) {
if (wchar_t_step_next(str, maxlen, pos)) { if (cursor_step_next_utf32(str, maxlen, pos)) {
if ((jump != STRCUR_JUMP_ALL) && if ((jump != STRCUR_JUMP_ALL) &&
(delim_type != cursor_delim_type_unicode((uint)str[*pos]))) { (delim_type != cursor_delim_type_unicode((uint)str[*pos]))) {
break; break;
@@ -273,7 +273,7 @@ void BLI_str_cursor_step_wchar(const wchar_t *str,
} }
else if (direction == STRCUR_DIR_PREV) { else if (direction == STRCUR_DIR_PREV) {
if (use_init_step) { if (use_init_step) {
wchar_t_step_prev(str, maxlen, pos); cursor_step_prev_utf32(str, maxlen, pos);
} }
else { else {
BLI_assert(jump == STRCUR_JUMP_DELIM); BLI_assert(jump == STRCUR_JUMP_DELIM);
@@ -288,7 +288,7 @@ void BLI_str_cursor_step_wchar(const wchar_t *str,
* list of special character, ctr -> */ * list of special character, ctr -> */
while ((*pos) > 0) { while ((*pos) > 0) {
const int pos_prev = *pos; const int pos_prev = *pos;
if (wchar_t_step_prev(str, maxlen, pos)) { if (cursor_step_prev_utf32(str, maxlen, pos)) {
if ((jump != STRCUR_JUMP_ALL) && if ((jump != STRCUR_JUMP_ALL) &&
(delim_type != cursor_delim_type_unicode((uint)str[*pos]))) { (delim_type != cursor_delim_type_unicode((uint)str[*pos]))) {
/* left only: compensate for index/change in direction */ /* left only: compensate for index/change in direction */

95
source/blender/blenlib/intern/string_utf8.c
View File

@@ -430,6 +430,11 @@ size_t BLI_strncpy_wchar_from_utf8(wchar_t *__restrict dst_w,
size_t step = 0; size_t step = 0;
uint unicode = BLI_str_utf8_as_unicode_and_size(src_c, &step); uint unicode = BLI_str_utf8_as_unicode_and_size(src_c, &step);
if (unicode != BLI_UTF8_ERR) { if (unicode != BLI_UTF8_ERR) {
/* TODO: `wchar_t` type is an implementation-defined and may represent
* 16-bit or 32-bit depending on operating system.
* So the ideal would be to do the corresponding encoding.
* But for now just assert that it has no conflicting use. */
BLI_assert(step <= sizeof(wchar_t));
*dst_w = (wchar_t)unicode; *dst_w = (wchar_t)unicode;
src_c += step; src_c += step;
} }
@@ -451,12 +456,12 @@ size_t BLI_strncpy_wchar_from_utf8(wchar_t *__restrict dst_w,
/* count columns that character/string occupies, based on wcwidth.c */ /* count columns that character/string occupies, based on wcwidth.c */
int BLI_wcwidth(wchar_t ucs) int BLI_wcwidth(char32_t ucs)
{ {
return mk_wcwidth(ucs); return mk_wcwidth(ucs);
} }
int BLI_wcswidth(const wchar_t *pwcs, size_t n) int BLI_wcswidth(const char32_t *pwcs, size_t n)
{ {
return mk_wcswidth(pwcs, n); return mk_wcswidth(pwcs, n);
} }
@@ -468,7 +473,7 @@ int BLI_str_utf8_char_width(const char *p)
return -1; return -1;
} }
return BLI_wcwidth((wchar_t)unicode); return BLI_wcwidth((char32_t)unicode);
} }
int BLI_str_utf8_char_width_safe(const char *p) int BLI_str_utf8_char_width_safe(const char *p)
@@ -480,7 +485,7 @@ int BLI_str_utf8_char_width_safe(const char *p)
return 1; return 1;
} }
columns = BLI_wcwidth((wchar_t)unicode); columns = BLI_wcwidth((char32_t)unicode);
return (columns < 0) ? 1 : columns; return (columns < 0) ? 1 : columns;
} }
@@ -726,6 +731,88 @@ size_t BLI_str_utf8_from_unicode(uint c, char *outbuf)
return len; return len;
} }
size_t BLI_str_utf8_as_utf32(char32_t *__restrict dst_w,
const char *__restrict src_c,
const size_t maxncpy)
{
const size_t maxlen = maxncpy - 1;
size_t len = 0;
BLI_assert(maxncpy != 0);
#ifdef DEBUG_STRSIZE
memset(dst_w, 0xff, sizeof(*dst_w) * maxncpy);
#endif
while (*src_c && len != maxlen) {
size_t step = 0;
uint unicode = BLI_str_utf8_as_unicode_and_size(src_c, &step);
if (unicode != BLI_UTF8_ERR) {
*dst_w = unicode;
src_c += step;
}
else {
*dst_w = '?';
src_c = BLI_str_find_next_char_utf8(src_c, NULL);
}
dst_w++;
len++;
}
*dst_w = 0;
return len;
}
size_t BLI_str_utf32_as_utf8(char *__restrict dst,
const char32_t *__restrict src,
const size_t maxncpy)
{
const size_t maxlen = maxncpy - 1;
/* 6 is max utf8 length of an unicode char. */
const int64_t maxlen_secured = (int64_t)maxlen - 6;
size_t len = 0;
BLI_assert(maxncpy != 0);
#ifdef DEBUG_STRSIZE
memset(dst, 0xff, sizeof(*dst) * maxncpy);
#endif
while (*src && len <= maxlen_secured) {
len += BLI_str_utf8_from_unicode((uint)*src++, dst + len);
}
/* We have to be more careful for the last six bytes,
* to avoid buffer overflow in case utf8-encoded char would be too long for our dst buffer. */
while (*src) {
char t[6];
size_t l = BLI_str_utf8_from_unicode((uint)*src++, t);
BLI_assert(l <= 6);
if (len + l > maxlen) {
break;
}
memcpy(dst + len, t, l);
len += l;
}
dst[len] = '\0';
return len;
}
/* utf32 len in utf8 */
size_t BLI_str_utf32_as_utf8_len(const char32_t *src)
{
size_t len = 0;
while (*src) {
len += BLI_str_utf8_from_unicode((uint)*src++, NULL);
}
return len;
}
/* was g_utf8_find_prev_char */ /* was g_utf8_find_prev_char */
/** /**
* BLI_str_find_prev_char_utf8: * BLI_str_find_prev_char_utf8:

52
source/blender/editors/curve/editfont.c
View File

@@ -468,7 +468,8 @@ static int kill_selection(Object *obedit, int ins) /* 1 == new character */
if (ins == 0) { if (ins == 0) {
getfrom++; getfrom++;
} }
size = (ef->len * sizeof(wchar_t)) - (selstart * sizeof(wchar_t)) + (offset * sizeof(wchar_t)); size = (ef->len * sizeof(*ef->textbuf)) - (selstart * sizeof(*ef->textbuf)) +
(offset * sizeof(*ef->textbuf));
memmove(ef->textbuf + selstart, ef->textbuf + getfrom, size); memmove(ef->textbuf + selstart, ef->textbuf + getfrom, size);
memmove(ef->textbufinfo + selstart, memmove(ef->textbufinfo + selstart,
ef->textbufinfo + getfrom, ef->textbufinfo + getfrom,
@@ -488,7 +489,7 @@ static int kill_selection(Object *obedit, int ins) /* 1 == new character */
/* text_update_edited(C, scene, obedit, 1, FO_EDIT); */ /* text_update_edited(C, scene, obedit, 1, FO_EDIT); */
static bool font_paste_wchar(Object *obedit, static bool font_paste_wchar(Object *obedit,
const wchar_t *str, const char32_t *str,
const size_t str_len, const size_t str_len,
/* optional */ /* optional */
struct CharInfo *str_info) struct CharInfo *str_info)
@@ -507,9 +508,10 @@ static bool font_paste_wchar(Object *obedit,
kill_selection(obedit, 0); kill_selection(obedit, 0);
if (str_len) { if (str_len) {
int size = (ef->len * sizeof(wchar_t)) - (ef->pos * sizeof(wchar_t)) + sizeof(wchar_t); int size = (ef->len * sizeof(*ef->textbuf)) - (ef->pos * sizeof(*ef->textbuf)) +
sizeof(*ef->textbuf);
memmove(ef->textbuf + ef->pos + str_len, ef->textbuf + ef->pos, size); memmove(ef->textbuf + ef->pos + str_len, ef->textbuf + ef->pos, size);
memcpy(ef->textbuf + ef->pos, str, str_len * sizeof(wchar_t)); memcpy(ef->textbuf + ef->pos, str, str_len * sizeof(*ef->textbuf));
memmove(ef->textbufinfo + ef->pos + str_len, memmove(ef->textbufinfo + ef->pos + str_len,
ef->textbufinfo + ef->pos, ef->textbufinfo + ef->pos,
@@ -538,9 +540,9 @@ static bool font_paste_utf8(bContext *C, const char *str, const size_t str_len)
int tmplen; int tmplen;
wchar_t *mem = MEM_mallocN((sizeof(wchar_t) * (str_len + 1)), __func__); char32_t *mem = MEM_mallocN((sizeof(*mem) * (str_len + 1)), __func__);
tmplen = BLI_strncpy_wchar_from_utf8(mem, str, str_len + 1); tmplen = BLI_str_utf8_as_utf32(mem, str, str_len + 1);
retval = font_paste_wchar(obedit, mem, tmplen, NULL); retval = font_paste_wchar(obedit, mem, tmplen, NULL);
@@ -919,7 +921,7 @@ static void copy_selection(Object *obedit)
Curve *cu = obedit->data; Curve *cu = obedit->data;
EditFont *ef = cu->editfont; EditFont *ef = cu->editfont;
char *buf = NULL; char *buf = NULL;
wchar_t *text_buf; char32_t *text_buf;
size_t len_utf8; size_t len_utf8;
/* internal clipboard (for style) */ /* internal clipboard (for style) */
@@ -930,7 +932,7 @@ static void copy_selection(Object *obedit)
/* system clipboard */ /* system clipboard */
buf = MEM_mallocN(len_utf8 + 1, __func__); buf = MEM_mallocN(len_utf8 + 1, __func__);
if (buf) { if (buf) {
BLI_strncpy_wchar_as_utf8(buf, text_buf, len_utf8 + 1); BLI_str_utf32_as_utf8(buf, text_buf, len_utf8 + 1);
WM_clipboard_text_set(buf, false); WM_clipboard_text_set(buf, false);
MEM_freeN(buf); MEM_freeN(buf);
} }
@@ -1004,7 +1006,7 @@ void FONT_OT_text_cut(wmOperatorType *ot)
static bool paste_selection(Object *obedit, ReportList *reports) static bool paste_selection(Object *obedit, ReportList *reports)
{ {
wchar_t *text_buf; char32_t *text_buf;
CharInfo *info_buf; CharInfo *info_buf;
size_t len; size_t len;
@@ -1024,7 +1026,7 @@ static int paste_text_exec(bContext *C, wmOperator *op)
Object *obedit = CTX_data_edit_object(C); Object *obedit = CTX_data_edit_object(C);
int retval; int retval;
size_t len_utf8; size_t len_utf8;
wchar_t *text_buf; char32_t *text_buf;
/* Store both clipboards as utf8 for comparison, /* Store both clipboards as utf8 for comparison,
* Give priority to the internal 'vfont' clipboard with its 'CharInfo' text styles * Give priority to the internal 'vfont' clipboard with its 'CharInfo' text styles
@@ -1050,7 +1052,7 @@ static int paste_text_exec(bContext *C, wmOperator *op)
return OPERATOR_CANCELLED; return OPERATOR_CANCELLED;
} }
BLI_strncpy_wchar_as_utf8(clipboard_vfont.buf, text_buf, len_utf8 + 1); BLI_str_utf32_as_utf8(clipboard_vfont.buf, text_buf, len_utf8 + 1);
} }
if (clipboard_vfont.buf && STREQ(clipboard_vfont.buf, clipboard_system.buf)) { if (clipboard_vfont.buf && STREQ(clipboard_vfont.buf, clipboard_system.buf)) {
@@ -1164,7 +1166,7 @@ static int move_cursor(bContext *C, int type, const bool select)
case PREV_WORD: { case PREV_WORD: {
int pos = ef->pos; int pos = ef->pos;
BLI_str_cursor_step_wchar( BLI_str_cursor_step_utf32(
ef->textbuf, ef->len, &pos, STRCUR_DIR_PREV, STRCUR_JUMP_DELIM, true); ef->textbuf, ef->len, &pos, STRCUR_DIR_PREV, STRCUR_JUMP_DELIM, true);
ef->pos = pos; ef->pos = pos;
cursmove = FO_CURS; cursmove = FO_CURS;
@@ -1173,7 +1175,7 @@ static int move_cursor(bContext *C, int type, const bool select)
case NEXT_WORD: { case NEXT_WORD: {
int pos = ef->pos; int pos = ef->pos;
BLI_str_cursor_step_wchar( BLI_str_cursor_step_utf32(
ef->textbuf, ef->len, &pos, STRCUR_DIR_NEXT, STRCUR_JUMP_DELIM, true); ef->textbuf, ef->len, &pos, STRCUR_DIR_NEXT, STRCUR_JUMP_DELIM, true);
ef->pos = pos; ef->pos = pos;
cursmove = FO_CURS; cursmove = FO_CURS;
@@ -1528,7 +1530,7 @@ static int delete_exec(bContext *C, wmOperator *op)
break; break;
case DEL_NEXT_WORD: { case DEL_NEXT_WORD: {
int pos = ef->pos; int pos = ef->pos;
BLI_str_cursor_step_wchar( BLI_str_cursor_step_utf32(
ef->textbuf, ef->len, &pos, STRCUR_DIR_NEXT, STRCUR_JUMP_DELIM, true); ef->textbuf, ef->len, &pos, STRCUR_DIR_NEXT, STRCUR_JUMP_DELIM, true);
range[0] = ef->pos; range[0] = ef->pos;
range[1] = pos; range[1] = pos;
@@ -1537,7 +1539,7 @@ static int delete_exec(bContext *C, wmOperator *op)
case DEL_PREV_WORD: { case DEL_PREV_WORD: {
int pos = ef->pos; int pos = ef->pos;
BLI_str_cursor_step_wchar( BLI_str_cursor_step_utf32(
ef->textbuf, ef->len, &pos, STRCUR_DIR_PREV, STRCUR_JUMP_DELIM, true); ef->textbuf, ef->len, &pos, STRCUR_DIR_PREV, STRCUR_JUMP_DELIM, true);
range[0] = pos; range[0] = pos;
range[1] = ef->pos; range[1] = ef->pos;
@@ -1864,12 +1866,12 @@ void ED_curve_editfont_make(Object *obedit)
if (ef == NULL) { if (ef == NULL) {
ef = cu->editfont = MEM_callocN(sizeof(EditFont), "editfont"); ef = cu->editfont = MEM_callocN(sizeof(EditFont), "editfont");
ef->textbuf = MEM_callocN((MAXTEXT + 4) * sizeof(wchar_t), "texteditbuf"); ef->textbuf = MEM_callocN((MAXTEXT + 4) * sizeof(*ef->textbuf), "texteditbuf");
ef->textbufinfo = MEM_callocN((MAXTEXT + 4) * sizeof(CharInfo), "texteditbufinfo"); ef->textbufinfo = MEM_callocN((MAXTEXT + 4) * sizeof(CharInfo), "texteditbufinfo");
} }
/* Convert the original text to wchar_t */ /* Convert the original text to wchar_t */
len_wchar = BLI_strncpy_wchar_from_utf8(ef->textbuf, cu->str, MAXTEXT + 4); len_wchar = BLI_str_utf8_as_utf32(ef->textbuf, cu->str, MAXTEXT + 4);
BLI_assert(len_wchar == cu->len_wchar); BLI_assert(len_wchar == cu->len_wchar);
ef->len = len_wchar; ef->len = len_wchar;
BLI_assert(ef->len >= 0); BLI_assert(ef->len >= 0);
@@ -1901,13 +1903,13 @@ void ED_curve_editfont_load(Object *obedit)
/* Calculate the actual string length in UTF-8 variable characters */ /* Calculate the actual string length in UTF-8 variable characters */
cu->len_wchar = ef->len; cu->len_wchar = ef->len;
cu->len = BLI_wstrlen_utf8(ef->textbuf); cu->len = BLI_str_utf32_as_utf8_len(ef->textbuf);
/* Alloc memory for UTF-8 variable char length string */ /* Alloc memory for UTF-8 variable char length string */
cu->str = MEM_mallocN(cu->len + sizeof(wchar_t), "str"); cu->str = MEM_mallocN(cu->len + sizeof(char32_t), "str");
/* Copy the wchar to UTF-8 */ /* Copy the wchar to UTF-8 */
BLI_strncpy_wchar_as_utf8(cu->str, ef->textbuf, cu->len + 1); BLI_str_utf32_as_utf8(cu->str, ef->textbuf, cu->len + 1);
if (cu->strinfo) { if (cu->strinfo) {
MEM_freeN(cu->strinfo); MEM_freeN(cu->strinfo);
@@ -1943,7 +1945,7 @@ static int set_case(bContext *C, int ccase)
Object *obedit = CTX_data_edit_object(C); Object *obedit = CTX_data_edit_object(C);
Curve *cu = obedit->data; Curve *cu = obedit->data;
EditFont *ef = cu->editfont; EditFont *ef = cu->editfont;
wchar_t *str; char32_t *str;
int len; int len;
int selstart, selend; int selstart, selend;
@@ -2010,18 +2012,16 @@ static int toggle_case_exec(bContext *C, wmOperator *UNUSED(op))
Object *obedit = CTX_data_edit_object(C); Object *obedit = CTX_data_edit_object(C);
Curve *cu = obedit->data; Curve *cu = obedit->data;
EditFont *ef = cu->editfont; EditFont *ef = cu->editfont;
wchar_t *str; char32_t *str;
int len, ccase = CASE_UPPER; int ccase = CASE_UPPER;
len = wcslen(ef->textbuf);
str = ef->textbuf; str = ef->textbuf;
while (len) { while (*str) {
if (*str >= 'a' && *str <= 'z') { if (*str >= 'a' && *str <= 'z') {
ccase = CASE_LOWER; ccase = CASE_LOWER;
break; break;
} }
len--;
str++; str++;
} }

8
source/blender/editors/curve/editfont_undo.c
View File

@@ -20,7 +20,6 @@
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <wchar.h>
#include "MEM_guardedalloc.h" #include "MEM_guardedalloc.h"
@@ -58,7 +57,7 @@
* \{ */ * \{ */
typedef struct UndoFont { typedef struct UndoFont {
wchar_t *textbuf; char32_t *textbuf;
struct CharInfo *textbufinfo; struct CharInfo *textbufinfo;
int len, pos, selstart, selend; int len, pos, selstart, selend;
@@ -233,7 +232,7 @@ static void undofont_to_editfont(UndoFont *uf, Curve *cu)
uf_arraystore_expand(uf); uf_arraystore_expand(uf);
#endif #endif
final_size = sizeof(wchar_t) * (uf->len + 1); final_size = sizeof(*ef->textbuf) * (uf->len + 1);
memcpy(ef->textbuf, uf->textbuf, final_size); memcpy(ef->textbuf, uf->textbuf, final_size);
final_size = sizeof(CharInfo) * (uf->len + 1); final_size = sizeof(CharInfo) * (uf->len + 1);
@@ -259,7 +258,8 @@ static void *undofont_from_editfont(UndoFont *uf, Curve *cu)
size_t final_size; size_t final_size;
final_size = sizeof(wchar_t) * (ef->len + 1); BLI_assert(sizeof(*uf->textbuf) == sizeof(*ef->textbuf));
final_size = sizeof(*uf->textbuf) * (ef->len + 1);
uf->textbuf = MEM_mallocN(final_size, __func__); uf->textbuf = MEM_mallocN(final_size, __func__);
memcpy(uf->textbuf, ef->textbuf, final_size); memcpy(uf->textbuf, ef->textbuf, final_size);

2
source/blender/makesrna/intern/rna_curve.c
View File

@@ -585,7 +585,7 @@ static void rna_Curve_body_set(PointerRNA *ptr, const char *value)
MEM_freeN(cu->strinfo); MEM_freeN(cu->strinfo);
} }
cu->str = MEM_mallocN(len_bytes + sizeof(wchar_t), "str"); cu->str = MEM_mallocN(len_bytes + sizeof(char32_t), "str");
cu->strinfo = MEM_callocN((len_chars + 4) * sizeof(CharInfo), "strinfo"); cu->strinfo = MEM_callocN((len_chars + 4) * sizeof(CharInfo), "strinfo");
BLI_strncpy(cu->str, value, len_bytes + 1); BLI_strncpy(cu->str, value, len_bytes + 1);