circlecloud/tmux
1
0
Fork 0
mirror of https://github.com/tmux/tmux.git synced 2024-12-12 17:38:48 +00:00
Code Issues Projects Releases Wiki Activity

Tidy up new UTF-8 code and make it more generic.

Browse Source
This commit is contained in:
nicm 2020-05-25 18:19:29 +00:00
parent dc893405e1
commit 49ec074271
4 changed files with 154 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
grid.c
View File

@ -114,7 +114,7 @@ grid_extended_cell(struct grid_line *gl, struct grid_cell_entry *gce,
gl->flags |= GRID_LINE_EXTENDED; gl->flags |= GRID_LINE_EXTENDED;
gee = &gl->extddata[gce->offset]; gee = &gl->extddata[gce->offset];
gee->data = utf8_map_big(&gc->data); utf8_from_data(&gc->data, &gee->data);
gee->attr = gc->attr; gee->attr = gc->attr;
gee->flags = flags; gee->flags = flags;
gee->fg = gc->fg; gee->fg = gc->fg;
@ -496,7 +496,7 @@ grid_get_cell1(struct grid_line *gl, u_int px, struct grid_cell *gc)
gc->fg = gee->fg; gc->fg = gee->fg;
gc->bg = gee->bg; gc->bg = gee->bg;
gc->us = gee->us; gc->us = gee->us;
utf8_get_big(gee->data, &gc->data); utf8_to_data(gee->data, &gc->data);
} }
return; return;
} }
@ -570,7 +570,7 @@ grid_set_cells(struct grid *gd, u_int px, u_int py, const struct grid_cell *gc,
gce = &gl->celldata[px + i]; gce = &gl->celldata[px + i];
if (grid_need_extended_cell(gce, gc)) { if (grid_need_extended_cell(gce, gc)) {
gee = grid_extended_cell(gl, gce, gc); gee = grid_extended_cell(gl, gce, gc);
gee->data = utf8_set_big(s[i], 1); gee->data = utf8_build_one(s[i], 1);
} else } else
grid_store_cell(gce, gc, s[i]); grid_store_cell(gce, gc, s[i]);
} }

15
tmux.h
View File

@ -596,10 +596,13 @@ struct msg_write_close {
#define ALL_MOUSE_MODES (MODE_MOUSE_STANDARD|MODE_MOUSE_BUTTON|MODE_MOUSE_ALL) #define ALL_MOUSE_MODES (MODE_MOUSE_STANDARD|MODE_MOUSE_BUTTON|MODE_MOUSE_ALL)
#define MOTION_MOUSE_MODES (MODE_MOUSE_BUTTON|MODE_MOUSE_ALL) #define MOTION_MOUSE_MODES (MODE_MOUSE_BUTTON|MODE_MOUSE_ALL)
/* A single UTF-8 character. */
typedef u_int utf8_char;
/* /*
* A single UTF-8 character. UTF8_SIZE must be big enough to hold combining * An expanded UTF-8 character. UTF8_SIZE must be big enough to hold combining
* characters as well. It can't be more than 32 bytes without changes to how * characters as well. It can't be more than 32 bytes without changes to how
* big characters are stored. * characters are stored.
*/ */
#define UTF8_SIZE 21 #define UTF8_SIZE 21
struct utf8_data { struct utf8_data {
@ -673,7 +676,7 @@ struct grid_cell {
/* Grid extended cell entry. */ /* Grid extended cell entry. */
struct grid_extd_entry { struct grid_extd_entry {
u_int data; utf8_char data;
u_short attr; u_short attr;
u_char flags; u_char flags;
int fg; int fg;
@ -2889,9 +2892,9 @@ u_int session_group_attached_count(struct session_group *);
void session_renumber_windows(struct session *); void session_renumber_windows(struct session *);
/* utf8.c */ /* utf8.c */
u_int utf8_set_big(char, u_int); utf8_char utf8_build_one(char, u_int);
u_int utf8_map_big(const struct utf8_data *); enum utf8_state utf8_from_data(const struct utf8_data *, utf8_char *);
void utf8_get_big(u_int, struct utf8_data *); void utf8_to_data(utf8_char, struct utf8_data *);
void utf8_set(struct utf8_data *, u_char); void utf8_set(struct utf8_data *, u_char);
void utf8_copy(struct utf8_data *, const struct utf8_data *); void utf8_copy(struct utf8_data *, const struct utf8_data *);
enum utf8_state utf8_open(struct utf8_data *, u_char); enum utf8_state utf8_open(struct utf8_data *, u_char);

295
utf8.c
View File

@ -27,153 +27,157 @@
#include "tmux.h" #include "tmux.h"
static int utf8_width(wchar_t); struct utf8_item {
u_int offset;
struct utf8_big_item { RB_ENTRY(utf8_item) entry;
u_int index;
RB_ENTRY(utf8_big_item) entry;
char data[UTF8_SIZE]; char data[UTF8_SIZE];
u_char size; u_char size;
}; };
RB_HEAD(utf8_big_tree, utf8_big_item); RB_HEAD(utf8_tree, utf8_item);
static int static int
utf8_big_cmp(struct utf8_big_item *bi1, struct utf8_big_item *bi2) utf8_cmp(struct utf8_item *ui1, struct utf8_item *ui2)
{ {
if (bi1->size < bi2->size) if (ui1->size < ui2->size)
return (-1); return (-1);
if (bi1->size > bi2->size) if (ui1->size > ui2->size)
return (1); return (1);
return (memcmp(bi1->data, bi2->data, bi1->size)); return (memcmp(ui1->data, ui2->data, ui1->size));
} }
RB_GENERATE_STATIC(utf8_big_tree, utf8_big_item, entry, utf8_big_cmp); RB_GENERATE_STATIC(utf8_tree, utf8_item, entry, utf8_cmp);
static struct utf8_big_tree utf8_big_tree = RB_INITIALIZER(utf8_big_tree); static struct utf8_tree utf8_tree = RB_INITIALIZER(utf8_tree);
static struct utf8_big_item *utf8_big_list; static struct utf8_item *utf8_list;
static u_int utf8_big_list_size; static u_int utf8_list_size;
static u_int utf8_big_list_used; static u_int utf8_list_used;
union utf8_big_map { union utf8_map {
u_int value; utf8_char uc;
struct { struct {
u_char flags; u_char flags;
#define UTF8_BIG_SIZE 0x1f #define UTF8_FLAG_SIZE 0x1f
#define UTF8_BIG_WIDTH2 0x20 #define UTF8_FLAG_WIDTH2 0x20
u_char data[3]; u_char data[3];
}; };
} __packed; } __packed;
static const union utf8_big_map utf8_big_space1 = { static const union utf8_map utf8_space1 = {
.flags = 1, .flags = 1,
.data = " " .data = " "
}; };
static const union utf8_big_map utf8_big_space2 = { static const union utf8_map utf8_space2 = {
.flags = UTF8_BIG_WIDTH2|2, .flags = UTF8_FLAG_WIDTH2|2,
.data = " " .data = " "
}; };
/* Get a big item by index. */ /* Get a UTF-8 item by offset. */
static struct utf8_big_item * static struct utf8_item *
utf8_get_big_item(const char *data, size_t size) utf8_get_item(const char *data, size_t size)
{ {
struct utf8_big_item bi; struct utf8_item ui;
memcpy(bi.data, data, size); memcpy(ui.data, data, size);
bi.size = size; ui.size = size;
return (RB_FIND(utf8_big_tree, &utf8_big_tree, &bi)); return (RB_FIND(utf8_tree, &utf8_tree, &ui));
} }
/* Add a big item. */ /* Expand UTF-8 list. */
static int static int
utf8_put_big_item(const char *data, size_t size, u_int *index) utf8_expand_list(void)
{ {
struct utf8_big_item *bi; if (utf8_list_size == 0xffffff)
return (-1);
bi = utf8_get_big_item(data, size); if (utf8_list_size == 0)
if (bi != NULL) { utf8_list_size = 256;
*index = bi->index; else if (utf8_list_size > 0x7fffff)
log_debug("%s: have %.*s at %u", __func__, (int)size, data, utf8_list_size = 0xffffff;
*index); else
return (0); utf8_list_size *= 2;
} utf8_list = xreallocarray(utf8_list, utf8_list_size, sizeof *utf8_list);
if (utf8_big_list_used == utf8_big_list_size) {
if (utf8_big_list_size == 0xffffff)
return (-1);
if (utf8_big_list_size == 0)
utf8_big_list_size = 256;
else if (utf8_big_list_size > 0x7fffff)
utf8_big_list_size = 0xffffff;
else
utf8_big_list_size *= 2;
utf8_big_list = xreallocarray(utf8_big_list, utf8_big_list_size,
sizeof *utf8_big_list);
}
*index = utf8_big_list_used++;
bi = &utf8_big_list[*index];
bi->index = *index;
memcpy(bi->data, data, size);
bi->size = size;
RB_INSERT(utf8_big_tree, &utf8_big_tree, bi);
log_debug("%s: added %.*s at %u", __func__, (int)size, data, *index);
return (0); return (0);
} }
/* Get UTF-8 as index into buffer. */ /* Add a UTF-8 item. */
u_int static int
utf8_map_big(const struct utf8_data *ud) utf8_put_item(const char *data, size_t size, u_int *offset)
{ {
union utf8_big_map m = { .value = 0 }; struct utf8_item *ui;
u_int o;
const char *data = ud->data;
size_t size = ud->size;
if (ud->width != 1 && ud->width != 2) ui = utf8_get_item(data, size);
return (utf8_big_space1.value); if (ui != NULL) {
*offset = ui->offset;
if (size > UTF8_BIG_SIZE) log_debug("%s: have %.*s at %u", __func__, (int)size, data,
goto fail; *offset);
if (size == 1) return (0);
return (utf8_set_big(data[0], 1));
m.flags = size;
if (ud->width == 2)
m.flags |= UTF8_BIG_WIDTH2;
if (size <= 3) {
memcpy(&m.data, data, size);
return (m.value);
} }
if (utf8_put_big_item(data, size, &o) != 0) if (utf8_list_used == utf8_list_size && utf8_expand_list() != 0)
return (-1);
*offset = utf8_list_used++;
ui = &utf8_list[*offset];
ui->offset = *offset;
memcpy(ui->data, data, size);
ui->size = size;
RB_INSERT(utf8_tree, &utf8_tree, ui);
log_debug("%s: added %.*s at %u", __func__, (int)size, data, *offset);
return (0);
}
/* Get UTF-8 character from data. */
enum utf8_state
utf8_from_data(const struct utf8_data *ud, utf8_char *uc)
{
union utf8_map m = { .uc = 0 };
u_int offset;
if (ud->width != 1 && ud->width != 2)
return (utf8_space1.uc);
if (ud->size > UTF8_FLAG_SIZE)
goto fail; goto fail;
m.data[0] = (o & 0xff); if (ud->size == 1)
m.data[1] = (o >> 8) & 0xff; return (utf8_build_one(ud->data[0], 1));
m.data[2] = (o >> 16);
return (m.value); m.flags = ud->size;
if (ud->width == 2)
m.flags |= UTF8_FLAG_WIDTH2;
if (ud->size <= 3)
memcpy(m.data, ud->data, ud->size);
else {
if (utf8_put_item(ud->data, ud->size, &offset) != 0)
goto fail;
m.data[0] = (offset & 0xff);
m.data[1] = (offset >> 8) & 0xff;
m.data[2] = (offset >> 16);
}
*uc = m.uc;
return (UTF8_DONE);
fail: fail:
if (ud->width == 1) if (ud->width == 1)
return (utf8_big_space1.value); *uc = utf8_space1.uc;
return (utf8_big_space2.value); else
*uc = utf8_space2.uc;
return (UTF8_ERROR);
} }
/* Get UTF-8 from index into buffer. */ /* Get UTF-8 data from character. */
void void
utf8_get_big(u_int v, struct utf8_data *ud) utf8_to_data(utf8_char uc, struct utf8_data *ud)
{ {
union utf8_big_map m = { .value = v }; union utf8_map m = { .uc = uc };
struct utf8_big_item *bi; struct utf8_item *ui;
u_int o; u_int offset;
memset(ud, 0, sizeof *ud); memset(ud, 0, sizeof *ud);
ud->size = ud->have = (m.flags & UTF8_BIG_SIZE); ud->size = ud->have = (m.flags & UTF8_FLAG_SIZE);
if (m.flags & UTF8_BIG_WIDTH2) if (m.flags & UTF8_FLAG_WIDTH2)
ud->width = 2; ud->width = 2;
else else
ud->width = 1; ud->width = 1;
@ -183,24 +187,24 @@ utf8_get_big(u_int v, struct utf8_data *ud)
return; return;
} }
o = ((u_int)m.data[2] << 16)|((u_int)m.data[1] << 8)|m.data[0]; offset = ((u_int)m.data[2] << 16)|((u_int)m.data[1] << 8)|m.data[0];
if (o >= utf8_big_list_used) if (offset >= utf8_list_used)
memset(ud->data, ' ', ud->size); memset(ud->data, ' ', ud->size);
else { else {
bi = &utf8_big_list[o]; ui = &utf8_list[offset];
memcpy(ud->data, bi->data, ud->size); memcpy(ud->data, ui->data, ud->size);
} }
} }
/* Get big value for UTF-8 single character. */ /* Get UTF-8 character from a single ASCII character. */
u_int u_int
utf8_set_big(char c, u_int width) utf8_build_one(char c, u_int width)
{ {
union utf8_big_map m = { .flags = 1, .data[0] = c }; union utf8_map m = { .flags = 1, .data[0] = c };
if (width == 2) if (width == 2)
m.flags |= UTF8_BIG_WIDTH2; m.flags |= UTF8_FLAG_WIDTH2;
return (m.value); return (m.uc);
} }
/* Set a single character. */ /* Set a single character. */
@ -225,6 +229,20 @@ utf8_copy(struct utf8_data *to, const struct utf8_data *from)
to->data[i] = '\0'; to->data[i] = '\0';
} }
/* Get width of Unicode character. */
static int
utf8_width(wchar_t wc)
{
int width;
width = wcwidth(wc);
if (width < 0 || width > 0xff) {
log_debug("Unicode %04lx, wcwidth() %d", (long)wc, width);
return (-1);
}
return (width);
}
/* /*
* Open UTF-8 sequence. * Open UTF-8 sequence.
* *
@ -279,20 +297,6 @@ utf8_append(struct utf8_data *ud, u_char ch)
return (UTF8_DONE); return (UTF8_DONE);
} }
/* Get width of Unicode character. */
static int
utf8_width(wchar_t wc)
{
int width;
width = wcwidth(wc);
if (width < 0 || width > 0xff) {
log_debug("Unicode %04lx, wcwidth() %d", (long)wc, width);
return (-1);
}
return (width);
}
/* Combine UTF-8 into Unicode. */ /* Combine UTF-8 into Unicode. */
enum utf8_state enum utf8_state
utf8_combine(const struct utf8_data *ud, wchar_t *wc) utf8_combine(const struct utf8_data *ud, wchar_t *wc)
@ -337,13 +341,10 @@ int
utf8_strvis(char *dst, const char *src, size_t len, int flag) utf8_strvis(char *dst, const char *src, size_t len, int flag)
{ {
struct utf8_data ud; struct utf8_data ud;
const char *start, *end; const char *start = dst, *end = src + len;
enum utf8_state more; enum utf8_state more;
size_t i; size_t i;
start = dst;
end = src + len;
while (src < end) { while (src < end) {
if ((more = utf8_open(&ud, *src)) == UTF8_MORE) { if ((more = utf8_open(&ud, *src)) == UTF8_MORE) {
while (++src < end && more == UTF8_MORE) while (++src < end && more == UTF8_MORE)
@ -369,7 +370,6 @@ utf8_strvis(char *dst, const char *src, size_t len, int flag)
dst = vis(dst, src[0], flag, '\0'); dst = vis(dst, src[0], flag, '\0');
src++; src++;
} }
*dst = '\0'; *dst = '\0';
return (dst - start); return (dst - start);
} }
@ -392,9 +392,9 @@ utf8_stravis(char **dst, const char *src, int flag)
int int
utf8_isvalid(const char *s) utf8_isvalid(const char *s)
{ {
struct utf8_data ud; struct utf8_data ud;
const char *end; const char *end;
enum utf8_state more; enum utf8_state more;
end = s + strlen(s); end = s + strlen(s);
while (s < end) { while (s < end) {
@ -420,15 +420,12 @@ utf8_isvalid(const char *s)
char * char *
utf8_sanitize(const char *src) utf8_sanitize(const char *src)
{ {
char *dst; char *dst = NULL;
size_t n; size_t n = 0;
enum utf8_state more; enum utf8_state more;
struct utf8_data ud; struct utf8_data ud;
u_int i; u_int i;
dst = NULL;
n = 0;
while (*src != '\0') { while (*src != '\0') {
dst = xreallocarray(dst, n + 1, sizeof *dst); dst = xreallocarray(dst, n + 1, sizeof *dst);
if ((more = utf8_open(&ud, *src)) == UTF8_MORE) { if ((more = utf8_open(&ud, *src)) == UTF8_MORE) {
@ -449,7 +446,6 @@ utf8_sanitize(const char *src)
dst[n++] = '_'; dst[n++] = '_';
src++; src++;
} }
dst = xreallocarray(dst, n + 1, sizeof *dst); dst = xreallocarray(dst, n + 1, sizeof *dst);
dst[n] = '\0'; dst[n] = '\0';
return (dst); return (dst);
@ -471,9 +467,8 @@ u_int
utf8_strwidth(const struct utf8_data *s, ssize_t n) utf8_strwidth(const struct utf8_data *s, ssize_t n)
{ {
ssize_t i; ssize_t i;
u_int width; u_int width = 0;
width = 0;
for (i = 0; s[i].size != 0; i++) { for (i = 0; s[i].size != 0; i++) {
if (n != -1 && n == i) if (n != -1 && n == i)
break; break;
@ -489,13 +484,10 @@ utf8_strwidth(const struct utf8_data *s, ssize_t n)
struct utf8_data * struct utf8_data *
utf8_fromcstr(const char *src) utf8_fromcstr(const char *src)
{ {
struct utf8_data *dst; struct utf8_data *dst = NULL;
size_t n; size_t n = 0;
enum utf8_state more; enum utf8_state more;
dst = NULL;
n = 0;
while (*src != '\0') { while (*src != '\0') {
dst = xreallocarray(dst, n + 1, sizeof *dst); dst = xreallocarray(dst, n + 1, sizeof *dst);
if ((more = utf8_open(&dst[n], *src)) == UTF8_MORE) { if ((more = utf8_open(&dst[n], *src)) == UTF8_MORE) {
@ -511,7 +503,6 @@ utf8_fromcstr(const char *src)
n++; n++;
src++; src++;
} }
dst = xreallocarray(dst, n + 1, sizeof *dst); dst = xreallocarray(dst, n + 1, sizeof *dst);
dst[n].size = 0; dst[n].size = 0;
return (dst); return (dst);
@ -521,18 +512,14 @@ utf8_fromcstr(const char *src)
char * char *
utf8_tocstr(struct utf8_data *src) utf8_tocstr(struct utf8_data *src)
{ {
char *dst; char *dst = NULL;
size_t n; size_t n = 0;
dst = NULL;
n = 0;
for(; src->size != 0; src++) { for(; src->size != 0; src++) {
dst = xreallocarray(dst, n + src->size, 1); dst = xreallocarray(dst, n + src->size, 1);
memcpy(dst + n, src->data, src->size); memcpy(dst + n, src->data, src->size);
n += src->size; n += src->size;
} }
dst = xreallocarray(dst, n + 1, 1); dst = xreallocarray(dst, n + 1, 1);
dst[n] = '\0'; dst[n] = '\0';
return (dst); return (dst);
@ -570,7 +557,7 @@ utf8_padcstr(const char *s, u_int width)
{ {
size_t slen; size_t slen;
char *out; char *out;
u_int n, i; u_int n, i;
n = utf8_cstrwidth(s); n = utf8_cstrwidth(s);
if (n >= width) if (n >= width)
@ -591,7 +578,7 @@ utf8_rpadcstr(const char *s, u_int width)
{ {
size_t slen; size_t slen;
char *out; char *out;
u_int n, i; u_int n, i;
n = utf8_cstrwidth(s); n = utf8_cstrwidth(s);
if (n >= width) if (n >= width)

2
window-copy.c
View File

@ -2571,7 +2571,7 @@ window_copy_cellstring(const struct grid_line *gl, u_int px, size_t *size,
return (&gce->data.data); return (&gce->data.data);
} }
utf8_get_big(gl->extddata[gce->offset].data, &ud); utf8_to_data(gl->extddata[gce->offset].data, &ud);
*size = ud.size; *size = ud.size;
*allocated = 1; *allocated = 1;