Initial UTF-8 support.

CHANGES

@@ -1,3 +1,8 @@
+09 September 2008
+
+* Initial UTF-8 support. A bit ugly and with a limit of 4096 UTF-8
+  characters per window.
+
 08 September 2008
 
 * 256 colour support. tmux attempts to autodetect the terminal by looking
@@ -655,4 +660,4 @@
   (including mutt, emacs). No status bar yet and no key remapping or other
   customisation.
 
-$Id: CHANGES,v 1.158 2008-09-08 17:40:50 nicm Exp $
+$Id: CHANGES,v 1.159 2008-09-09 22:16:36 nicm Exp $

GNUmakefile

@@ -1,4 +1,4 @@
-# $Id: GNUmakefile,v 1.38 2008-08-28 17:45:24 nicm Exp $
+# $Id: GNUmakefile,v 1.39 2008-09-09 22:16:36 nicm Exp $
 
 .PHONY: clean
 
@@ -31,7 +31,7 @@ SRCS= tmux.c server.c server-msg.c server-fn.c buffer.c buffer-poll.c status.c \
       cmd-list-commands.c cmd-move-window.c cmd-select-prompt.c \
       cmd-respawn-window.c \
       window-scroll.c window-more.c window-copy.c options.c paste.c \
-      tty.c tty-keys.c tty-write.c screen-write.c screen-redraw.c
+      tty.c tty-keys.c tty-write.c screen-write.c screen-redraw.c utf8.c
 
 CC?= gcc
 INCDIRS+= -I. -I-

Makefile

@@ -1,4 +1,4 @@
-# $Id: Makefile,v 1.72 2008-08-07 05:15:21 nicm Exp $
+# $Id: Makefile,v 1.73 2008-09-09 22:16:36 nicm Exp $
 
 .SUFFIXES: .c .o .y .h
 .PHONY: clean update-index.html upload-index.html
@@ -35,7 +35,7 @@ SRCS= tmux.c server.c server-msg.c server-fn.c buffer.c buffer-poll.c status.c \
       cmd-list-commands.c cmd-move-window.c cmd-select-prompt.c \
       cmd-respawn-window.c \
       window-scroll.c window-more.c window-copy.c options.c paste.c \
-      tty.c tty-keys.c tty-write.c screen-write.c screen-redraw.c
+      tty.c tty-keys.c tty-write.c screen-write.c screen-redraw.c utf8.c
 
 CC?= cc
 INCDIRS+= -I. -I- -I/usr/local/include

TODO

@@ -41,6 +41,46 @@
 
 -- For 0.5 --------------------------------------------------------------------
 
+- FINISH UTF8: fix copy and paste
+- SPLIT u_short attr into attr,flags?
+- maybe rethink backend data structure?
+  	- utf8 can be 1-4 bytes
+	- most common is 1 bytes
+	- there can be double-width characters which take n bytes but 2 columns on screen
+	- they are not only drawn as two characters, they also require two backspaces to remove
+- three operations:
+  	- simultaneously update screen and ttys
+	- redraw screen or section of screen to ttys
+	- write to ttys without updating screen
+
+---
+split off grid manip:
+      16-bit characters
+      8-bit flags
+      8-bit attributes
+      8-bit fg colour
+      8-bit bg colour
+
+struct grid_data {
+	struct grid_cell **cells;
+	u_int	sx;
+	u_int	sy;
+};
+struct grid_cell {
+       u_short	data;
+       u_char	attr;
+       u_char	flags;
+       u_char	fg;
+       u_char	bg;
+}
+---
+
+Would it be better to just expand char to 16-bits and use it as an index only
+for >2-byte characters? or - better - don't support entire UTF range? only the BMP?
+this would get rid of UTF table and limits, but still leave double-width character annoyances
+also would double memory usage
+----
+
 21:09 < merdely> NicM: if I run 'tmux attach -t main' and there is no tmux session named main, start a new one.
 - commands:	save-buffer -b number filename
 		load-buffer -b number filename

buffer-poll.c

@@ -1,4 +1,4 @@
-/* $Id: buffer-poll.c,v 1.9 2008-08-28 17:45:25 nicm Exp $ */
+/* $Id: buffer-poll.c,v 1.10 2008-09-09 22:16:36 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicm@users.sourceforge.net>
@@ -40,9 +40,11 @@ buffer_poll(struct pollfd *pfd, struct buffer *in, struct buffer *out)
 {
 	ssize_t	n;
 
+#if 0
 	log_debug("buffer_poll (%ld): fd=%d, revents=%d; out=%zu in=%zu",
 	    (long) getpid(),
 	    pfd->fd, pfd->revents, BUFFER_USED(out), BUFFER_USED(in));
+#endif
 
 #ifndef BROKEN_POLL
 	if (pfd->revents & (POLLERR|POLLNVAL|POLLHUP))
@@ -51,7 +53,9 @@ buffer_poll(struct pollfd *pfd, struct buffer *in, struct buffer *out)
 	if (pfd->revents & POLLIN) {
 		buffer_ensure(in, BUFSIZ);
 		n = read(pfd->fd, BUFFER_IN(in), BUFFER_FREE(in));
+#if 0
 		log_debug("buffer_poll: fd=%d, read=%zd", pfd->fd, n);
+#endif
 		if (n == 0)
 			return (-1);
 		if (n == -1) {
@@ -62,7 +66,9 @@ buffer_poll(struct pollfd *pfd, struct buffer *in, struct buffer *out)
 	}
 	if (BUFFER_USED(out) > 0 && pfd->revents & POLLOUT) {
 		n = write(pfd->fd, BUFFER_OUT(out), BUFFER_USED(out));
+#if 0
 		log_debug("buffer_poll: fd=%d, write=%zd", pfd->fd, n);
+#endif
 		if (n == -1) {
 			if (errno != EINTR && errno != EAGAIN)
 				return (-1);

client.c

@@ -1,4 +1,4 @@
-/* $Id: client.c,v 1.34 2008-07-01 19:47:02 nicm Exp $ */
+/* $Id: client.c,v 1.35 2008-09-09 22:16:36 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicm@users.sourceforge.net>
@@ -34,7 +34,8 @@
 void	client_handle_winch(struct client_ctx *);
 
 int
-client_init(const char *path, struct client_ctx *cctx, int start_server)
+client_init(
+    const char *path, struct client_ctx *cctx, int start_server, int flags)
 {
 	struct sockaddr_un		sa;
 	struct stat			sb;
@@ -94,6 +95,7 @@ retry:
 		if (ioctl(STDIN_FILENO, TIOCGWINSZ, &ws) == -1)
 			fatal("ioctl(TIOCGWINSZ)");
 		data.version = PROTOCOL_VERSION;
+		data.flags = flags;
 		data.sx = ws.ws_col;
 		data.sy = ws.ws_row;
 		*data.tty = '\0';

cmd-list-windows.c

@@ -1,4 +1,4 @@
-/* $Id: cmd-list-windows.c,v 1.22 2008-09-08 17:40:50 nicm Exp $ */
+/* $Id: cmd-list-windows.c,v 1.23 2008-09-09 22:16:36 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicm@users.sourceforge.net>
@@ -72,10 +72,12 @@ cmd_list_windows_exec(struct cmd *self, struct cmd_ctx *ctx)
 		else
 			name = "";
 		ctx->print(ctx,
-		    "%d: %s \"%s\" (%s) [%ux%u] [history %u/%u, %llu bytes]",
+		    "%d: %s \"%s\" (%s) [%ux%u] [history %u/%u, %llu bytes] "
+		    "[UTF8 table %u/%u]",
 		    wl->idx, w->name, w->base.title, name,
 		    screen_size_x(&w->base), screen_size_y(&w->base),
-		    w->base.hsize, w->base.hlimit, size);
+		    w->base.hsize, w->base.hlimit, size,
+		    ARRAY_LENGTH(&w->base.utf8_table.array), UTF8_LIMIT);
 	}
 
 	if (ctx->cmdclient != NULL)

input.c

@@ -1,4 +1,4 @@
-/* $Id: input.c,v 1.57 2008-09-09 17:35:04 nicm Exp $ */
+/* $Id: input.c,v 1.58 2008-09-09 22:16:36 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicm@users.sourceforge.net>
@@ -56,6 +56,7 @@ void	 input_state_sequence_next(u_char, struct input_ctx *);
 void	 input_state_sequence_intermediate(u_char, struct input_ctx *);
 void	 input_state_string_next(u_char, struct input_ctx *);
 void	 input_state_string_escape(u_char, struct input_ctx *);
+void	 input_state_utf8(u_char, struct input_ctx *);
 
 void	 input_handle_character(u_char, struct input_ctx *);
 void	 input_handle_c0_control(u_char, struct input_ctx *);
@@ -247,6 +248,7 @@ input_state_first(u_char ch, struct input_ctx *ictx)
 		return;
 	}
 
+#if 0
   	if (INPUT_C1CONTROL(ch)) {
 		ch -= 0x40;
 		if (ch == '[')
@@ -257,6 +259,10 @@ input_state_first(u_char ch, struct input_ctx *ictx)
 			input_handle_c1_control(ch, ictx);
 		return;
 	}
+#endif
+
+	if (INPUT_DELETE(ch))
+		return;
 
 	input_handle_character(ch, ictx);
 }
@@ -481,11 +487,54 @@ input_state_string_escape(u_char ch, struct input_ctx *ictx)
 	input_state_string_next(ch, ictx);
 }
 
+void
+input_state_utf8(u_char ch, struct input_ctx *ictx)
+{
+	log_debug2("-- un %zu: %hhu (%c)", ictx->off, ch, ch);
+
+	ictx->utf8_buf.data[ictx->utf8_off++] = ch;
+	if (--ictx->utf8_len != 0)
+		return;
+	input_state(ictx, input_state_first);
+
+	screen_write_put_utf8(&ictx->ctx, &ictx->utf8_buf);
+}
+
 void
 input_handle_character(u_char ch, struct input_ctx *ictx)
 {
 	log_debug2("-- ch %zu: %hhu (%c)", ictx->off, ch, ch);
 
+	if (ch > 0x7f) {
+		/* 
+		 * UTF8 sequence.
+		 *
+		 * 11000010-11011111 C2-DF start of 2-byte sequence
+		 * 11100000-11101111 E0-EF start of 3-byte sequence
+		 * 11110000-11110100 F0-F4 start of 4-byte sequence
+		 */
+		memset(&ictx->utf8_buf.data, 0xff, sizeof &ictx->utf8_buf.data);
+		ictx->utf8_buf.data[0] = ch;
+		ictx->utf8_off = 1;
+			    
+		if (ch >= 0xc2 && ch <= 0xdf) {
+			log_debug2(":: u2");
+			input_state(ictx, input_state_utf8);
+			ictx->utf8_len = 1;
+		}
+		if (ch >= 0xe0 && ch <= 0xef) {
+			log_debug2(":: u3");
+			input_state(ictx, input_state_utf8);
+			ictx->utf8_len = 2;
+		}
+		if (ch >= 0xf0 && ch <= 0xf4) {	
+			log_debug2(":: u4");
+			input_state(ictx, input_state_utf8);
+			ictx->utf8_len = 3;
+		}
+		return;
+	}
+
 	screen_write_put_character(&ictx->ctx, ch);
 }
 

screen-display.c

@@ -1,4 +1,4 @@
-/* $Id: screen-display.c,v 1.20 2008-09-08 22:03:54 nicm Exp $ */
+/* $Id: screen-display.c,v 1.21 2008-09-09 22:16:36 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicm@users.sourceforge.net>
@@ -22,6 +22,15 @@
 
 #include "tmux.h"
 
+/* Get a cell. */
+void
+screen_display_get_cell(struct screen *s,
+    u_int px, u_int py, u_char *data, u_short *attr, u_char *fg, u_char *bg)
+{
+	screen_get_cell(
+	    s, screen_x(s, px), screen_y(s, py), data, attr, fg, bg);
+}
+
 /* Set a cell. */
 void
 screen_display_set_cell(struct screen *s,

screen-write.c

@@ -1,4 +1,4 @@
-/* $Id: screen-write.c,v 1.12 2008-09-08 22:03:54 nicm Exp $ */
+/* $Id: screen-write.c,v 1.13 2008-09-09 22:16:36 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicm@users.sourceforge.net>
@@ -99,6 +99,7 @@ void
 screen_write_put_character(struct screen_write_ctx *ctx, u_char ch)
 {
 	struct screen	*s = ctx->s;
+	u_short		 attr;
 
 	if (s->cx == screen_size_x(s)) {
 		s->cx = 0;
@@ -109,14 +110,87 @@ screen_write_put_character(struct screen_write_ctx *ctx, u_char ch)
 		SCREEN_DEBUG(s);
 		return;
 	}
+	attr = s->attr & ~(ATTR_UTF8|ATTR_PAD);
 
-	screen_display_set_cell(s, s->cx, s->cy, ch, s->attr, s->fg, s->bg);
+	screen_display_set_cell(s, s->cx, s->cy, ch, attr, s->fg, s->bg);
 	s->cx++;
 
 	if (ctx->write != NULL)
 		ctx->write(ctx->data, TTY_CHARACTER, ch);
 }
 
+/* Put a UTF8 character. */
+void
+screen_write_put_utf8(struct screen_write_ctx *ctx, struct utf8_data *udat)
+{
+	struct screen	*s = ctx->s;
+	u_char		 ch, ch2, fg, bg;
+	u_short		 attr, attr2;
+	int		 idx, wide;
+	u_int		 n;
+
+	wide = 0;
+	if (udat->data[2] == 0xff)
+		n = ((udat->data[0] & 0x1f)<<6) + (udat->data[1] & 0x3f);
+	else if (udat->data[3] == 0xff) {
+		n = ((udat->data[0] & 0x0f)<<12) +
+		    ((udat->data[1] & 0x3f)<<6) + (udat->data[2] & 0x3f);
+	} else
+		n = 0;
+	if ((n >= 0x1100 && n <= 0x115f) || n == 0x2329 || n == 0x232a ||
+	    (n >= 0x2e80 && n <= 0xa4cf && n != 0x303f) ||
+	    (n >= 0xac00 && n <= 0xd7a3) || (n >= 0xf900 && n <= 0xfaff) ||
+	    (n >= 0xfe10 && n <= 0xfe19) || (n >= 0xfe30 && n <= 0xfe6f) ||
+	    (n >= 0xff00 && n <= 0xff60) || (n >= 0xffe0 && n <= 0xffe6) ||
+	    (n >= 0x20000 && n <= 0x2fffd) || (n >= 0x30000 && n <= 0x3fffd))
+		wide = 1;
+
+	if (s->cx >= screen_size_x(s) - wide) {
+		s->cx = 0;
+		if (ctx->write != NULL)
+			ctx->write(ctx->data, TTY_CHARACTER, '\r');
+		screen_write_cursor_down_scroll(ctx);
+	} else if (!screen_in_x(s, s->cx) || !screen_in_y(s, s->cy)) {
+		SCREEN_DEBUG(s);
+		return;
+	}
+	attr = s->attr & ~(ATTR_UTF8|ATTR_PAD);
+
+	if ((idx = utf8_add(&s->utf8_table, udat)) == -1)
+		ch = '_';
+	else {
+		utf8_pack(idx, &ch, &attr);
+		attr |= ATTR_UTF8;
+	}
+
+	/* Remove padding before and after, if any. */
+	screen_display_get_cell(s, s->cx, s->cy, &ch2, &attr2, &fg, &bg);
+	if (s->cx > 0 && (attr2 & ATTR_PAD))
+		screen_display_set_cell(s, s->cx - 1, s->cy, ' ', 0, 8, 8);
+	if (s->cx < screen_last_x(s) && (attr2 & ATTR_UTF8)) {
+		screen_display_get_cell(
+		    s, s->cx + 1, s->cy, &ch2, &attr2, &fg, &bg);
+		if (s->cx > 0 && (attr2 & ATTR_PAD)) {
+			screen_display_set_cell(
+			    s, s->cx + 1, s->cy, ' ', 0, 8, 8);
+		}
+	}
+
+	screen_display_set_cell(s, s->cx, s->cy, ch, attr, s->fg, s->bg);
+	s->cx++;
+
+	if (wide) {
+		screen_display_set_cell(s, s->cx, s->cy, ' ', ATTR_PAD, 8, 8);
+		s->cx++;
+	}
+
+	if (ctx->write != NULL) {
+		ctx->write(ctx->data, TTY_ATTRIBUTES, attr, s->fg, s->bg);
+		ctx->write(ctx->data, TTY_CHARACTER, ch);
+		ctx->write(ctx->data, TTY_ATTRIBUTES, s->attr, s->fg, s->bg);
+	}
+}
+
 /* Put a string right-justified. */
 size_t printflike2
 screen_write_put_string_rjust(

screen.c

@@ -1,4 +1,4 @@
-/* $Id: screen.c,v 1.68 2008-09-08 22:03:54 nicm Exp $ */
+/* $Id: screen.c,v 1.69 2008-09-09 22:16:36 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicm@users.sourceforge.net>
@@ -155,6 +155,8 @@ screen_create(struct screen *s, u_int dx, u_int dy, u_int hlimit)
 	s->grid_size = xmalloc(dy * (sizeof *s->grid_size));
 	screen_make_lines(s, 0, dy);
 
+	utf8_init(&s->utf8_table, UTF8_LIMIT);
+
 	screen_clear_selection(s);
 }
 
@@ -348,6 +350,7 @@ screen_set_cell(struct screen *s,
 void
 screen_destroy(struct screen *s)
 {
+	utf8_free(&s->utf8_table);
 	xfree(s->title);
 	screen_free_lines(s, 0, s->dy + s->hsize);
 	xfree(s->grid_data);

server-msg.c

@@ -1,4 +1,4 @@
-/* $Id: server-msg.c,v 1.49 2008-07-01 19:47:02 nicm Exp $ */
+/* $Id: server-msg.c,v 1.50 2008-09-09 22:16:37 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicm@users.sourceforge.net>
@@ -187,6 +187,8 @@ server_msg_fn_identify(struct hdr *hdr, struct client *c)
 
 	data.tty[(sizeof data.tty) - 1] = '\0';
 	tty_init(&c->tty, data.tty, xstrdup(term));
+	if (data.flags & IDENTIFY_UTF8)
+		c->tty.flags |= TTY_UTF8;
 	xfree(term);
 
 	c->flags |= CLIENT_TERMINAL;

tmux.c

@@ -1,4 +1,4 @@
-/* $Id: tmux.c,v 1.74 2008-08-28 17:45:27 nicm Exp $ */
+/* $Id: tmux.c,v 1.75 2008-09-09 22:16:37 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicm@users.sourceforge.net>
@@ -175,10 +175,11 @@ main(int argc, char **argv)
 	struct passwd		*pw;
 	char			*path, *cause, *home;
 	char			 rpath[MAXPATHLEN];
-	int	 		 n, opt;
+	int	 		 n, opt, flags;
 
+	flags = 0;
 	path = NULL;
-        while ((opt = getopt(argc, argv, "f:qS:Vv")) != EOF) {
+        while ((opt = getopt(argc, argv, "f:qS:uVv")) != EOF) {
                 switch (opt) {
 		case 'f':
 			cfg_file = xstrdup(optarg);
@@ -189,6 +190,9 @@ main(int argc, char **argv)
 		case 'q':
 			be_quiet = 1;
 			break;
+		case 'u':
+			flags |= IDENTIFY_UTF8;
+			break;
 		case 'v':
 			debug_level++;
 			break;
@@ -286,7 +290,8 @@ main(int argc, char **argv)
 
 	memset(&cctx, 0, sizeof cctx);
 	client_fill_session(&data);
-	if (client_init(rpath, &cctx, cmd->entry->flags & CMD_STARTSERVER) != 0)
+	if (client_init(
+	    rpath, &cctx, cmd->entry->flags & CMD_STARTSERVER, flags) != 0)
 		exit(1);
 	b = buffer_create(BUFSIZ);
 	cmd_send(cmd, b);

tmux.h

@@ -1,4 +1,4 @@
-/* $Id: tmux.h,v 1.185 2008-09-08 22:18:03 nicm Exp $ */
+/* $Id: tmux.h,v 1.186 2008-09-09 22:16:37 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicm@users.sourceforge.net>
@@ -19,7 +19,7 @@
 #ifndef TMUX_H
 #define TMUX_H
 
-#define PROTOCOL_VERSION -1
+#define PROTOCOL_VERSION -2
 
 /* Shut up gcc warnings about empty if bodies. */
 #define RB_AUGMENT(x) do {} while (0)
@@ -375,6 +375,9 @@ struct msg_identify_data {
 	char		tty[TTY_NAME_MAX];
 	int	        version;
 
+#define IDENTIFY_UTF8 0x1
+	int		flags;
+
 	u_int		sx;
 	u_int		sy;
 
@@ -386,6 +389,17 @@ struct msg_resize_data {
 	u_int		sy;
 };
 
+/* UTF8 data. */
+struct utf8_data {
+	u_char		data[4];
+};
+
+struct utf8_table {
+	u_int		limit;
+	ARRAY_DECL(, struct utf8_data) array;
+};
+#define UTF8_LIMIT ((1<<11) - 1)
+
 /* Attributes. */
 #define ATTR_BRIGHT 0x1
 #define ATTR_DIM 0x2
@@ -395,9 +409,18 @@ struct msg_resize_data {
 #define ATTR_HIDDEN 0x20
 #define ATTR_ITALICS 0x40
 #define ATTR_CHARSET 0x80	/* alternative character set */
+
 #define ATTR_FG256 0x100
 #define ATTR_BG256 0x200
 
+#define ATTR_UTF8 0x400
+#define ATTR_PAD 0x800
+
+#define ATTR_UTF8b8 0x8000
+#define ATTR_UTF8b9 0x4000
+#define ATTR_UTF8b10 0x2000
+#define ATTR_UTF8b11 0x1000
+
 /* Modes. */
 #define MODE_CURSOR 0x1
 #define MODE_INSERT 0x2
@@ -442,6 +465,8 @@ struct screen {
 	u_char		 fg;
 	u_char		 bg;
 
+	struct utf8_table utf8_table;
+
 	u_int		 saved_cx;
 	u_int		 saved_cy;
 	u_short		 saved_attr;
@@ -541,6 +566,10 @@ struct input_ctx {
 #define STRING_NAME 1
 #define STRING_IGNORE 2
 
+	struct utf8_data utf8_buf;
+	u_int		 utf8_len;
+	u_int		 utf8_off;
+
 	void 		*(*state)(u_char, struct input_ctx *);
 
 	u_char		 private;
@@ -692,6 +721,8 @@ struct tty {
 	struct buffer	*in;
 	struct buffer	*out;
 
+	int		 log_fd;
+
 	struct termios   tio;
 
 	u_short		 attr;
@@ -703,6 +734,7 @@ struct tty {
 #define TTY_NOCURSOR 0x1
 #define TTY_FREEZE 0x2
 #define TTY_ESCAPE 0x4
+#define TTY_UTF8 0x8
 	int		 flags;
 
 	struct timeval	 key_timer;
@@ -1066,7 +1098,7 @@ void	cmd_buffer_free(struct cmd *);
 void	cmd_buffer_print(struct cmd *, char *, size_t);
 
 /* client.c */
-int	 client_init(const char *, struct client_ctx *, int);
+int	 client_init(const char *, struct client_ctx *, int, int);
 int	 client_flush(struct client_ctx *);
 int	 client_main(struct client_ctx *);
 
@@ -1140,6 +1172,8 @@ void	 input_parse(struct window *);
 void	 input_key(struct window *, int);
 
 /* screen-display.c */
+void	 screen_display_get_cell(struct screen *,
+    	     u_int, u_int, u_char *, u_short *, u_char *, u_char *);
 void	 screen_display_set_cell(
 	     struct screen *, u_int, u_int, u_char, u_short, u_char, u_char);
 void	 screen_display_make_lines(struct screen *, u_int, u_int);
@@ -1168,6 +1202,7 @@ void	 screen_write_start(struct screen_write_ctx *,
 void	 screen_write_stop(struct screen_write_ctx *);
 void	 screen_write_set_title(struct screen_write_ctx *, char *);
 void	 screen_write_put_character(struct screen_write_ctx *, u_char);
+void	 screen_write_put_utf8(struct screen_write_ctx *, struct utf8_data *);
 size_t printflike2 screen_write_put_string_rjust(
 	     struct screen_write_ctx *, const char *, ...);
 void printflike2 screen_write_put_string(
@@ -1290,6 +1325,15 @@ int		 session_previous(struct session *);
 int		 session_select(struct session *, int);
 int		 session_last(struct session *);
 
+/* utf8.c */
+void	utf8_pack(int, u_char *, u_short *);
+int	utf8_unpack(u_char, u_short);
+void	utf8_init(struct utf8_table *, int);
+void	utf8_free(struct utf8_table *);
+struct utf8_data *utf8_lookup(struct utf8_table *, int);
+int	utf8_search(struct utf8_table *, struct utf8_data *);
+int	utf8_add(struct utf8_table *, struct utf8_data *);
+
 /* buffer.c */
 struct buffer 	*buffer_create(size_t);
 void		 buffer_destroy(struct buffer *);

tty.c

@@ -1,4 +1,4 @@
-/* $Id: tty.c,v 1.40 2008-09-08 22:03:56 nicm Exp $ */
+/* $Id: tty.c,v 1.41 2008-09-09 22:16:37 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicm@users.sourceforge.net>
@@ -54,6 +54,7 @@ tty_init(struct tty *tty, char *path, char *term)
 		tty->termname = xstrdup("unknown");
 	else
 		tty->termname = xstrdup(term);
+	tty->flags = 0;
 }
 
 int
@@ -73,18 +74,23 @@ tty_open(struct tty *tty, char **cause)
 	if (fcntl(tty->fd, F_SETFL, mode|O_NONBLOCK) == -1)
 		fatal("fcntl");
 
+	if (debug_level > 3) {
+		tty->log_fd = open("tmux.out", O_WRONLY|O_CREAT|O_TRUNC, 0644);
+	} else
+		tty->log_fd = -1;
+
 	if ((tty->term = tty_find_term(tty->termname, tty->fd, cause)) == NULL)
 		goto error;
 
 	tty->in = buffer_create(BUFSIZ);
 	tty->out = buffer_create(BUFSIZ);
 
+	tty->flags &= TTY_UTF8;
+
 	tty->attr = 0;
 	tty->fg = 8;
 	tty->bg = 8;
 
-	tty->flags = 0;
-
 	if (tcgetattr(tty->fd, &tty->tio) != 0)
 		fatal("tcgetattr failed");
 	memcpy(&tio, &tty->tio, sizeof tio);
@@ -140,6 +146,11 @@ tty_close(struct tty *tty)
 	if (tty->fd == -1)
 		return;
 
+	if (tty->log_fd != -1) {
+		close(tty->log_fd);
+		tty->log_fd = -1;
+	}
+
 	/*
 	 * Skip any writing if the fd is invalid. Things like ssh -t can
 	 * easily leave us with a dead tty.
@@ -384,6 +395,9 @@ tty_puts(struct tty *tty, const char *s)
 
 	t = tty_strip(s);
 	buffer_write(tty->out, t, strlen(t));
+
+	if (tty->log_fd != -1)
+		write(tty->log_fd, t, strlen(t));
 }
 
 void
@@ -392,6 +406,9 @@ tty_putc(struct tty *tty, char ch)
 	if (tty->attr & ATTR_CHARSET)
 		ch = tty_get_acs(tty, ch);
 	buffer_write8(tty->out, ch);
+
+	if (tty->log_fd != -1)
+		write(tty->log_fd, &ch, 1);
 }
 
 void
@@ -410,6 +427,7 @@ tty_set_title(struct tty *tty, const char *title)
 void
 tty_vwrite(struct tty *tty, struct screen *s, int cmd, va_list ap)
 {
+	struct utf8_data	*udat;
 	char			 ch;
 	u_int			 i, ua, ub, uc;
 
@@ -423,6 +441,33 @@ tty_vwrite(struct tty *tty, struct screen *s, int cmd, va_list ap)
 	switch (cmd) {
 	case TTY_CHARACTER:
 		ch = va_arg(ap, int);
+		
+		if (tty->attr & ATTR_PAD)
+			break;
+
+		if ((tty->attr & ATTR_UTF8) && (tty->flags & TTY_UTF8)) {
+			udat = utf8_lookup(
+			    &s->utf8_table, utf8_unpack(ch, tty->attr));
+			if (udat == NULL)
+				ch = '_';
+			else {
+				if (udat->data[0] == 0xff)
+					break;
+				tty_putc(tty, udat->data[0]);
+				if (udat->data[1] == 0xff)
+					break;
+				tty_putc(tty, udat->data[1]);
+				if (udat->data[2] == 0xff)
+					break;
+				tty_putc(tty, udat->data[2]);	
+				if (udat->data[3] == 0xff)
+					break;
+				tty_putc(tty, udat->data[3]);
+				break;
+			}
+		} else if (tty->attr & ATTR_UTF8)
+			ch = '_';
+
 		switch (ch) {
 		case '\n':	/* LF */
 			tty_putc(tty, '\n');

utf8.c

@@ -0,0 +1,114 @@
+/* $Id: utf8.c,v 1.1 2008-09-09 22:16:37 nicm Exp $ */
+
+/*
+ * Copyright (c) 2008 Nicholas Marriott <nicm@users.sourceforge.net>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF MIND, USE, DATA OR PROFITS, WHETHER
+ * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/types.h>
+
+#include <string.h>
+
+#include "tmux.h"
+
+/*
+ * UTF8 data structures. Just crappy array + linear search for now.
+ */
+
+/* Pack UTF8 index into attr, data. */
+void
+utf8_pack(int idx, u_char *data, u_short *attr)
+{
+	*data = idx & 0xff;
+
+	*attr &= ~(ATTR_UTF8b8|ATTR_UTF8b9);
+	if (idx & 0x100)
+		*attr |= ATTR_UTF8b8;
+	if (idx & 0x200)
+		*attr |= ATTR_UTF8b9;
+	if (idx & 0x400)
+		*attr |= ATTR_UTF8b10;
+	if (idx & 0x800)
+		*attr |= ATTR_UTF8b11;
+}
+
+/* Unpack UTF8 index from attr, data. */
+int
+utf8_unpack(u_char data, u_short attr)
+{
+	int	idx;
+
+	idx = data;
+	if (attr & ATTR_UTF8b8)
+		idx |= 0x100;
+	if (attr & ATTR_UTF8b9)
+		idx |= 0x200;
+	if (attr & ATTR_UTF8b10)
+		idx |= 0x400;
+	if (attr & ATTR_UTF8b11)
+		idx |= 0x800;
+
+	return (idx);
+}
+
+void
+utf8_init(struct utf8_table *utab, int limit)
+{
+	utab->limit = limit;
+	ARRAY_INIT(&utab->array);
+}
+
+void
+utf8_free(struct utf8_table *utab)
+{
+	ARRAY_FREE(&utab->array);
+}
+
+struct utf8_data *
+utf8_lookup(struct utf8_table *utab, int idx)
+{
+	if (idx < 0 || idx >= (int) ARRAY_LENGTH(&utab->array))
+		return (NULL);
+	return (&ARRAY_ITEM(&utab->array, idx));
+}
+
+int
+utf8_search(struct utf8_table *utab, struct utf8_data *udat)
+{
+	u_int	idx;
+
+	for (idx = 0; idx < ARRAY_LENGTH(&utab->array); idx++) {
+		if (memcmp(udat->data,
+		    ARRAY_ITEM(&utab->array, idx).data, sizeof udat->data) == 0)
+			return (idx);
+	}
+	return (-1);
+}
+
+int
+utf8_add(struct utf8_table *utab, struct utf8_data *udat)
+{
+	int	idx;
+
+	if (ARRAY_LENGTH(&utab->array) == utab->limit)
+		return (-1);
+
+	if ((idx = utf8_search(utab, udat)) != -1)
+		return (idx);
+
+	ARRAY_EXPAND(&utab->array, 1);
+	memcpy(
+	    &ARRAY_LAST(&utab->array), udat, sizeof ARRAY_LAST(&utab->array));
+	return (ARRAY_LENGTH(&utab->array) - 1);
+}