Fix Kitty key encoding: legacy fallback, F3, case, send-keys drop.

cmd-send-keys.c

@@ -132,10 +132,13 @@ cmd_send_keys_inject_string(struct cmdq_item *item, struct cmdq_item *after,
 	if (!literal) {
 		key = key_string_lookup_string(s);
 		if (key != KEYC_NONE && key != KEYC_UNKNOWN) {
+			/*
+			 * A recognized key name that the pane could not
+			 * encode is dropped, not typed as literal text.
+			 */
 			after = cmd_send_keys_inject_key(item, after, args,
 			    key);
-			if (after != NULL)
-				return (after);
+			return (after == NULL ? item : after);
 		}
 		literal = 1;
 	}

input-keys.c

@@ -485,6 +485,7 @@ input_key_vt10x(struct bufferevent *bev, key_code key)
 	struct utf8_data	 ud;
 	key_code		 onlykey;
 	const char		*p;
+	int			 meta;
 	static const char	*standard_map[2] = {
 		"1!9(0)=+;:'\",<.>/-8? 2",
 		"119900=+;;'',,..\x1f\x1f\x7f\x7f\0\0",
@@ -492,14 +493,20 @@ input_key_vt10x(struct bufferevent *bev, key_code key)
 
 	log_debug("%s: key in %llx", __func__, key);
 
-	if (key & KEYC_META)
-		input_key_write(__func__, bev, "\033", 1);
+	/*
+	 * Remember whether Meta was set and write its ESC prefix only once
+	 * the key below it is known to be encodable, so a key that turns
+	 * out to fail does not leave a stray ESC in the pane.
+	 */
+	meta = !!(key & KEYC_META);
 
 	/*
 	 * There's no way to report modifiers for unicode keys in standard mode
 	 * so lose the modifiers.
 	 */
 	if (KEYC_IS_UNICODE(key)) {
+		if (meta)
+			input_key_write(__func__, bev, "\033", 1);
 		utf8_to_data(key, &ud);
                 input_key_write(__func__, bev, ud.data, ud.size);
 		return (0);
@@ -536,6 +543,8 @@ input_key_vt10x(struct bufferevent *bev, key_code key)
 
 	log_debug("%s: key out %llx", __func__, key);
 
+	if (meta)
+		input_key_write(__func__, bev, "\033", 1);
 	ud.data[0] = key & 0x7f;
 	input_key_write(__func__, bev, &ud.data[0], 1);
 	return (0);

input-kitty.c

@@ -250,11 +250,23 @@ input_key_kitty(struct screen *s, struct bufferevent *bev, key_code key)
 	ikk = input_kitty_lookup(key);
 	if (ikk != NULL) {
 		if ((flags & KITTY_KEY_REPORT_ALL) == 0) {
-			if (ikk->key == C0_CR || ikk->key == C0_HT ||
-			    ikk->key == KEYC_BSPACE)
+			/*
+			 * Enter, Tab and Backspace stay in legacy form only
+			 * when unmodified, so a plain Enter still works after
+			 * a crash, but a modified one is disambiguated.
+			 */
+			if ((ikk->key == C0_CR || ikk->key == C0_HT ||
+			    ikk->key == KEYC_BSPACE) && modifiers == 0)
 				return (-1);
+
+			/*
+			 * A modified F3 in legacy form ("\033[1;mR") is
+			 * indistinguishable from a cursor position report,
+			 * so it must always be disambiguated too.
+			 */
 			if (ikk->final != 'u' &&
-			    (modifiers & (KEYC_SUPER|KEYC_HYPER)) == 0)
+			    (modifiers & (KEYC_SUPER|KEYC_HYPER)) == 0 &&
+			    (ikk->key != KEYC_F3 || modifiers == 0))
 				return (-1);
 		}
 		number = ikk->number;
@@ -296,8 +308,10 @@ input_key_kitty(struct screen *s, struct bufferevent *bev, key_code key)
 		number = onlykey;
 	else
 		return (-1);
-	if ((key & KEYC_SHIFT) && number >= 'A' && number <= 'Z')
+	if (number >= 'A' && number <= 'Z') {
 		number += 'a' - 'A';
+		key |= KEYC_SHIFT;
+	}
 
 	modifier = input_kitty_modifiers(key);
 	if (modifier == 1)

regress/kitty-keys.sh

@@ -104,5 +104,58 @@ $TMUX respawn-pane -k -t: \
 wait_for_output
 check_output 1b5b3f31751b5b3f30751b5b3f38751b5b3f3175
 
+: >"$OUT"
+# Modified Enter, Tab and Backspace are disambiguated even under flag 1;
+# only the unmodified keys stay in legacy form.
+$TMUX respawn-pane -k -t: \
+    "stty raw -echo; printf '\033[>1u'; dd bs=1 count=7 2>/dev/null | od -An -v -t x1 >'$OUT'; sleep 5"
+wait_for_mode 'Kitty 1'
+$TMUX send-keys -t: S-Enter
+wait_for_output
+check_output 1b5b31333b3275
+
+: >"$OUT"
+$TMUX respawn-pane -k -t: \
+    "stty raw -echo; printf '\033[>1u'; dd bs=1 count=1 2>/dev/null | od -An -v -t x1 >'$OUT'; sleep 5"
+wait_for_mode 'Kitty 1'
+$TMUX send-keys -t: Enter
+wait_for_output
+check_output 0d
+
+: >"$OUT"
+# An unshifted uppercase letter, as delivered by a legacy terminal or by
+# send-keys, is reported as the lowercase code point plus Shift.
+$TMUX respawn-pane -k -t: \
+    "stty raw -echo; printf '\033[>9u'; dd bs=1 count=7 2>/dev/null | od -An -v -t x1 >'$OUT'; sleep 5"
+wait_for_mode 'Kitty 9'
+$TMUX send-keys -t: A
+wait_for_output
+check_output 1b5b39373b3275
+
+: >"$OUT"
+# A modified F3 must use the unambiguous form even under flag 1 alone, since
+# its legacy form ("CSI 1;mR") cannot be told apart from a cursor position
+# report.
+$TMUX respawn-pane -k -t: \
+    "stty raw -echo; printf '\033[>1u'; dd bs=1 count=7 2>/dev/null | od -An -v -t x1 >'$OUT'; sleep 5"
+wait_for_mode 'Kitty 1'
+$TMUX send-keys -t: S-F3
+wait_for_output
+check_output 1b5b31333b327e
+
+# The following are not specific to the Kitty protocol, but were found by
+# the same testing: send-keys must drop a key name it cannot encode for the
+# pane's current mode rather than typing the name itself as text, and must
+# not leave a stray Escape behind while deciding that.
+: >"$OUT"
+$TMUX respawn-pane -k -t: \
+    "stty raw -echo; timeout 2 dd bs=1 2>/dev/null | od -An -v -t x1 >'$OUT'"
+sleep 0.3
+for key in C-Escape C-BSpace M-C-BSpace; do
+	$TMUX send-keys -t: "$key"
+done
+sleep 2.3
+check_output ''
+
 $TMUX kill-server 2>/dev/null
 exit 0