Fix to drop SIXEL cells that a newer image completely covers. And regression test added.

image.c

@@ -371,9 +371,14 @@ image_span_free(struct image_span *span)
 	free(span);
 }
 
-/* Remove a range from selected spans on a line. */
+/*
+ * Remove a range from selected spans on a line - those of one placement if
+ * only is not NULL, otherwise those of the given input type (or all of them
+ * if that is -1).
+ */
 static void
-image_line_remove(struct image_line *line, u_int x, u_int width, int input)
+image_line_remove(struct image_line *line, u_int x, u_int width, int input,
+    struct image_placement *only)
 {
 	struct image_span	*span, *next;
 	u_int			 end, span_end, right;
@@ -384,6 +389,8 @@ image_line_remove(struct image_line *line, u_int x, u_int width, int input)
 	if (end < x)
 		end = UINT_MAX;
 	TAILQ_FOREACH_SAFE(span, &line->spans, line_entry, next) {
+		if (only != NULL && span->placement != only)
+			continue;
 		if (input != -1 && span->placement->input != (u_int)input)
 			continue;
 		span_end = span->x + span->sx;
@@ -439,7 +446,7 @@ image_grid_damage(struct grid *gd, u_int x, u_int y, u_int width,
 		height = gd->hsize + gd->sy - y;
 	for (row = y; row < y + height; row++)
 		image_line_remove(gd->linedata[row].images, x, width,
-		    IMAGE_INPUT_SIXEL);
+		    IMAGE_INPUT_SIXEL, NULL);
 	image_store_prune(gd->images);
 }
 
@@ -508,8 +515,8 @@ image_grid_move_cells(struct grid *gd, u_int dx, u_int px, u_int py,
 		moves[count].source_y = span->source_y;
 		count++;
 	}
-	image_line_remove(line, px, nx, -1);
-	image_line_remove(line, dx, nx, -1);
+	image_line_remove(line, px, nx, -1, NULL);
+	image_line_remove(line, dx, nx, -1, NULL);
 	for (size_t i = 0; i < count; i++)
 		image_span_add(line, moves[i].placement, moves[i].x,
 		    moves[i].sx, moves[i].source_x, moves[i].source_y);
@@ -1474,6 +1481,66 @@ image_cell_has_alpha(struct image *im, u_int x, u_int y)
 	return (0);
 }
 
+/*
+ * Return whether every visible pixel of one cell of an old image is also
+ * visible in the same cell of a new image, so that drawing the new image
+ * over the old one leaves nothing of the old cell showing. Only works if
+ * both images divide their cells into the same number of pixels.
+ */
+static int
+image_cell_covers(struct image *new, u_int nx, u_int ny, struct image *old,
+    u_int ox, u_int oy)
+{
+	u_int		 npx, npy, nsx, nsy, opx, opy, osx, osy, xx, yy;
+	const u_char	*np, *op;
+
+	if (new->canvas_width / new->sx != old->canvas_width / old->sx ||
+	    new->canvas_height / new->sy != old->canvas_height / old->sy)
+		return (0);
+	image_get_pixel_rect(new, nx, ny, 1, 1, &npx, &npy, &nsx, &nsy);
+	image_get_pixel_rect(old, ox, oy, 1, 1, &opx, &opy, &osx, &osy);
+
+	for (yy = 0; yy < osy; yy++) {
+		op = old->pixels + (size_t)(opy + yy) * old->stride +
+		    (size_t)opx * 4;
+		np = new->pixels + (size_t)(npy + yy) * new->stride +
+		    (size_t)npx * 4;
+		for (xx = 0; xx < osx; xx++) {
+			if (op[xx * 4 + 3] == 0)
+				continue;
+			if (yy >= nsy || xx >= nsx || np[xx * 4 + 3] != 255)
+				return (0);
+		}
+	}
+	return (1);
+}
+
+/*
+ * Remove the cells at one column of a line that a new SIXEL image completely
+ * covers. SIXEL is a single bitmap: drawing over another image replaces its
+ * pixels for good, so there is no point keeping the old cell to be drawn
+ * again underneath the new one on every repaint.
+ */
+static void
+image_line_cover(struct image_line *line, struct image_placement *placement,
+    u_int x, u_int source_x, u_int source_y)
+{
+	struct image_span	*span, *next;
+	struct image_placement	*old;
+
+	TAILQ_FOREACH_SAFE(span, &line->spans, line_entry, next) {
+		old = span->placement;
+		if (old == placement || old->input != IMAGE_INPUT_SIXEL)
+			continue;
+		if (x < span->x || x >= span->x + span->sx)
+			continue;
+		if (!image_cell_covers(placement->image, source_x, source_y,
+		    old->image, span->source_x + x - span->x, span->source_y))
+			continue;
+		image_line_remove(line, x, 1, -1, old);
+	}
+}
+
 /* Add spans for one row of a placement between two source columns. */
 static void
 image_extend_row(struct image_line *line, struct image_placement *placement,
@@ -1590,7 +1657,7 @@ image_write(struct screen_write_ctx *ctx, struct image *im, u_int bg,
 	struct image_placement	*placement;
 	struct image_line	*line;
 	u_int			 cx = s->cx, cy = s->cy;
-	u_int			 x, y, run, sx, sy, lines, origin_y = 0;
+	u_int			 x, y, i, run, sx, sy, lines, origin_y = 0;
 	u_int			 hist_origin_y, region_height, remaining, chunk;
 
 	sx = im->sx;
@@ -1678,6 +1745,12 @@ image_write(struct screen_write_ctx *ctx, struct image *im, u_int bg,
 				    origin_y + y))
 					break;
 			}
+			if (input == IMAGE_INPUT_SIXEL) {
+				for (i = 0; i < run; i++) {
+					image_line_cover(line, placement,
+					    cx + x + i, x + i, origin_y + y);
+				}
+			}
 			image_span_add(line, placement, cx + x, run, x,
 			    origin_y + y);
 		}

regress/image-sixel-overlap.sh

@@ -0,0 +1,79 @@
+#!/bin/sh
+
+# A SIXEL image drawn over an earlier one replaces the pixels underneath
+# for good, so an earlier image that is completely covered must not be kept
+# and drawn again on every repaint - an animation would otherwise pile up a
+# layer per frame.
+#
+# Draw two images over each other, then attach a client: it should be sent
+# no more than it would be for one image. An image that is only partly
+# covered must survive, so a second case draws a narrower image over a wider
+# one and expects both.
+
+PATH=/bin:/usr/bin
+TERM=screen
+LC_ALL=C.UTF-8
+export TERM LC_ALL
+
+[ -z "$TEST_TMUX" ] && TEST_TMUX=$(readlink -f ../tmux)
+TMUX="$TEST_TMUX -LtestA$$ -f/dev/null"
+TMUX2="$TEST_TMUX -LtestB$$ -f/dev/null"
+
+cleanup()
+{
+	$TMUX kill-server >/dev/null 2>&1
+	$TMUX2 kill-server >/dev/null 2>&1
+}
+fail()
+{
+	echo "$*" >&2
+	cleanup
+	exit 1
+}
+
+TMP=$(mktemp)
+trap "cleanup; rm -f $TMP" 0 1 15
+
+# Draw the given sixel commands in a new session, attach a sixel client and
+# set N to the number of SIXEL images the client was sent (other DCS
+# sequences, such as capability queries, do not count).
+count_images()
+{
+	cleanup
+	$TMUX new-session -d -s inner -x 40 -y 10 "printf '$1'; exec sh" ||
+		exit 1
+	sleep 0.3
+	[ "$($TMUX display-message -p '#{image_support}')" = 0 ] && exit 0
+	$TMUX set -as terminal-features ',*:sixel' || exit 1
+
+	$TMUX2 new-session -d -x 40 -y 10 || exit 1
+	OUTER=$($TMUX2 list-panes -F '#{pane_id}' | head -1)
+	[ -n "$OUTER" ] || fail "No outer pane."
+	$TMUX2 set -as terminal-features ',*:sixel@' ||
+	    fail "disable outer sixel failed"
+	: >"$TMP"
+	$TMUX2 pipe-pane -t "$OUTER" -O "cat >$TMP" || fail "pipe-pane failed"
+	$TMUX2 send-keys -t "$OUTER" -l "$TMUX attach -t inner" ||
+	    fail "send attach failed"
+	$TMUX2 send-keys -t "$OUTER" Enter || fail "send enter failed"
+	sleep 2
+	N=$(grep -aoE "$(printf '\033')P[0-9;]*q" "$TMP" | wc -l)
+}
+
+# One image on its own, as a baseline: a client is sent each image more than
+# once, as the terminal features are confirmed after attaching.
+count_images '\0337\033P9;1q#1!100~\033\\'
+SINGLE=$N
+[ "$SINGLE" -gt 0 ] || fail "sanity: client was sent no images"
+
+# The same footprint twice, blue then red.
+count_images '\0337\033P9;1q#1!100~\033\\\0338\033P9;1q#2!100~\033\\'
+[ "$N" -eq "$SINGLE" ] ||
+	fail "covered image was kept: client was sent $N images, expected $SINGLE"
+
+# A narrower image over a wider one leaves the wider one partly visible.
+count_images '\0337\033P9;1q#1!100~\033\\\0338\033P9;1q#2!40~\033\\'
+[ "$N" -gt "$SINGLE" ] ||
+	fail "partly covered image was lost: client was sent $N images, expected more than $SINGLE"
+
+exit 0