diff --git a/server-window.c b/server-window.c
index 020af041..cb819dfe 100644
--- a/server-window.c
+++ b/server-window.c
@@ -70,10 +70,12 @@ server_window_loop(void)
 			continue;
 
 		TAILQ_FOREACH(wp, &w->panes, entry) {
-			if (server_window_backoff(wp))
-				bufferevent_disable(wp->event, EV_READ);
-			else
-				bufferevent_enable(wp->event, EV_READ);
+			if (wp->fd != -1) {
+				if (server_window_backoff(wp))
+					bufferevent_disable(wp->event, EV_READ);
+				else
+					bufferevent_enable(wp->event, EV_READ);
+			}
 		}
 
 		for (j = 0; j < ARRAY_LENGTH(&sessions); j++) {