diff --git a/server-fn.c b/server-fn.c
index 8486b6c9..d3cdbd60 100644
--- a/server-fn.c
+++ b/server-fn.c
@@ -233,8 +233,8 @@ server_unlock(const char *s)
 	return (0);
 
 wrong:
-	password_backoff = server_activity;
 	password_failures++;
+	password_backoff = 0;
 
 	for (i = 0; i < ARRAY_LENGTH(&clients); i++) {
 		c = ARRAY_ITEM(&clients, i);
@@ -260,7 +260,8 @@ wrong:
 	}
 	failures = password_failures % tries;
 	if (failures > backoff) {
-		password_backoff += ((failures - backoff) * tries / 2);
+		password_backoff = 
+		    server_activity + ((failures - backoff) * tries / 2);
 		return (-2);
 	}
 	return (-1);
diff --git a/server.c b/server.c
index 82396b2e..9841816b 100644
--- a/server.c
+++ b/server.c
@@ -1210,11 +1210,12 @@ server_second_timers(void)
 		}
 	}
 
-	if (t > password_backoff) {
+	if (password_backoff != 0 && t >= password_backoff) {
 		for (i = 0; i < ARRAY_LENGTH(&clients); i++) {
 			if ((c = ARRAY_ITEM(&clients, i)) != NULL)
 				server_redraw_client(c);
 		}
+		password_backoff = 0;
 	}
 
 	/* Check for a minute having passed. */