dovecot-2.1: master: If 10 service processes die within a second...

[dovecot at dovecot.org]

details:   http://hg.dovecot.org/dovecot-2.1/rev/300736775b89
changeset: 14014:300736775b89
user:      Timo Sirainen <tss at iki.fi>
date:      Fri Jan 27 23:22:22 2012 +0200
description:
master: If 10 service processes die within a second with no successes inbetween, throttle the service.

diffstat:

 src/master/service-monitor.c |  10 ++++++++++
 src/master/service.h         |   3 +++
 2 files changed, 13 insertions(+), 0 deletions(-)

diffs (51 lines):

diff -r c9b5ea1b9709 -r 300736775b89 src/master/service-monitor.c
--- a/src/master/service-monitor.c	Fri Jan 27 22:15:29 2012 +0200
+++ b/src/master/service-monitor.c	Fri Jan 27 23:22:22 2012 +0200
@@ -24,6 +24,7 @@
 #define SERVICE_DROP_WARN_INTERVAL_SECS 60
 #define SERVICE_DROP_TIMEOUT_MSECS (10*1000)
 #define MAX_DIE_WAIT_SECS 5
+#define SERVICE_MAX_EXIT_FAILURES_IN_SEC 10
 
 static void service_monitor_start_extra_avail(struct service *service);
 static void service_status_more(struct service_process *process,
@@ -524,10 +525,17 @@
 static bool
 service_process_failure(struct service_process *process, int status)
 {
+	struct service *service = process->service;
 	bool throttle;
 
 	service_process_log_status_error(process, status);
 	throttle = process->to_status != NULL;
+	if (service->exit_failure_last != ioloop_time) {
+		service->exit_failure_last = ioloop_time;
+		service->exit_failures_in_sec = 0;
+	}
+	if (++service->exit_failures_in_sec > SERVICE_MAX_EXIT_FAILURES_IN_SEC)
+		throttle = TRUE;
 	service_process_notify_add(service_anvil_global->kills, process);
 	return throttle;
 }
@@ -554,6 +562,8 @@
 			if (service->listen_pending &&
 			    !service->list->destroying)
 				service_monitor_listen_start(service);
+			/* one success resets all failures */
+			service->exit_failures_in_sec = 0;
 			throttle = FALSE;
 		} else {
 			throttle = service_process_failure(process, status);
diff -r c9b5ea1b9709 -r 300736775b89 src/master/service.h
--- a/src/master/service.h	Fri Jan 27 22:15:29 2012 +0200
+++ b/src/master/service.h	Fri Jan 27 23:22:22 2012 +0200
@@ -80,6 +80,9 @@
 	int status_fd[2];
 	struct io *io_status;
 
+	time_t exit_failure_last;
+	unsigned int exit_failures_in_sec;
+
 	/* Login process's notify fd. We change its seek position to
 	   communicate state to login processes. */
 	int login_notify_fd;