dovecot-2.2: director: Don't become desynced if two directors ch...

[dovecot at dovecot.org]

details:   http://hg.dovecot.org/dovecot-2.2/rev/0f9d4f1a083d
changeset: 19287:0f9d4f1a083d
user:      Timo Sirainen <tss at iki.fi>
date:      Sat Oct 10 18:32:06 2015 +0300
description:
director: Don't become desynced if two directors change the same backend in incompatible ways.
This would have caused "User hash .. is being redirected to two hosts"
errors, which wouldn't easily go away as the directors have a different view
of what hosts currently exist.

diffstat:

 src/director/director-connection.c |  20 +++++++++++++++++++-
 src/director/director.c            |   3 +++
 src/director/mail-host.c           |   8 ++++++++
 src/director/mail-host.h           |   4 ++++
 4 files changed, 34 insertions(+), 1 deletions(-)

diffs (103 lines):

diff -r 9d2dd826ae7e -r 0f9d4f1a083d src/director/director-connection.c
--- a/src/director/director-connection.c	Tue Oct 06 18:17:45 2015 +0300
+++ b/src/director/director-connection.c	Sat Oct 10 18:32:06 2015 +0300
@@ -842,6 +842,7 @@
 director_cmd_host_int(struct director_connection *conn, const char *const *args,
 		      struct director_host *dir_host)
 {
+	struct director_host *src_host = conn->host;
 	struct mail_host *host;
 	struct ip_addr ip;
 	const char *tag = "";
@@ -889,6 +890,23 @@
 			mail_host_set_tag(host, tag);
 			update = TRUE;
 		}
+		if (update && host->desynced) {
+			vhost_count = I_MIN(vhost_count, host->vhost_count);
+			if (host->down != down) {
+				if (host->last_updown_change <= last_updown_change)
+					down = host->last_updown_change;
+			}
+			last_updown_change = I_MAX(last_updown_change,
+						   host->last_updown_change);
+			i_warning("director(%s): Host %s is being updated before previous update had finished - "
+				  "setting to state=%s vhosts=%u",
+				  conn->name, net_ip2addr(&host->ip),
+				  down ? "down" : "up", vhost_count);
+			/* make the change appear to come from us, so it
+			   reaches the full ring */
+			dir_host = NULL;
+			src_host = conn->dir->self_host;
+		}
 	}
 
 	if (update) {
@@ -896,7 +914,7 @@
 				   down, last_updown_change);
 		mail_host_set_vhost_count(conn->dir->mail_hosts,
 					  host, vhost_count);
-		director_update_host(conn->dir, conn->host, dir_host, host);
+		director_update_host(conn->dir, src_host, dir_host, host);
 	}
 	return TRUE;
 }
diff -r 9d2dd826ae7e -r 0f9d4f1a083d src/director/director.c
--- a/src/director/director.c	Tue Oct 06 18:17:45 2015 +0300
+++ b/src/director/director.c	Sat Oct 10 18:32:06 2015 +0300
@@ -314,6 +314,7 @@
 		timeout_remove(&dir->to_sync);
 	dir->ring_synced = TRUE;
 	dir->ring_last_sync_time = ioloop_time;
+	mail_hosts_set_synced(dir->mail_hosts);
 	director_set_state_changed(dir);
 }
 
@@ -548,6 +549,8 @@
 	}
 	str_append_c(str, '\n');
 	director_update_send(dir, src, str_c(str));
+
+	host->desynced = TRUE;
 	director_sync(dir);
 }
 
diff -r 9d2dd826ae7e -r 0f9d4f1a083d src/director/mail-host.c
--- a/src/director/mail-host.c	Tue Oct 06 18:17:45 2015 +0300
+++ b/src/director/mail-host.c	Sat Oct 10 18:32:06 2015 +0300
@@ -388,6 +388,14 @@
 		return mail_host_get_by_hash_direct(list, hash, tag);
 }
 
+void mail_hosts_set_synced(struct mail_host_list *list)
+{
+	struct mail_host *const *hostp;
+
+	array_foreach(&list->hosts, hostp)
+		(*hostp)->desynced = FALSE;
+}
+
 bool mail_hosts_have_usable(struct mail_host_list *list)
 {
 	if (list->hosts_unsorted)
diff -r 9d2dd826ae7e -r 0f9d4f1a083d src/director/mail-host.h
--- a/src/director/mail-host.h	Tue Oct 06 18:17:45 2015 +0300
+++ b/src/director/mail-host.h	Sat Oct 10 18:32:06 2015 +0300
@@ -15,6 +15,9 @@
 
 	struct ip_addr ip;
 	char *tag;
+
+	/* host was recently changed and ring hasn't synced yet since */
+	unsigned int desynced:1;
 };
 ARRAY_DEFINE_TYPE(mail_host, struct mail_host *);
 
@@ -37,6 +40,7 @@
 			       unsigned int vhost_count);
 void mail_host_remove(struct mail_host_list *list, struct mail_host *host);
 
+void mail_hosts_set_synced(struct mail_host_list *list);
 bool mail_hosts_have_usable(struct mail_host_list *list);
 const ARRAY_TYPE(mail_host) *mail_hosts_get(struct mail_host_list *list);