Dovecot mailstore performance tuning

[Murray Trainer]

HI All,

We have a couple of dovecot director proxies and six backed mailstores
each accessing mailboxes stored on five NFSv4 filsystems with about
1TB of mail on each in maildir format.  We have about 800 max users
on each mailstore at peak times and performance appears to starting to
degrade at these times.  The mailstores are pretty recent hardware
with 64GB of RAM and 24 cores.   The NFS storage is EMC VNX and we
are doing about 250 I/O per sec upto max of 500 on each
filesystem.   I need to squeeze more performance out of these
servers whether that is in the NFS client, Dovecot or Linux OS/kernel
areas.   We use LDAP for auth and I am doing some tuning in that
area.   The NFS filesystems are mounted with the options below:

10.11.0.238:/mailbox_store_01 on /home1 type nfs4
(rw,relatime,vers=4.0,rsize=65536,wsize=65536,namlen=255,hard,nordirplus,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=10.11.0.96,local_lock=none,addr=10.11.0.238)

My dovecot config is below.  I am not seeing any obvious issues in
the server logs.    Any suggestions on how to improve performance
would be appreciated.

Thanks

Murray

# doveconf -n
# 2.2.9: /etc/dovecot/dovecot.conf
doveconf: Warning: service auth { client_limit=40960 } is lower than
required under max. load (41280)
doveconf: Warning: service anvil { client_limit=40970 } is lower than
required under max. load (41183)
# OS: Linux 3.13-0.bpo.1-amd64 x86_64 Debian 7.5
auth_cache_size = 64 M
auth_cache_ttl = 2 hours
auth_failure_delay = 0
auth_mechanisms = plain login
auth_username_chars =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$-=?^_{}~./@+%"
auth_username_translation = +@
auth_worker_max_count = 256
base_dir = /var/run/dovecot/
disable_plaintext_auth = no
first_valid_gid = 1001
first_valid_uid = 1001
mail_fsync = always
mail_location = maildir:~/
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope
encoded-character vacation subaddress comparator-i;ascii-numeric
relational regex imap4flags copy include variables body enotify
environment mailbox date ihave
mmap_disable = yes
namespace {
  inbox = yes
  location =
  prefix = INBOX.
  type = private
}
passdb {
  args = /etc/dovecot/dovecot-ldap.conf.ext
  driver = ldap
}
plugin {
  acl = vfile
  quota = maildir
  quota_rule = INBOX.Trash:ignore
}
protocols = " imap lmtp sieve pop3"
service anvil {
  client_limit = 40970
}
service auth-worker {
  user = dovecot
}
service auth {
  client_limit = 40960
  unix_listener auth-userdb {
    group = mail
    mode = 0666
    user = dovecot
  }
}
service imap-login {
  chroot = login
  inet_listener imap {
    address = *, [::]
    port = 143
  }
  inet_listener imaps {
    address = *
    port = 993
    ssl = yes
  }
  process_limit = 20480
  process_min_avail = 32
  service_count = 1
  user = dovecot
  vsz_limit = 256 M
}
service imap {
  process_limit = 40960
  vsz_limit = 256 M
}
service lmtp {
  inet_listener lmtp {
    address = 27.54.95.41
    port = 24
  }
  process_min_avail = 32
}
service managesieve-login {
  client_limit = 40960
  process_limit = 120
  process_min_avail = 5
  service_count = 0
  vsz_limit = 64 M
}
service managesieve {
  process_limit = 4096
  vsz_limit = 256 M
}
service pop3-login {
  chroot = login
  inet_listener pop3 {
    address = *, [::]
    port = 110
  }
  inet_listener pop3s {
    address = *
    port = 995
    ssl = yes
  }
  process_limit = 20480
  process_min_avail = 32
  service_count = 1
  user = dovecot
  vsz_limit = 256 M
}
service pop3 {
  process_limit = 40960
  vsz_limit = 256 M
}
ssl_cert =