dovecot-2.2: dsync: Added -t <timestamp> parameter to save only ...
dovecot at dovecot.org
dovecot at dovecot.org
Mon Jan 19 21:59:53 UTC 2015
details: http://hg.dovecot.org/dovecot-2.2/rev/f393f63764e0
changeset: 18177:f393f63764e0
user: Timo Sirainen <tss at iki.fi>
date: Mon Jan 19 23:43:37 2015 +0200
description:
dsync: Added -t <timestamp> parameter to save only mails newer than <timestamp>
If one side has old mails that don't exist on the other side, they are
ignored (not synced and not deleted).
diffstat:
src/doveadm/doveadm-dsync.c | 9 ++++++++-
src/doveadm/dsync/dsync-brain-mailbox.c | 3 +++
src/doveadm/dsync/dsync-brain-private.h | 1 +
src/doveadm/dsync/dsync-brain.c | 3 +++
src/doveadm/dsync/dsync-brain.h | 2 ++
src/doveadm/dsync/dsync-ibc-stream.c | 26 ++++++++++++++++++++++++--
src/doveadm/dsync/dsync-ibc.h | 2 ++
src/doveadm/dsync/dsync-mail.c | 1 +
src/doveadm/dsync/dsync-mail.h | 3 +++
src/doveadm/dsync/dsync-mailbox-export.c | 20 +++++++++++++++++++-
src/doveadm/dsync/dsync-mailbox-export.h | 3 ++-
src/doveadm/dsync/dsync-mailbox-import.c | 25 ++++++++++++++++++++++++-
src/doveadm/dsync/dsync-mailbox-import.h | 1 +
13 files changed, 93 insertions(+), 6 deletions(-)
diffs (truncated from 375 to 300 lines):
diff -r b0fb4113fbbe -r f393f63764e0 src/doveadm/doveadm-dsync.c
--- a/src/doveadm/doveadm-dsync.c Mon Jan 19 23:40:27 2015 +0200
+++ b/src/doveadm/doveadm-dsync.c Mon Jan 19 23:43:37 2015 +0200
@@ -16,6 +16,7 @@
#include "settings-parser.h"
#include "master-service.h"
#include "master-service-ssl-settings.h"
+#include "mail-storage.h"
#include "mail-storage-service.h"
#include "mail-user.h"
#include "mail-namespace.h"
@@ -36,7 +37,7 @@
#include <ctype.h>
#include <sys/wait.h>
-#define DSYNC_COMMON_GETOPT_ARGS "+1dEfg:l:m:n:NPr:Rs:Ux:"
+#define DSYNC_COMMON_GETOPT_ARGS "+1dEfg:l:m:n:NPr:Rs:t:Ux:"
#define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30
/* The broken_char is mainly set to get a proper error message when trying to
convert a mailbox with a name that can't be used properly translated between
@@ -59,6 +60,7 @@
const char *state_input, *rawlog_path;
ARRAY_TYPE(const_string) exclude_mailboxes;
ARRAY_TYPE(const_string) namespace_prefixes;
+ time_t sync_since_timestamp;
const char *remote_name;
const char *local_location;
@@ -539,6 +541,7 @@
set.process_title_prefix = t_strdup_printf(
"%s ", net_ip2addr(&_ctx->cur_client_ip));
}
+ set.sync_since_timestamp = ctx->sync_since_timestamp;
set.sync_box = ctx->mailbox;
memcpy(set.sync_box_guid, ctx->mailbox_guid, sizeof(set.sync_box_guid));
set.lock_timeout_secs = ctx->lock_timeout;
@@ -960,6 +963,10 @@
ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE;
ctx->state_input = optarg;
break;
+ case 't':
+ if (mail_parse_human_timestamp(optarg, &ctx->sync_since_timestamp) < 0)
+ i_fatal("Invalid -t parameter: %s", optarg);
+ break;
case 'U':
ctx->replicator_notify = TRUE;
break;
diff -r b0fb4113fbbe -r f393f63764e0 src/doveadm/dsync/dsync-brain-mailbox.c
--- a/src/doveadm/dsync/dsync-brain-mailbox.c Mon Jan 19 23:40:27 2015 +0200
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c Mon Jan 19 23:43:37 2015 +0200
@@ -220,6 +220,7 @@
remote_dsync_box->first_recent_uid,
remote_dsync_box->highest_modseq,
remote_dsync_box->highest_pvt_modseq,
+ brain->sync_since_timestamp,
import_flags);
}
@@ -306,6 +307,8 @@
exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS;
if (brain->no_mail_prefetch)
exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL;
+ if (brain->sync_since_timestamp > 0)
+ exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS;
brain->box_exporter = brain->backup_recv ? NULL :
dsync_mailbox_export_init(brain->box, brain->log_scan,
diff -r b0fb4113fbbe -r f393f63764e0 src/doveadm/dsync/dsync-brain-private.h
--- a/src/doveadm/dsync/dsync-brain-private.h Mon Jan 19 23:40:27 2015 +0200
+++ b/src/doveadm/dsync/dsync-brain-private.h Mon Jan 19 23:43:37 2015 +0200
@@ -55,6 +55,7 @@
guid_128_t sync_box_guid;
const char *const *exclude_mailboxes;
enum dsync_brain_sync_type sync_type;
+ time_t sync_since_timestamp;
char alt_char;
unsigned int lock_timeout;
diff -r b0fb4113fbbe -r f393f63764e0 src/doveadm/dsync/dsync-brain.c
--- a/src/doveadm/dsync/dsync-brain.c Mon Jan 19 23:40:27 2015 +0200
+++ b/src/doveadm/dsync/dsync-brain.c Mon Jan 19 23:43:37 2015 +0200
@@ -175,6 +175,7 @@
}
brain->alt_char = set->mailbox_alt_char == '\0' ? '_' :
set->mailbox_alt_char;
+ brain->sync_since_timestamp = set->sync_since_timestamp;
brain->sync_box = p_strdup(brain->pool, set->sync_box);
brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL :
p_strarray_dup(brain->pool, set->exclude_mailboxes);
@@ -207,6 +208,7 @@
NULL : str_c(sync_ns_str);
ibc_set.sync_box = set->sync_box;
ibc_set.exclude_mailboxes = set->exclude_mailboxes;
+ ibc_set.sync_since_timestamp = set->sync_since_timestamp;
memcpy(ibc_set.sync_box_guid, set->sync_box_guid,
sizeof(ibc_set.sync_box_guid));
ibc_set.sync_type = sync_type;
@@ -452,6 +454,7 @@
brain->sync_box = p_strdup(brain->pool, ibc_set->sync_box);
brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL :
p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes);
+ brain->sync_since_timestamp = ibc_set->sync_since_timestamp;
memcpy(brain->sync_box_guid, ibc_set->sync_box_guid,
sizeof(brain->sync_box_guid));
i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN);
diff -r b0fb4113fbbe -r f393f63764e0 src/doveadm/dsync/dsync-brain.h
--- a/src/doveadm/dsync/dsync-brain.h Mon Jan 19 23:40:27 2015 +0200
+++ b/src/doveadm/dsync/dsync-brain.h Mon Jan 19 23:43:37 2015 +0200
@@ -55,6 +55,8 @@
/* Alternative character to use in mailbox names where the original
character cannot be used. */
char mailbox_alt_char;
+ /* Sync only mails with received timestamp at least this high. */
+ time_t sync_since_timestamp;
/* If non-zero, use dsync lock file for this user */
unsigned int lock_timeout_secs;
diff -r b0fb4113fbbe -r f393f63764e0 src/doveadm/dsync/dsync-ibc-stream.c
--- a/src/doveadm/dsync/dsync-ibc-stream.c Mon Jan 19 23:40:27 2015 +0200
+++ b/src/doveadm/dsync/dsync-ibc-stream.c Mon Jan 19 23:43:37 2015 +0200
@@ -73,9 +73,10 @@
.chr = 'H',
.required_keys = "hostname",
.optional_keys = "sync_ns_prefix sync_box sync_box_guid sync_type "
- "debug sync_visible_namespaces exclude_mailboxes "
+ "debug sync_visible_namespaces exclude_mailboxes "
"send_mail_requests backup_send backup_recv lock_timeout "
- "no_mail_sync no_backup_overwrite purge_remote"
+ "no_mail_sync no_backup_overwrite purge_remote "
+ "sync_since_timestamp"
},
{ .name = "mailbox_state",
.chr = 'S',
@@ -649,6 +650,10 @@
dsync_serializer_encode_add(encoder, "lock_timeout",
t_strdup_printf("%u", set->lock_timeout));
}
+ if (set->sync_since_timestamp > 0) {
+ dsync_serializer_encode_add(encoder, "sync_since_timestamp",
+ t_strdup_printf("%ld", (long)set->sync_since_timestamp));
+ }
if ((set->brain_flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0)
dsync_serializer_encode_add(encoder, "send_mail_requests", "");
if ((set->brain_flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0)
@@ -741,6 +746,14 @@
return DSYNC_IBC_RECV_RET_TRYAGAIN;
}
}
+ if (dsync_deserializer_decode_try(decoder, "sync_since_timestamp", &value)) {
+ if (str_to_time(value, &set->sync_since_timestamp) < 0 ||
+ set->sync_since_timestamp == 0) {
+ dsync_ibc_input_error(ibc, decoder,
+ "Invalid sync_since_timestamp: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ }
if (dsync_deserializer_decode_try(decoder, "send_mail_requests", &value))
set->brain_flags |= DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS;
if (dsync_deserializer_decode_try(decoder, "backup_send", &value))
@@ -1539,6 +1552,10 @@
dsync_serializer_encode_add(encoder, "keyword_changes",
str_c(kw_str));
}
+ if (change->received_timestamp > 0) {
+ dsync_serializer_encode_add(encoder, "received_timestamp",
+ t_strdup_printf("%lx", (unsigned long)change->received_timestamp));
+ }
dsync_serializer_encode_finish(&encoder, str);
dsync_ibc_stream_send_string(ibc, str);
@@ -1619,6 +1636,11 @@
array_append(&change->keyword_changes, &value, 1);
}
}
+ if (dsync_deserializer_decode_try(decoder, "received_timestamp", &value) &&
+ str_to_time(value, &change->received_timestamp) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid received_timestamp");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
*change_r = change;
return DSYNC_IBC_RECV_RET_OK;
diff -r b0fb4113fbbe -r f393f63764e0 src/doveadm/dsync/dsync-ibc.h
--- a/src/doveadm/dsync/dsync-ibc.h Mon Jan 19 23:40:27 2015 +0200
+++ b/src/doveadm/dsync/dsync-ibc.h Mon Jan 19 23:43:37 2015 +0200
@@ -52,6 +52,8 @@
/* Exclude these mailboxes from the sync. They can contain '*'
wildcards and be \special-use flags. */
const char *const *exclude_mailboxes;
+ /* Sync only mails with received timestamp at least this high. */
+ time_t sync_since_timestamp;
enum dsync_brain_sync_type sync_type;
enum dsync_brain_flags brain_flags;
diff -r b0fb4113fbbe -r f393f63764e0 src/doveadm/dsync/dsync-mail.c
--- a/src/doveadm/dsync/dsync-mail.c Mon Jan 19 23:40:27 2015 +0200
+++ b/src/doveadm/dsync/dsync-mail.c Mon Jan 19 23:43:37 2015 +0200
@@ -156,4 +156,5 @@
dest_r->keywords_reset = src->keywords_reset;
const_string_array_dup(pool, &src->keyword_changes,
&dest_r->keyword_changes);
+ dest_r->received_timestamp = src->received_timestamp;
}
diff -r b0fb4113fbbe -r f393f63764e0 src/doveadm/dsync/dsync-mail.h
--- a/src/doveadm/dsync/dsync-mail.h Mon Jan 19 23:40:27 2015 +0200
+++ b/src/doveadm/dsync/dsync-mail.h Mon Jan 19 23:43:37 2015 +0200
@@ -77,6 +77,9 @@
bool keywords_reset;
/* +add, -remove, =final, &add_and_final. */
ARRAY_TYPE(const_string) keyword_changes;
+
+ /* Received timestamp for saves, if brain.sync_since_timestamp is set */
+ time_t received_timestamp;
};
struct mailbox_header_lookup_ctx *
diff -r b0fb4113fbbe -r f393f63764e0 src/doveadm/dsync/dsync-mailbox-export.c
--- a/src/doveadm/dsync/dsync-mailbox-export.c Mon Jan 19 23:40:27 2015 +0200
+++ b/src/doveadm/dsync/dsync-mailbox-export.c Mon Jan 19 23:43:37 2015 +0200
@@ -59,6 +59,7 @@
unsigned int mails_have_guids:1;
unsigned int minimal_dmail_fill:1;
unsigned int return_all_mails:1;
+ unsigned int export_received_timestamps:1;
};
static int dsync_mail_error(struct dsync_mailbox_exporter *exporter,
@@ -271,21 +272,36 @@
{
struct dsync_mail_change *change;
const char *guid, *hdr_hash;
+ enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID;
+ time_t received_timestamp = 0;
int ret;
/* update wanted fields in case we didn't already set them for the
search */
- mail_add_temp_wanted_fields(mail, MAIL_FETCH_GUID,
+ if (exporter->export_received_timestamps)
+ wanted_fields |= MAIL_FETCH_RECEIVED_DATE;
+ mail_add_temp_wanted_fields(mail, wanted_fields,
exporter->wanted_headers);
/* If message is already expunged here, just skip it */
if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0)
return ret;
+ if (exporter->export_received_timestamps) {
+ if (mail_get_received_date(mail, &received_timestamp) < 0)
+ return dsync_mail_error(exporter, mail, "received-time");
+ if (received_timestamp == 0) {
+ /* don't allow timestamps to be zero. we want to have
+ asserts verify that the timestamp is set properly. */
+ received_timestamp = 1;
+ }
+ }
+
change = export_save_change_get(exporter, mail->uid);
change->guid = *guid == '\0' ? "" :
p_strdup(exporter->pool, guid);
change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
+ change->received_timestamp = received_timestamp;
search_update_flag_changes(exporter, mail, change);
export_add_mail_instance(exporter, change, mail->seq);
@@ -478,6 +494,8 @@
(flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0;
exporter->minimal_dmail_fill =
(flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
+ exporter->export_received_timestamps =
+ (flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0;
p_array_init(&exporter->requested_uids, pool, 16);
p_array_init(&exporter->search_uids, pool, 16);
hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
diff -r b0fb4113fbbe -r f393f63764e0 src/doveadm/dsync/dsync-mailbox-export.h
--- a/src/doveadm/dsync/dsync-mailbox-export.h Mon Jan 19 23:40:27 2015 +0200
+++ b/src/doveadm/dsync/dsync-mailbox-export.h Mon Jan 19 23:43:37 2015 +0200
@@ -4,7 +4,8 @@
enum dsync_mailbox_exporter_flags {
DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS = 0x01,
DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS = 0x02,
- DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL = 0x04
+ DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL = 0x04,
+ DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS = 0x08
};
struct dsync_mailbox_exporter *
diff -r b0fb4113fbbe -r f393f63764e0 src/doveadm/dsync/dsync-mailbox-import.c
--- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 19 23:40:27 2015 +0200
+++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Jan 19 23:43:37 2015 +0200
@@ -58,6 +58,7 @@
uint32_t remote_uid_next;
uint32_t remote_first_recent_uid;
uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
+ time_t sync_since_timestamp;
struct mailbox_transaction_context *trans, *ext_trans;
struct mail_search_context *search_ctx;
@@ -177,6 +178,7 @@
uint32_t remote_first_recent_uid,
More information about the dovecot-cvs
mailing list