Index: sync_client.c =================================================================== RCS file: /afs/andrew/system/cvs/src/cyrus/imap/sync_client.c,v retrieving revision 1.20 diff -u -r1.20 sync_client.c --- sync_client.c 24 Sep 2007 12:48:32 -0000 1.20 +++ sync_client.c 24 Sep 2007 17:47:08 -0000 @@ -1206,66 +1206,24 @@ return(r); } -static int upload_messages_list(struct mailbox *mailbox, - struct sync_msg_list *list) +static int index_list_work(struct mailbox *mailbox, struct sync_index_list *index_list) { - unsigned long msgno = 1; + struct sync_index *index; int r = 0; - struct index_record record; - struct sync_msg *msg; - int count; int c = ' '; static struct buf token; /* BSS */ - int max_count = config_getint(IMAPOPT_SYNC_BATCH_SIZE); - if (max_count <= 0) max_count = INT_MAX; + if (index_list->count == 0) return(0); - if (chdir(mailbox->path)) { - syslog(LOG_ERR, "Couldn't chdir to %s: %s", - mailbox->path, strerror(errno)); - return(IMAP_IOERROR); - } - -repeatupload: - - msg = list->head; - for (count = 0; count < max_count && msgno <= mailbox->exists ; msgno++) { - r = mailbox_read_index_record(mailbox, msgno, &record); - - if (r) { - syslog(LOG_ERR, - "IOERROR: reading index entry for nsgno %lu of %s: %m", - record.uid, mailbox->name); - return(IMAP_IOERROR); - } - - /* Skip over messages recorded on server which are missing on client - * (either will be expunged or have been expunged already) */ - while (msg && (record.uid > msg->uid)) - msg = msg->next; - - if (msg && (record.uid == msg->uid) && - message_guid_compare_allow_null(&record.guid, &msg->guid)) { - msg = msg->next; /* Ignore exact match */ - continue; - } - - if (count++ == 0) - prot_printf(toserver, "UPLOAD %lu %lu", - mailbox->last_uid, mailbox->last_appenddate); - - /* Message with this GUID exists on client but not server */ - if ((r=upload_message_work(mailbox, msgno, &record))) - break; + prot_printf(toserver, "UPLOAD %lu %lu", + index_list->last_uid, mailbox->last_appenddate); - if (msg && (msg->uid == record.uid)) /* Overwritten on server */ - msg = msg->next; + for (index = index_list->head; index; index = index->next) { + r = upload_message_work(mailbox, index->msgno, &(index->record)); + if (r) break; } - if (count == 0) - return(r); - - prot_printf(toserver, "\r\n"); + prot_printf(toserver, "\r\n"); prot_flush(toserver); if (r) { @@ -1289,27 +1247,23 @@ sync_msgid_list_free(&msgid_onserver); msgid_onserver = sync_msgid_list_create(hash_size); - syslog(LOG_INFO, "UPLOAD: received RESTART"); - } - - /* don't overload the server with too many uploads at once! */ - if (count >= max_count) { - syslog(LOG_INFO, "UPLOAD: hit %d uploads at msgno %d", count, msgno); - goto repeatupload; + syslog(LOG_INFO, "UPLOAD: received RESTART"); } return(0); } -static int upload_messages_from(struct mailbox *mailbox, - unsigned long old_last_uid) +static int upload_messages_list(struct mailbox *mailbox, + struct sync_msg_list *list) { unsigned long msgno; int r = 0; struct index_record record; - int count = 0; - int c = ' '; - static struct buf token; /* BSS */ + struct sync_msg *msg; + struct sync_index_list *index_list; + int max_count = config_getint(IMAPOPT_SYNC_BATCH_SIZE); + + if (max_count <= 0) max_count = INT_MAX; if (chdir(mailbox->path)) { syslog(LOG_ERR, "Couldn't chdir to %s: %s", @@ -1317,58 +1271,104 @@ return(IMAP_IOERROR); } - for (msgno = 1 ; msgno <= mailbox->exists ; msgno++) { - r = mailbox_read_index_record(mailbox, msgno, &record); - - if (r) { - syslog(LOG_ERR, - "IOERROR: reading index entry for nsgno %lu of %s: %m", - record.uid, mailbox->name); - return(IMAP_IOERROR); - } - - if (record.uid <= old_last_uid) - continue; - - if (count++ == 0) - prot_printf(toserver, "UPLOAD %lu %lu", - mailbox->last_uid, mailbox->last_appenddate); + msgno = 1; + msg = list->head; + do { + /* Break UPLOAD into chunks of <=max_count messages */ + index_list = sync_index_list_create(); + + for (; (index_list->count <= max_count) && + (msgno <= mailbox->exists); msgno++) { + r = mailbox_read_index_record(mailbox, msgno, &record); + + if (r) { + syslog(LOG_ERR, + "IOERROR: reading index entry for nsgno %lu of %s: %m", + record.uid, mailbox->name); + return(IMAP_IOERROR); + } + + /* Skip over messages recorded on server which are missing on client + * (either will be expunged or have been expunged already) */ + while (msg && (record.uid > msg->uid)) + msg = msg->next; + + if (msg && (record.uid == msg->uid) && + message_guid_compare_allow_null(&record.guid, &msg->guid)) { + msg = msg->next; /* Ignore exact match */ + continue; + } + + /* Message with this GUID exists on client but not server */ + sync_index_list_add(index_list, msgno, &record); + } + + if (msgno > mailbox->exists) { + /* last chunk - final UID might not be same as UIDLAST */ + index_list->last_uid = mailbox->last_uid; + } else { + syslog(LOG_NOTICE, "Hit upload limit %d at UID %lu for %s, sending", + max_count, index_list->last_uid, mailbox->name); + } - if ((r=upload_message_work(mailbox, msgno, &record))) - break; - } + r = index_list_work(mailbox, index_list); + sync_index_list_free(&index_list); - if (count == 0) - return(r); + } while (!r); - prot_printf(toserver, "\r\n"); - prot_flush(toserver); + return(r); +} - if (r) { - sync_parse_code("UPLOAD", fromserver, SYNC_PARSE_EAT_OKLINE, NULL); - return(r); - } - r = sync_parse_code("UPLOAD", fromserver, SYNC_PARSE_NOEAT_OKLINE, NULL); - if (r) return(r); +static int upload_messages_from(struct mailbox *mailbox, + unsigned long old_last_uid) +{ + unsigned long msgno; + int r = 0; + struct index_record record; + struct sync_index_list *index_list; + int max_count = config_getint(IMAPOPT_SYNC_BATCH_SIZE); - if ((c = getword(fromserver, &token)) != ' ') { - eatline(fromserver, c); - syslog(LOG_ERR, "Garbage on Upload response"); - return(IMAP_PROTOCOL_ERROR); + if (chdir(mailbox->path)) { + syslog(LOG_ERR, "Couldn't chdir to %s: %s", + mailbox->path, strerror(errno)); + return(IMAP_IOERROR); } - eatline(fromserver, c); - /* Clear out msgid_on_server list if server restarted */ - if (!strcmp(token.s, "[RESTART]")) { - int hash_size = msgid_onserver->hash_size; + do { + /* Break UPLOAD into chunks of <=max_count messages */ + index_list = sync_index_list_create(); + + for (; (index_list->count <= max_count) && + (msgno <= mailbox->exists); msgno++) { + r = mailbox_read_index_record(mailbox, msgno, &record); + + if (r) { + syslog(LOG_ERR, + "IOERROR: reading index entry for nsgno %lu of %s: %m", + record.uid, mailbox->name); + return(IMAP_IOERROR); + } + + if (record.uid <= old_last_uid) continue; + + /* Message with this GUID exists on client but not server */ + sync_index_list_add(index_list, msgno, &record); + } + + if (msgno > mailbox->exists) { + /* last chunk - final UID might not be same as UIDLAST */ + index_list->last_uid = mailbox->last_uid; + } else { + syslog(LOG_NOTICE, "Hit upload limit %d at UID %lu for %s, sending", + max_count, index_list->last_uid, mailbox->name); + } - sync_msgid_list_free(&msgid_onserver); - msgid_onserver = sync_msgid_list_create(hash_size); + r = index_list_work(mailbox, index_list); + sync_index_list_free(&index_list); - syslog(LOG_INFO, "UPLOAD: received RESTART"); - } + } while(!r); - return(0); + return(r); } /* upload_messages() null operations still requires UIDLAST update */ Index: sync_support.c =================================================================== RCS file: /afs/andrew/system/cvs/src/cyrus/imap/sync_support.c,v retrieving revision 1.13 diff -u -r1.13 sync_support.c --- sync_support.c 22 Sep 2007 12:08:23 -0000 1.13 +++ sync_support.c 24 Sep 2007 17:47:08 -0000 @@ -415,6 +415,56 @@ /* ====================================================================== */ +/* sync_index stuff */ + +struct sync_index_list *sync_index_list_create() +{ + struct sync_index_list *l = xzmalloc(sizeof (struct sync_index_list)); + + l->head = NULL; + l->tail = NULL; + l->count = 0; + l->last_uid = 0; + + return l; +} + +void sync_index_list_add(struct sync_index_list *l, + unsigned long msgno, struct index_record *record) +{ + struct sync_index *result = xzmalloc(sizeof(struct sync_index)); + + result->msgno = msgno; + memcpy(&(result->record), record, sizeof(struct index_record)); + + if (l->tail) + l->tail = l->tail->next = result; + else + l->head = l->tail = result; + + l->count++; + l->last_uid = record->uid; +} + +void sync_index_list_free(struct sync_index_list **lp) +{ + struct sync_index_list *l = *lp; + struct sync_index *current, *next; + + current = l->head; + while (current) { + next = current->next; + free(current); + current = next; + } + free(l); + + *lp = NULL; +} + + +/* ====================================================================== */ + /* sync_msg stuff */ struct sync_msg_list *sync_msg_list_create(char **flagname, Index: sync_support.h =================================================================== RCS file: /afs/andrew/system/cvs/src/cyrus/imap/sync_support.h,v retrieving revision 1.6 diff -u -r1.6 sync_support.h --- sync_support.h 24 Sep 2007 12:48:32 -0000 1.6 +++ sync_support.h 24 Sep 2007 17:47:08 -0000 @@ -48,6 +48,7 @@ #define INCLUDED_SYNC_SUPPORT_H #include "prot.h" +#include "mailbox.h" #define SYNC_MSGID_LIST_HASH_SIZE (65536) #define SYNC_MESSAGE_LIST_HASH_SIZE (65536) @@ -94,6 +95,29 @@ struct sync_flags *flags, struct sync_flags_meta *meta); void sync_flags_meta_to_list(struct sync_flags_meta *meta, char **flagname); + +/* ====================================================================== */ + +/* sync_index_list records index records for upload */ + +struct sync_index { + struct sync_index *next; + struct index_record record; + unsigned long msgno; +}; + +struct sync_index_list { + struct sync_index *head, *tail; + unsigned long count; + unsigned long last_uid; +}; + +struct sync_index_list *sync_index_list_create(void); + +void sync_index_list_add(struct sync_index_list *l, + unsigned long msgno, struct index_record *record); + +void sync_index_list_free(struct sync_index_list **lp); /* ====================================================================== */