From 5aef5ab7b0f4aabe2736ba5e5182c06b604c7f6f Mon Sep 17 00:00:00 2001 From: grimhilt Date: Fri, 5 May 2023 19:09:27 +0200 Subject: [PATCH] improve initial sync behavior --- back/mails/imap/Mailbox.ts | 119 ++++++++++++++++++++++--------------- 1 file changed, 70 insertions(+), 49 deletions(-) diff --git a/back/mails/imap/Mailbox.ts b/back/mails/imap/Mailbox.ts index 7fd944e..9334cb3 100644 --- a/back/mails/imap/Mailbox.ts +++ b/back/mails/imap/Mailbox.ts @@ -1,4 +1,5 @@ import Imap, { Box } from "imap"; +import { resolve } from "path"; import { getMailbox, getMailboxModseq, updateMailbox, updateMailboxModseq } from "../../db/imap/imap-db"; import { Attrs, AttrsWithEnvelope } from "../../interfaces/mail/attrs.interface"; import logger from "../../system/Logger"; @@ -44,7 +45,7 @@ export default class Mailbox { this.imap.on("mail", (numNewMsgs: number) => { if (!this.syncing) { // if not syncing restart a sync - this.syncMail(this.box.uidnext, this.box.uidnext + numNewMsgs); + this.syncManager(this.box.uidnext - 1, this.box.uidnext + numNewMsgs - 1); } else { // else save number of message to sync latter this.msgToSync += numNewMsgs; @@ -57,6 +58,10 @@ export default class Mailbox { const updateMsg = new updateMessage(info.uid, info.flags); updateMsg.updateFlags(); }); + + this.imap.on("expunge", (seqno: number) => { + console.log("Message with sequence number " + seqno + " has been deleted from the server."); + }); }); } @@ -69,7 +74,7 @@ export default class Mailbox { async initSync(box: Box) { // sync mail only if has new messages if (this.box.uidnext < box.uidnext) { - this.syncMail(this.box.uidnext, box.uidnext); + this.syncManager(this.box.uidnext, box.uidnext); } else { logger.log("Mail already up to date"); } @@ -91,62 +96,78 @@ export default class Mailbox { logger.log("Done fetching new flags"); }); } else { - logger.log("Flags already up to date") + logger.log("Flags already up to date"); } this.updateModseq(parseInt(box.highestmodseq)); } - async syncMail(savedUid: number, currentUid: number) { + syncManager = async (savedUid: number, currentUid: number) => { this.syncing = true; - const promises: Promise[] = []; - const mails: Attrs[] = []; - logger.log(`Syncing from ${savedUid} to ${currentUid} uid`); - const f = this.imap.seq.fetch(`${savedUid}:${currentUid}`, { - size: true, - envelope: true, - }); + logger.log(`Fetching from ${savedUid} to ${currentUid} uid`); + const nbMessageToSync = currentUid - savedUid; + let STEP = nbMessageToSync > 300 ? Math.floor(nbMessageToSync / 7) : nbMessageToSync; + let mails: AttrsWithEnvelope[] = []; - f.on("message", (msg, seqno) => { - msg.once("attributes", (attrs: AttrsWithEnvelope) => { - mails.push(attrs); - promises.push(saveMessage(attrs, this.id, this.imap)); - }); - }); - - f.once("error", (err) => { - logger.err("Fetch error: " + err); - }); - - f.once("end", async () => { - let step = 20; - for (let i = 0; i < promises.length; i += step) { - for (let j = i; j < (i + step && promises.length); j++) { - await new Promise((resolve, reject) => { - promises[j] - .then(async (res: number) => { - const register = new RegisterMessageInApp(res, mails[j], this.id); - await register.save(); - resolve(""); - }) - .catch((err) => { - reject(err); - }); - }); + for (let i = 0; i < nbMessageToSync; i += STEP) { + mails = []; + try { + // fetch mails + let secondUid = savedUid + STEP < currentUid ? savedUid + STEP : currentUid; + await this.mailFetcher(savedUid, secondUid, mails) + logger.log(`Fetched ${STEP} uids (${mails.length} messages)`); + // save same in the database + for (let k = 0; k < mails.length; k++) { + try { + const messageId = await saveMessage(mails[k], this.id, this.imap); + const register = new RegisterMessageInApp(messageId, mails[k], this.id); + await register.save(); + } catch (error) { + logger.err("Failed to save a message: " + error); + } } - logger.log(`Saved messages ${i + step > promises.length ? promises.length : i + step}/${mails.length}`); - updateMailbox(this.id, mails[i].uid); - } - updateMailbox(this.id, currentUid); - this.syncing = false; + savedUid = secondUid; + this.box.uidnext += savedUid; - // if has receive new msg during last sync then start a new sync - if (this.msgToSync > 0) { - const currentUid = this.box.uidnext; - this.box.uidnext += this.msgToSync; - // reset value to allow to detect new incoming message while syncing - this.msgToSync = 0; - this.syncMail(currentUid, this.box.uidnext); + updateMailbox(this.id, savedUid); + } catch (error) { + logger.err("Failed to sync message " + error); } + logger.log(`Saved messages ${i + STEP > nbMessageToSync ? nbMessageToSync : i + STEP}/${nbMessageToSync}`); + } + + // if has receive new msg during last sync then start a new sync + if (this.msgToSync > 0) { + const currentUid = this.box.uidnext; + this.box.uidnext += this.msgToSync; + // reset value to allow to detect new incoming message while syncing + this.msgToSync = 0; + await this.syncManager(currentUid, this.box.uidnext); + } + this.syncing = false; + logger.log(`Finished syncing messages`); + }; + + async mailFetcher(startUid: number, endUid: number, mails: Attrs[]): Promise { + return new Promise((resolve, reject) => { + const f = this.imap.seq.fetch(`${startUid}:${endUid}`, { + size: true, + envelope: true, + }); + + f.on("message", (msg, seqno) => { + msg.once("attributes", (attrs: AttrsWithEnvelope) => { + mails.push(attrs); + }); + }); + + f.once("error", (err) => { + logger.err("Fetch error: " + err); + reject(1); + }); + + f.once("end", async () => { + resolve(0); + }); }); }