advancements in tests and storing messages

back/mails/index.js

@@ -0,0 +1,88 @@
+const Imap = require("imap");
+const { simpleParser } = require("mailparser");
+const inspect = require("util").inspect;
+const saveMessage = require("./storeMessage").saveMessage;
+const registerMessageInApp = require("./saveMessage").registerMessageInApp;
+const imapConfig = require("./config.json").mail;
+
+const fs = require("fs");
+const { DEBUG } = require("../utils/debug");
+const imap = new Imap({
+    user: imapConfig.user,
+    password: imapConfig.password,
+    tlsOptions: { servername: "imap.gmail.com" },
+    host: "imap.gmail.com",
+    port: 993,
+    tls: true,
+});
+
+// reset table;
+
+let shouldReset = false;
+// let shouldReset = true;
+
+if (shouldReset) {
+    const { execQuery, execQueryAsync } = require("../db/db");
+    const query = "SELECT table_name FROM INFORMATION_SCHEMA.tables WHERE table_schema = 'mail'";
+    execQueryAsync(query).then((results) => {
+        execQuery("SET FOREIGN_KEY_CHECKS=0");
+        results.map((table) => {
+            execQuery("DELETE FROM " + table.table_name);
+            // execQuery("DROP TABLE " + table.table_name);
+        });
+    });
+    return;
+}
+
+imap.once("ready", function () {
+    const readOnly = true;
+    imap.openBox("INBOX", readOnly, (err, box) => {
+        // console.log(box); // uidvalidty uidnext, messages total and new
+        // imap.search(["ALL"], function (err, results) {
+        //     console.log(results[results.length - 1]);
+        // });
+
+        // const f = imap.fetch(970, {
+        //     size: true,
+        //     envelope: true,
+        // });
+        const promises = [];
+        const mails = [];
+        const f = imap.seq.fetch('1:10', {
+            size: true,
+            envelope: true
+          });
+        f.on("message", function (msg, seqno) {
+            msg.once("attributes", (attrs) => {
+                // todo find boxId
+                const boxId = 1;
+                console.log(attrs.envelope)
+                // mails.push(attrs);
+                // promises.push(saveMessage(attrs, boxId, imap));
+            });
+        });
+
+        f.once("error", function (err) {
+            DEBUG.log("Fetch error: " + err);
+        });
+        f.once("end", async function () {
+            Promise.all(promises).then(async (res) => {
+                DEBUG.log("Done fetching all messages!");
+                for (let i = 0; i < mails.length; i++) {
+                    await registerMessageInApp(res[i], mails[i]);
+                }
+            });
+        });
+        //    imap.end()
+    });
+});
+
+imap.once("error", function (err) {
+    console.log(err);
+});
+
+imap.once("end", function () {
+    console.log("Connection ended");
+});
+
+imap.connect();

back/mails/saveMessage.js

@@ -0,0 +1,95 @@
+const {
+    createRoom,
+    registerMessageInRoom,
+    createThread,
+    registerMessageInThread,
+    isRoomGroup,
+    findRoomsFromMessage,
+    hasSameMembersAsParent,
+} = require("../db/saveMessageApp");
+
+const { findRoomByOwner, getAddresseId } = require("../db/mail");
+const { isDmOnEnvelope } = require("./utils/statusUtils");
+
+/**
+ * take object address and join mailbox and host to return mailbox@host
+ */
+function createAddress(elt) {
+    return `${elt.mailbox}@${elt.host}`;
+}
+
+async function registerMessageInApp(messageId, attrs) {
+    const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify
+    const envelope = attrs.envelope;
+
+    await getAddresseId(createAddress(envelope.sender[0])).then(async (ownerId) => {
+        if (envelope.inReplyTo) {
+            await registerReplyMessage(envelope, messageId, isSeen, ownerId);
+        } else {
+            await findRoomByOwner(ownerId).then(async (res) => {
+                if (res.length == 0) {
+                    // first message of this sender
+                    await createRoom(envelope.subject, ownerId, messageId).then(async (roomId) => {
+                        await registerMessageInRoom(messageId, roomId, isSeen);
+                    });
+                } else {
+                    // not a reply, add to the list of message if this sender
+                    await registerMessageInRoom(messageId, res[0].room_id, isSeen);
+                }
+            });
+        }
+    });
+}
+
+async function registerReplyMessage(envelope, messageId, isSeen, ownerId) {
+    const messageID = envelope.messageId;
+    await findRoomsFromMessage(messageId).then(async (rooms) => {
+        if (rooms.length == 0) {
+            // no rooms, so is a transfer
+            // todo test if members of transferred message are included
+        } else if (rooms.length === 1) {
+            // only one room so message is only in a room and not in a thread
+            // as a thread is associated to a room to begin
+            await isRoomGroup(rooms[0].room_id).then(async (isGroup) => {
+                if (isGroup) {
+                    const hasSameMembers = await hasSameMembersAsParent(messageID, envelope.inReplyTo);
+                    if (hasSameMembers) {
+                        await registerMessageInRoom(messageId, rooms[0].room_id, isSeen);
+                    } else {
+                        // is a group and has not the same member as the previous message
+                        // some recipient has been removed create a thread
+                        const isDm = isDmOnEnvelope(envelope);
+                        await createThread(envelope.subject, ownerId, messageId, rooms[0].room_id, isDm).then(async (threadId) => {
+                            await registerMessageInThread(messageId, threadId, isSeen);
+                        });
+                    }
+                } else {
+                    // reply from channel
+                    // todo
+                    // if (sender == owner) { // correction from the original sender
+                    //     // leave in the same channel
+                    // }
+                }
+            });
+        } else if (rooms.length > 1) {
+            // get the lowest thread (order by room_id)
+            const room = rooms[rooms.length-1];
+            const hasSameMembers = await hasSameMembersAsParent(messageID, envelope.inReplyTo);
+            if (hasSameMembers) {
+                await registerMessageInThread(messageId, room.room_id, isSeen);
+            } else {
+                // has not the same members so it is a derivation of this thread
+                // todo put this in a function and add default message in the reply chain
+                const isDm = isDmOnEnvelope(envelope);
+                await createThread(envelope.subject, ownerId, messageId, room.room_id, isDm).then(async (threadId) => {
+                    await registerMessageInThread(messageId, threadId, isSeen);
+                });
+            }
+        }
+    });
+}
+
+module.exports = {
+    registerMessageInApp
+};
+

back/mails/storeMessage.js

@@ -0,0 +1,129 @@
+const { getAddresseId } = require("../db/mail");
+const { DEBUG } = require("../utils/debug");
+const { simpleParser } = require("mailparser");
+const moment = require("moment");
+const fs = require("fs");
+const {
+    registerMessage,
+    registerMailbox_message,
+    saveHeader_fields,
+    saveAddress_fields,
+    registerBodypart,
+    saveBodypart,
+    saveSource,
+} = require("../db/saveMessage");
+
+const { getFieldId } = require("../db/mail");
+
+function saveMessage(attrs, mailboxId, imap) {
+    const envelope = attrs.envelope;
+    const ts = moment(new Date(envelope.date).getTime()).format("YYYY-MM-DD HH:mm:ss");
+    const rfc822size = attrs.size;
+    const messageID = envelope.messageId;
+
+    return new Promise((resolve, reject) => {
+        registerMessage(ts, rfc822size, messageID)
+            .then((messageId) => {
+                const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify
+                const deleted = attrs.flags.includes("Deleted") ? 1 : 0; // todo verify
+
+                registerMailbox_message(mailboxId, attrs.uid, messageId, attrs.modseq, isSeen, deleted);
+                const f = imap.fetch(attrs.uid, { bodies: "" });
+                let buffer = "";
+
+                f.on("message", function (msg, seqno) {
+                    msg.on("body", function (stream, info) {
+                        stream.on("data", function (chunk) {
+                            buffer += chunk.toString("utf8");
+                        });
+
+                        stream.once("end", () => {
+                            // save raw data
+                            saveSource(messageId, buffer);
+
+                            // parse data
+                            simpleParser(buffer, async (err, parsed) => {
+                                saveFromParsedData(parsed, messageId)
+                                    .then(() => {
+                                        resolve(messageId);
+                                    })
+                                    .catch((err) => {
+                                        reject(err);
+                                    });
+                            });
+                        });
+                    });
+                });
+                f.once("error", function (err) {
+                    console.log("Fetch error: " + err);
+                });
+                f.once("end", function () {
+                    DEBUG.log("Done fetching data of "+messageID);
+                });
+            })
+            .catch((err) => {
+                DEBUG.log("Unable to register message: " + err);
+                reject(err);
+            });
+    });
+}
+
+async function saveFromParsedData(parsed, messageId) {
+    const promises = [];
+    Object.keys(parsed).forEach((key) => {
+        if (["from", "to", "cc", "bcc", "replyTo"].includes(key)) {
+            promises.push(
+                // save address field
+                getFieldId(key).then((fieldId) => {
+                    parsed[key].value.forEach((addr, nb) => {
+                        getAddresseId(addr.address, addr.name).then(async (addressId) => {
+                            await saveAddress_fields(messageId, fieldId, addressId, nb);
+                        });
+                    });
+                }),
+            );
+        } else if (["subject", "inReplyTo"].includes(key)) {
+            // todo :  "references" (array)
+            promises.push(
+                getFieldId(key).then(async (fieldId) => {
+                    await saveHeader_fields(messageId, fieldId, undefined, undefined, parsed[key]);
+                }),
+            );
+        } else if (["html", "text", "textAsHtml"].includes(key)) {
+            const hash = "0";
+            const size = "0";
+            saveBodypart(size, hash, parsed[key], "").then((bodypartId) => {
+                getFieldId(key).then((fieldId) => {
+                    saveHeader_fields(
+                        messageId,
+                        fieldId,
+                        bodypartId,
+                        undefined,  // todo ?
+                        undefined
+                    );
+                });
+            });
+        } else if (key == "attachments") {
+            // todo
+        } else if (["date", "messageId", "headers", "headerLines"].includes(key)) {
+            // messageId and date are already saved
+            // other field are not important and can be retrieved in source
+            return;
+        } else {
+            DEBUG.log("doesn't know key: " + key);
+            return;
+        }
+    });
+    return Promise.all(promises);
+    // todo when transfered
+}
+
+module.exports = {
+    saveMessage,
+};
+
+if (process.env['NODE_DEV'] == 'TEST') {
+    module.exports = {
+        saveFromParsedData
+    };
+}

back/mails/utils/statusUtils.js

@@ -0,0 +1,13 @@
+function isDmOnEnvelope(envelope) {
+    const members =
+        envelope.bcc?.length +
+        envelope.cc?.length +
+        envelope.to?.length +
+        envelope.sender?.length +
+        envelope.from?.length;
+    return members === 2;
+}
+
+module.exports = {
+    isDmOnEnvelope,
+};