save message working without reply

back/app/saveMessage.js

@@ -0,0 +1,92 @@
+const {
+    createRoom,
+    registerMessageInRoom,
+    createThread,
+    registerMessageInThread,
+    isRoomGroup,
+    findSpacesFromMessage,
+    hasSameMembersAsParent,
+} = require("../db/saveMessageApp");
+
+const { findRoomByOwner, getAddresseId } = require("../db/mail");
+
+/**
+ * take object address and join mailbox and host to return mailbox@host
+ */
+function createAddress(elt) {
+    return `${elt.mailbox}@${elt.host}`;
+}
+
+async function registerMessageInApp(messageId, attrs) {
+    const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify
+    const envelope = attrs.envelope;
+
+    await getAddresseId(createAddress(envelope.sender[0])).then(async (ownerId) => {
+        if (envelope.inReplyTo) {
+            await registerReplyMessage(envelope, messageId, isSeen);
+        } else {
+            await findRoomByOwner(ownerId).then(async (res) => {
+                if (res.length == 0) {
+                    await createRoom(envelope.subject, ownerId).then(async (roomId) => {
+                        await registerMessageInRoom(messageId, roomId, isSeen);
+                    });
+                } else {
+                    await registerMessageInRoom(messageId, res[0].room_id, isSeen);
+                }
+            });
+        }
+    });
+}
+
+async function registerReplyMessage(envelope, messageId, isSeen) {
+    const messageID = envelope.messageId;
+    await findSpacesFromMessage(messageId).then(async (spaces) => {
+        // todo sub thread will not be in index 0 so look in all indexes
+        if (spaces.length == 0) {
+            // no space, so is a transfer
+            // todo test if members of transferred message are included
+        } else if (spaces[0].thread_id) {
+            await registerMessageInThread(messageId, spaces[0].thread_id, isSeen);
+            // todo
+            // if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) {
+            //     // register new message in thread
+            //     // possibly convert to room only if parent is channel
+            // } else {
+            //     // todo create sub thread
+            // }
+        } else if (spaces[0].room_id) {
+            // message in room and not thread
+            await isRoomGroup(spaces[0].room_id).then(async (isGroup) => {
+                if (isGroup) {
+                    const hasSameMembers = await hasSameMembersAsParent(messageID, envelope.inReplyTo);
+                    if (hasSameMembers) {
+                        await registerMessageInRoom(messageId, spaces[0].room_id, isSeen);
+                    } else {
+                        // group and not the same member as the reply
+                        // some recipient has been removed create a thread
+                        const isDm = 0; // todo
+                        await createThread(space[0].room_id, envelope.subject, isSeen, isDm).then(async (threadId) => {
+                            await registerMessageInThread(messageId, threadId, isSeen);
+                        });
+                    }
+                } else {
+                    // reply from channel
+                    // todo
+                    // if (messageInRoom == 1) { // was new channel transform to group
+                    //     // register new message in group
+                    // } else if (sender == owner) { // correction from the original sender
+                    //     // leave in the same channel
+                    // } else { // user response to announcement
+                    //     // create new thread
+                    // }
+                }
+            });
+        }
+    });
+
+    //     `SELECT app_room_messages.room, app_room_messages.thread FROM app_room_messages INNER JOIN messages WHERE messages.messageID = '${envelope.inReplyTo}' AND app_room_messages.message = messages.id`;
+}
+
+module.exports = {
+    registerMessageInApp
+};

back/db/db.js

@@ -3,14 +3,14 @@ const MYSQL = require("./config.json").mysql;
 const DEBUG = require("../utils/debug.js").DEBUG;
 
 
-const bdd = mysql.createConnection({
+const db = mysql.createConnection({
     host: MYSQL.host,
     user: MYSQL.user,
     password: MYSQL.pwd,
     database: MYSQL.database,
 });
 
-bdd.connect(function (err) {
+db.connect(function (err) {
     if (err) {
         DEBUG.log("Impossible de se connecter", err.code);
     } else {
@@ -20,7 +20,7 @@ bdd.connect(function (err) {
 
 function execQueryAsync(query, values) {
     return new Promise((resolve, reject) => {
-        bdd.query(query, values, (err, results, fields) => {
+        db.query(query, values, (err, results, fields) => {
             if (err) {
                 reject(err);
             } else {
@@ -32,7 +32,7 @@ function execQueryAsync(query, values) {
 
 function execQueryAsyncWithId(query, values) {
     return new Promise((resolve, reject) => {
-        bdd.query(query, values, (err, results, fields) => {
+        db.query(query, values, (err, results, fields) => {
             if (err) {
                 reject(err);
             } else {
@@ -43,7 +43,7 @@ function execQueryAsyncWithId(query, values) {
 }
 
 function execQuery(query, values) {
-    bdd.query(query, values, (err, results, fields) => {
+    db.query(query, values, (err, results, fields) => {
         if (err) {
             DEBUG.log(err);
             throw (err);
@@ -53,7 +53,7 @@ function execQuery(query, values) {
 }
 
 module.exports = {
-    bdd,    // todo remove this
+    db,    // todo remove this
     execQuery,
     execQueryAsync,
     execQueryAsyncWithId

back/db/mail.js

@@ -1,4 +1,4 @@
-const { bdd, execQueryAsync, execQueryAsyncWithId } = require("./bdd.js");
+const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js");
 const DEBUG = require("../utils/debug").DEBUG;
 
 function isValidEmail(email) {

back/db/saveMessage.js

@@ -1,25 +1,32 @@
-const { bdd, execQuery, execQueryAsync, execQueryAsyncWithId } = require("./bdd.js");
+const { db, execQuery, execQueryAsync, execQueryAsyncWithId } = require("./db.js");
 const DEBUG = require("../utils/debug").DEBUG;
 
 async function registerMessage(timestamp, rfc822size, messageId) {
-    const query = `INSERT INTO message 
+    const query = `
+        INSERT INTO message 
             (idate, messageID, rfc822size) VALUES (?, ?, ?)
-        ON DUPLICATE KEY UPDATE message_id = LAST_INSERT_ID(message_id)`;
+            ON DUPLICATE KEY UPDATE message_id = LAST_INSERT_ID(message_id)
+        `;
     const values = [timestamp, messageId, rfc822size];
     return await execQueryAsyncWithId(query, values);
 }
 
 function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, deleted) {
-    const query = `INSERT IGNORE INTO mailbox_message (mailbox_id, uid, message_id, modseq, seen, deleted) VALUES (?, ?, ?, ?, ?, ?)`;
+    const query = `
+        INSERT IGNORE INTO mailbox_message 
+            (mailbox_id, uid, message_id, modseq, seen, deleted) VALUES (?, ?, ?, ?, ?, ?)
+        `;
     const values = [mailboxId, uid, messageId, modseq, seen, deleted];
     execQuery(query, values);
 }
 
 function registerBodypart(messageId, part, bodypartId, bytes, nbLines) {
-    const query = `INSERT IGNORE INTO part_number (message_id, part, bodypart_id, bytes, nb_lines) VALUES ('${messageId}', '${part}', '${bodypartId}', '${bytes}', '${nbLines}')`;
+    const query = `
-    bdd.query(query, (err, results, fields) => {
+        INSERT IGNORE INTO part_number
-        if (err) DEBUG.log(err);
+            (message_id, part, bodypart_id, bytes, nb_lines) VALUES (?, ?, ?, ?, ?)
-    });
+        `;
+    const values = [messageId, part, bodypartId, bytes, nbLines];
+    execQuery(query, values);
 }
 
 async function saveBodypart(bytes, hash, text, data) {
@@ -29,19 +36,30 @@ async function saveBodypart(bytes, hash, text, data) {
 }
 
 async function saveHeader_fields(messageId, fieldId, bodypartId, part, value) {
-    const query = `INSERT IGNORE INTO header_field (message_id, field_id, bodypart_id, part, value) VALUES (?, ?, ?, ?, ?)`;
+    const query = `
+        INSERT IGNORE INTO header_field
+            (message_id, field_id, bodypart_id, part, value) VALUES (?, ?, ?, ?, ?)
+        `;
     const values = [messageId, fieldId, bodypartId, part, value];
     return await execQueryAsync(query, values);
 }
 
 async function saveAddress_fields(messageId, fieldId, addressId, number) {
-    const query = `INSERT IGNORE INTO address_field (message_id , field_id, address_id, number) VALUES (?, ?, ?, ?)`;
+    const query = `
+        INSERT IGNORE INTO address_field
+            (message_id , field_id, address_id, number) VALUES (?, ?, ?, ?)
+        `;
     const values = [messageId, fieldId, addressId, number];
-    return execQueryAsync(query, values);
+    return await execQueryAsync(query, values);
 }
 
 function saveSource(messageId, content) {
-    // todo
+    const query = `
+        INSERT INTO source (message_id, content) VALUES (?, ?)
+            ON DUPLICATE KEY UPDATE content = ?
+        `;
+    const values = [messageId, content, content];
+    execQuery(query, values);
 }
 
 module.exports = {

back/db/saveMessageApp.js

@@ -1,4 +1,4 @@
-const { bdd, execQueryAsync, execQueryAsyncWithId } = require("./bdd.js");
+const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js");
 const DEBUG = require("../utils/debug").DEBUG;
 
 async function createRoom(roomName, ownerId) {
@@ -8,11 +8,10 @@ async function createRoom(roomName, ownerId) {
     // todo add members
 }
 
-function registerMessageInRoom(messageId, roomId, isSeen) {
+async function registerMessageInRoom(messageId, roomId, isSeen) {
-    const query = `INSERT INTO app_space_message (message_id, room_id) VALUES ('${messageId}', '${roomId}')`;
+    const query = `INSERT INTO app_space_message (message_id, room_id) VALUES (?, ?)`;
-    bdd.query(query, (err, results, fields) => {
+    const values = [messageId, roomId];
-        if (err) throw err;
+    await execQueryAsync(query, values);
-    });
 
     updateLastUpdateRoom(roomId);
 
@@ -29,31 +28,19 @@ function incrementNotSeenRoom(roomId) {
     // todo
 }
 
-function createThread(roomId, threadName, isDm) {
+async function createThread(roomId, threadName, isDm) {
-    return new Promise((resolve, reject) => {
+    const query = `INSERT INTO app_thread (room_id, thread_name, isDm) VALUES (?, ?, ?)`;
-        const query = `INSERT INTO app_thread 
+    const values = [roomId, threadName, isDm];
-            (room_id, thread_name, isDm)
+    return await execQueryAsync(query, values);
-            VALUES (
-                '${roomId}',
-                '${threadName}',
-                '${isDm}',
-            )`;
-        bdd.query(query, (err, results, fields) => {
-            if (err) reject(err);
-            resolve(results.insertId);
-        });
-    });
-
     // todo add members
 }
 
-function registerMessageInThread(messageId, threadId, isSeen) {
+async function registerMessageInThread(messageId, threadId, isSeen) {
     // todo check if it is still a thread or should be a room
     const query = `INSERT IGNORE INTO app_space_message
-        (message_id, thread_id) VALUES ('${messageId}', '${threadId}')`;
+        (message_id, thread_id) VALUES (?, ?)`;
-    bdd.query(query, (err, results, fields) => {
+    const values = [messageId, threadId];
-        if (err) throw err;
+    await execQueryAsync(query, values);
-    });
     updateLastUpdateThread(threadId);
 
     if (!isSeen) {
@@ -71,10 +58,10 @@ function incrementNotSeenThread(threadId) {
     // also increment parent room
 }
 
-function isRoomGroup(roomId) {
+async function isRoomGroup(roomId) {
     return new Promise((resolve, reject) => {
         const query = `SELECT isGroup FROM app_room WHERE room_id = '${roomId}'`;
-        bdd.query(query, (err, results, fields) => {
+        db.query(query, (err, results, fields) => {
             if (err) reject(err);
             resolve(results[0].isGroup);
         });
@@ -98,5 +85,5 @@ module.exports = {
     createThread,
     registerMessageInThread,
     isRoomGroup,
-    findSpacesFromMessage
+    findSpacesFromMessage,
 };

back/db/structureV2.sql

@@ -145,6 +145,7 @@ CREATE TABLE app_space_message (
   message_id      INT NOT NULL,
   room_id         INT,
   thread_id       INT,
+  UNIQUE KEY (member_id, room_id, thread_id),
   FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE,
   FOREIGN KEY (room_id) REFERENCES app_room(room_id) ON DELETE SET NULL,
   FOREIGN KEY (thread_id) REFERENCES app_thread(thread_id) ON DELETE SET NULL

back/imap/index.js

@@ -2,9 +2,11 @@ const Imap = require("imap");
 const { simpleParser } = require("mailparser");
 const inspect = require("util").inspect;
 const saveMessage = require("./storeMessage").saveMessage;
+const registerMessageInApp = require("../app/saveMessage").registerMessageInApp;
 const imapConfig = require("./config.json").mail;
 
 const fs = require("fs");
+const { DEBUG } = require("../utils/debug");
 const imap = new Imap({
     user: imapConfig.user,
     password: imapConfig.password,
@@ -20,55 +22,54 @@ let shouldReset = false;
 // let shouldReset = true;
 
 if (shouldReset) {
-
+    const { execQuery, execQueryAsync } = require("../db/db");
-    
+    const query = "SELECT table_name FROM INFORMATION_SCHEMA.tables WHERE table_schema = 'mail'";
-    const { execQuery, execQueryAsync } = require("../sql/bdd");
-    const query =
-        "SELECT table_name FROM INFORMATION_SCHEMA.tables WHERE table_schema = 'mail'";
     execQueryAsync(query).then((results) => {
         execQuery("SET FOREIGN_KEY_CHECKS=0");
         results.map((table) => {
             execQuery("DELETE FROM " + table.table_name);
-    
         });
     });
     return;
 }
 
-
 imap.once("ready", function () {
     const readOnly = true;
     imap.openBox("INBOX", readOnly, (err, box) => {
         // console.log(box); // uidvalidty uidnext, messages total and new
-        imap.search(["ALL"], function (err, results) {
+        // imap.search(["ALL"], function (err, results) {
-            console.log(results[results.length - 1]);
+        //     console.log(results[results.length - 1]);
-        });
-
-        const f = imap.fetch(970, {
-            size: true,
-            envelope: true,
-        });
-        let messageIDs = [];
-        // var f = imap.seq.fetch('1:3', {
-        //     bodies: 'HEADER.FIELDS (FROM TO SUBJECT DATE)',
-        //     struct: true
         // });
+
+        // const f = imap.fetch(970, {
+        //     size: true,
+        //     envelope: true,
+        // });
+        const promises = [];
+        const mails = [];
+        var f = imap.seq.fetch('1:10', {
+            size: true,
+            envelope: true
+          });
         f.on("message", function (msg, seqno) {
             msg.once("attributes", (attrs) => {
                 // todo find boxId
                 const boxId = 1;
-                messageIDs.push(attrs.envelope.messageId)
+                mails.push(attrs);
-                saveMessage(attrs, boxId, imap);
+                promises.push(saveMessage(attrs, boxId, imap));
             });
         });
 
         f.once("error", function (err) {
-            console.log("Fetch error: " + err);
+            DEBUG.log("Fetch error: " + err);
+        });
+        f.once("end", async function () {
+            Promise.all(promises).then(async (res) => {
+                DEBUG.log("Done fetching all messages!");
+                for (let i = 0; i < mails.length; i++) {
+                    await registerMessageInApp(res[i], mails[i]);
+                }
             });
-        f.once("end", function () {
-            console.log(messageIDs)
-            isDone = true;
-            console.log("Done fetching all messages!");
         });
         //    imap.end()
     });

back/imap/storeMessage.js

@@ -1,4 +1,4 @@
-const { getAddresseId } = require("../sql/mail");
+const { getAddresseId } = require("../db/mail");
 const { DEBUG } = require("../utils/debug");
 const { simpleParser } = require("mailparser");
 const moment = require("moment");
@@ -11,40 +11,23 @@ const {
     registerBodypart,
     saveBodypart,
     saveSource,
-} = require("../sql/saveMessage");
+} = require("../db/saveMessage");
 
-const {
+const { getFieldId } = require("../db/mail");
-    createRoom,
-    registerMessageInRoom,
-    createThread,
-    registerMessageInThread,
-    isRoomGroup,
-    findSpacesFromMessage,
-    hasSameMembersAsParent,
-} = require("../sql/saveMessageApp");
-const { getFieldId, findRoomByOwner } = require("../sql/mail");
 
 function saveMessage(attrs, mailboxId, imap) {
     const envelope = attrs.envelope;
-    const ts = moment(new Date(envelope.date).getTime()).format(
+    const ts = moment(new Date(envelope.date).getTime()).format("YYYY-MM-DD HH:mm:ss");
-        "YYYY-MM-DD HH:mm:ss"
-    );
     const rfc822size = attrs.size;
     const messageID = envelope.messageId;
 
+    return new Promise((resolve, reject) => {
         registerMessage(ts, rfc822size, messageID)
             .then((messageId) => {
                 const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify
                 const deleted = attrs.flags.includes("Deleted") ? 1 : 0; // todo verify
 
-            registerMailbox_message(
+                registerMailbox_message(mailboxId, attrs.uid, messageId, attrs.modseq, isSeen, deleted);
-                mailboxId,
-                attrs.uid,
-                messageId,
-                attrs.modseq,
-                isSeen,
-                deleted
-            );
                 const f = imap.fetch(attrs.uid, { bodies: "" });
                 let buffer = "";
 
@@ -60,7 +43,13 @@ function saveMessage(attrs, mailboxId, imap) {
 
                             // parse data
                             simpleParser(buffer, async (err, parsed) => {
-                            saveFromParsedData(parsed, messageId);
+                                saveFromParsedData(parsed, messageId)
+                                    .then(() => {
+                                        resolve(messageId);
+                                    })
+                                    .catch((err) => {
+                                        reject(err);
+                                    });
                             });
                         });
                     });
@@ -69,37 +58,37 @@ function saveMessage(attrs, mailboxId, imap) {
                     console.log("Fetch error: " + err);
                 });
                 f.once("end", function () {
-                console.log("Done fetching all messages!");
+                    DEBUG.log("Done fetching data of "+messageID);
                 });
             })
             .catch((err) => {
                 DEBUG.log("Unable to register message: " + err);
+                reject(err);
+            });
     });
 }
 
-function saveFromParsedData(parsed, messageId) {
+async function saveFromParsedData(parsed, messageId) {
-    fs.writeFileSync("./test.txt", JSON.stringify(parsed));
+    const promises = [];
     Object.keys(parsed).forEach((key) => {
         if (["from", "to", "cc", "bcc", "reply-to"].includes(key)) {
+            promises.push(
                 // save address field
                 getFieldId(key).then((fieldId) => {
                     parsed[key].value.forEach((addr, nb) => {
-                    getAddresseId(addr.address, addr.name).then((addressId) => {
+                        getAddresseId(addr.address, addr.name).then(async (addressId) => {
-                        saveAddress_fields(messageId, fieldId, addressId, nb);
+                            await saveAddress_fields(messageId, fieldId, addressId, nb);
-                    });
                         });
                     });
+                }),
+            );
         } else if (["subject", "inReplyTo"].includes(key)) {
             // todo :  "references"
-            getFieldId(key).then((fieldId) => {
+            promises.push(
-                saveHeader_fields(
+                getFieldId(key).then(async (fieldId) => {
-                    messageId,
+                    await saveHeader_fields(messageId, fieldId, undefined, undefined, parsed[key]);
-                    fieldId,
+                }),
-                    undefined,
-                    undefined,
-                    parsed[key]
             );
-            });
         } else if (["html", "text", "textAsHtml"].includes(key)) {
             const hash = "0";
             const size = "0";
@@ -116,9 +105,7 @@ function saveFromParsedData(parsed, messageId) {
             // });
         } else if (key == "attachments") {
             // todo
-        } else if (
+        } else if (["date", "messageId", "headers", "headerLines"].includes(key)) {
-            ["date", "messageId", "headers", "headerLines"].includes(key)
-        ) {
             // messageId and date are already saved
             // other field are not improted and can be retrieved in source
             return;
@@ -127,104 +114,10 @@ function saveFromParsedData(parsed, messageId) {
             return;
         }
     });
+    return Promise.all(promises);
     // todo when transfered
 }
 
-function haveSameReceivers() {
-    // take cc and to
-}
-
-/**
- * take object address and join mailbox and host to return mailbox@host
- */
-function createAddress(elt) {
-    return `${elt.mailbox}@${elt.host}`;
-}
-
-function registerMessageInApp(envelope, messageId, isSeen) {
-    getAddresseId(createAddress(envelope.sender[0])).then((ownerId) => {
-        if (envelope.inReplyTo) {
-            registerReplyMessage(envelope, messageId, isSeen);
-        } else {
-            findRoomByOwner(ownerId).then((res) => {
-                if (res.length == 0) {
-                    createRoom(envelope.subject, ownerId).then((roomId) => {
-                        registerMessageInRoom(messageId, roomId, isSeen);
-                    });
-                } else {
-                    registerMessageInRoom(messageId, res[0].room_id, isSeen);
-                }
-            });
-        }
-    });
-}
-
-function registerReplyMessage(envelope, messageId, isSeen) {
-    const messageID = envelope.messageId;
-    findSpacesFromMessage(messageId).then((spaces) => {
-        // todo sub thread will not be in index 0 so look in all indexes
-        if (spaces.length == 0) {
-            // no space, so is a transfer
-            // todo test if members of transferred message are included
-        } else if (spaces[0].thread_id) {
-            registerMessageInThread(messageId, spaces[0].thread_id, isSeen);
-            // todo
-            // if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) {
-            //     // register new message in thread
-            //     // possibly convert to room only if parent is channel
-            // } else {
-            //     // todo create sub thread
-            // }
-        } else if (spaces[0].room_id) {
-            // message in room and not thread
-            isRoomGroup(spaces[0].room_id).then((isGroup) => {
-                if (isGroup) {
-                    if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) {
-                        registerMessageInRoom(
-                            messageId,
-                            spaces[0].room_id,
-                            isSeen
-                        );
-                    } else {
-                        // group and not the same member as the reply
-                        // some recipient has been removed create a thread
-                        const isDm = 0; // todo
-                        createThread(
-                            space[0].room_id,
-                            envelope.subject,
-                            isSeen,
-                            isDm
-                        ).then((threadId) => {
-                            registerMessageInThread(
-                                messageId,
-                                threadId,
-                                isSeen
-                            );
-                        });
-                    }
-                } else {
-                    // reply from channel
-                    // todo
-                    // if (messageInRoom == 1) { // was new channel transform to group
-                    //     // register new message in group
-                    // } else if (sender == owner) { // correction from the original sender
-                    //     // leave in the same channel
-                    // } else { // user response to announcement
-                    //     // create new thread
-                    // }
-                }
-            });
-        }
-    });
-
-    //     `SELECT app_room_messages.room, app_room_messages.thread FROM app_room_messages INNER JOIN messages WHERE messages.messageID = '${envelope.inReplyTo}' AND app_room_messages.message = messages.id`;
-}
-
-function keyNormalizer(key) {
-    // todo
-    return key;
-}
-
 module.exports = {
     saveMessage,
 };

back/utils/debug.js

@@ -1,9 +1,9 @@
-const DEBUG = (function() {
+const DEBUG = (function () {
-    const timestamp = function(){};
+    const timestamp = function () {};
-    timestamp.toString = () => "[DEBUG " + new Date().toLocaleDateString() + "]";
+    timestamp.toString = () => "[" + new Date().toLocaleString() + "]";
     return { log: console.log.bind(console, "%s", timestamp) };
 })();
 
 module.exports = {
-  DEBUG
+    DEBUG,
-}
+};