link imap sync to server and show email on front

back/controllers/addAccount.js

@@ -1,11 +1,11 @@
 const statusCode = require("../utils/statusCodes").statusCodes;
-const { registerMailbox } = require("../db/api");
+const { registerAccount } = require("../db/api");
 const { getAddresseId } = require("../db/mail");
 
-async function addMailbox(body, res) {
+async function addAccount(body, res) {
     const { email, pwd, xoauth, xoauth2, host, port, tls } = body;
     getAddresseId(email).then((addressId) => {
-        registerMailbox(addressId, pwd, xoauth, xoauth2, host, port, tls)
+        registerAccount(addressId, pwd, xoauth, xoauth2, host, port, tls)
             .then((mailboxId) => {
                 res.status(statusCode.OK).json({ id: mailboxId });
             })
@@ -16,5 +16,7 @@ async function addMailbox(body, res) {
 }
 
 module.exports = {
-    addMailbox,
+    addAccount,
 };
+
+// todo change mailbox to account

back/db/api.js

@@ -2,7 +2,7 @@ const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js");
 const { queryCcId, queryToId, queryFromId } = require("./utils/addressQueries.js");
 const DEBUG = require("../utils/debug").DEBUG;
 
-async function registerMailbox(userId, pwd, xoauth, xoauth2, host, port, tls) {
+async function registerAccount(userId, pwd, xoauth, xoauth2, host, port, tls) {
     const query = `
         INSERT INTO app_account
             (user_id, account_pwd, xoauth, xoauth2, host, port, tls) VALUES (?, ?, ?, ?, ?, ?, ?)
@@ -11,13 +11,18 @@ async function registerMailbox(userId, pwd, xoauth, xoauth2, host, port, tls) {
     return await execQueryAsyncWithId(query, values);
 }
 
-async function getMailboxes() {
+async function getAccounts() {
+    // todo mailbox or account id ?
     const query = `
         SELECT 
-            app_account.account_id AS id,
+            mailbox.mailbox_id AS id,
             address.email
-        FROM app_account INNER JOIN address
-        WHERE address.address_id = app_account.user_id
+        FROM app_account
+        INNER JOIN address
+        INNER JOIN mailbox
+        WHERE 
+            address.address_id = app_account.user_id AND
+            mailbox.account_id = app_account.account_id
         `;
     const values = [];
     return await execQueryAsync(query, values);
@@ -38,12 +43,11 @@ async function getRooms(mailboxId) {
             INNER JOIN address
             WHERE 
                 message.message_id = app_room.message_id AND
-                mailbox_message.mailbox_id = 1 AND 
+                mailbox_message.mailbox_id = ? AND 
                 mailbox_message.message_id = message.message_id AND
                 address.address_id = app_room.owner_id
         `;
-    // todo mailboxId
-    const values = [];
+    const values = [mailboxId];
     return await execQueryAsync(query, values);
 }
 
@@ -92,8 +96,8 @@ async function getMessages(roomId) {
 }
 
 module.exports = {
-    registerMailbox,
-    getMailboxes,
+    registerAccount,
+    getAccounts,
     getRooms,
     getMessages
 };

back/db/imap/imap.js

@@ -0,0 +1,49 @@
+const { execQueryAsyncWithId, execQueryAsync, execQuery } = require("../db");
+
+async function getAllAccounts() {
+    const query = `
+        SELECT
+            app_account.account_id AS id,
+            address.email AS user,
+            app_account.account_pwd AS password,
+            app_account.host AS host,
+            app_account.port AS port,
+            app_account.tls AS tls
+        FROM app_account INNER JOIN address
+        WHERE address.address_id = app_account.user_id
+    `;
+    const values = [];
+    return await execQueryAsync(query, values);
+}
+
+async function getAllMailboxes(accountId) {
+    const query = 'SELECT * FROM mailbox WHERE mailbox.account_id = ?';
+    const values = [accountId];
+    return await execQueryAsync(query, values)
+}
+
+async function registerMailbox(accountId, mailboxName) {
+    const query = `INSERT INTO mailbox (account_id, mailbox_name) VALUES (?, ?)`;
+    const values = [accountId, mailboxName];
+    return await execQueryAsyncWithId(query, values);
+}
+
+async function getMailbox(mailboxId) {
+    const query = `SELECT * FROM mailbox WHERE mailbox_id = ?`;
+    const values = [mailboxId];
+    return await execQueryAsync(query, values);
+}
+
+function updateMailbox(mailboxId, uidnext) {
+    const query = `UPDATE mailbox SET uidnext = ? WHERE mailbox_id = ?`;
+    const values = [uidnext, mailboxId];
+    execQuery(query, values);
+}
+
+module.exports = {
+    getAllAccounts,
+    getAllMailboxes,
+    registerMailbox,
+    getMailbox,
+    updateMailbox
+}

back/db/mail.js

@@ -1,19 +1,14 @@
-const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js");
+const { execQueryAsync, execQueryAsyncWithId } = require("./db.js");
 const DEBUG = require("../utils/debug").DEBUG;
 
-function isValidEmail(email) {
-    // todo
-    return true;
-}
-
 
 async function getAddresseId(email, name) {
     const localpart = email.split("@")[0];
     const domain = email.split("@")[1];
     const query = `INSERT INTO address 
                     (address_name, localpart, domain, email) VALUES (?, ?, ?, ?)
-                    ON DUPLICATE KEY UPDATE email = ?, address_id = LAST_INSERT_ID(address_id)`;
-    const values = [name, localpart, domain, email, email];
+                    ON DUPLICATE KEY UPDATE address_name = ?, address_id = LAST_INSERT_ID(address_id)`;
+    const values = [name, localpart, domain, email, name];
     return await execQueryAsyncWithId(query, values);
 }
 

back/db/saveMessage.js

@@ -17,7 +17,6 @@ function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, delete
             (mailbox_id, uid, message_id, modseq, seen, deleted) VALUES (?, ?, ?, ?, ?, ?)
         `;
     const values = [mailboxId, uid, messageId, modseq, seen, deleted];
-    console.log(values)
     execQuery(query, values);
 }
 
@@ -55,6 +54,7 @@ async function saveAddress_fields(messageId, fieldId, addressId, number) {
 }
 
 function saveSource(messageId, content) {
+    content = Buffer.from(content);
     const query = `
         INSERT INTO source (message_id, content) VALUES (?, ?)
             ON DUPLICATE KEY UPDATE content = ?

back/db/structureV2.sql

@@ -67,7 +67,7 @@ CREATE TABLE bodypart (
   bodypart_id   INT AUTO_INCREMENT,
   bytes         INT NOT NULL,
   hash          TEXT NOT NULL,
-  text          TEXT,
+  text          TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci,
   data          BINARY,
   PRIMARY KEY (bodypart_id)
 );
@@ -75,10 +75,10 @@ CREATE TABLE bodypart (
 -- 7
 CREATE TABLE source (
   message_id    INT NOT NULL,
-  content       TEXT NOT NULL,
+  content       BLOB NOT NULL,
   PRIMARY KEY (message_id),
   FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE
-);
+)
 
 -- 8
 CREATE TABLE field_name (
@@ -94,8 +94,9 @@ CREATE TABLE header_field (
   field_id        INT NOT NULL,
   bodypart_id     INT,
   part            VARCHAR(128),
-  value           TEXT,
-  UNIQUE KEY (message_id, field_id, bodypart_id),-- todo multiple raws
+  value           TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci,
+  UNIQUE KEY (message_id, field_id, bodypart_id),
+  UNIQUE KEY (message_id, field_id, part),
   FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE,
   FOREIGN KEY (field_id) REFERENCES field_name(field_id) ON DELETE CASCADE,
   FOREIGN KEY (bodypart_id) REFERENCES bodypart(bodypart_id)

back/mails/imap/Box.js

@@ -0,0 +1,59 @@
+const { getMailbox, updateMailbox } = require("../../db/imap/imap");
+const { DEBUG } = require("../../utils/debug");
+const { registerMessageInApp } = require("../saveMessage");
+const { saveMessage } = require("../storeMessage");
+
+class Box {
+    constructor(_imap, boxId, _boxName) {
+        this.imap = _imap;
+        this.boxName = _boxName;
+        this.id = boxId;
+        this.box;
+        this.init();
+    }
+
+    async init() {
+        this.box = (await getMailbox(this.id))[0];
+
+        const readOnly = true;
+        this.imap.openBox(this.boxName, readOnly, (err, box) => {
+            if (err) DEBUG.log(err);
+            this.sync(this.box.uidnext, box.uidnext);
+        });
+    }
+
+    sync(savedUid, currentUid) {
+        const promises = [];
+        const mails = [];
+        console.log(savedUid, currentUid);
+        const f = this.imap.seq.fetch(`${savedUid}:${currentUid}`, {
+            size: true,
+            envelope: true,
+        });
+
+        f.on("message", (msg, seqno) => {
+            msg.once("attributes", (attrs) => {
+                mails.push(attrs);
+                promises.push(saveMessage(attrs, this.id, this.imap));
+            });
+        });
+
+        f.once("error", (err) => {
+            DEBUG.log("Fetch error: " + err);
+        });
+
+        f.once("end", async () => {
+            await Promise.all(promises).then(async (res) => {
+                for (let i = 0; i < mails.length; i++) {
+                    console.log(i, mails[i].uid)
+                    await registerMessageInApp(res[i], mails[i]);
+                }
+            });
+            updateMailbox(this.id, currentUid);
+        });
+    }
+}
+
+module.exports = {
+    Box,
+};

back/mails/imap/ImapInstance.js

@@ -0,0 +1,73 @@
+const Imap = require("imap");
+const { getAllMailboxes, registerMailbox } = require("../../db/imap/imap");
+const { DEBUG } = require("../../utils/debug");
+const { Box } = require("./Box");
+
+class ImapInstance {
+    constructor(account) {
+        this.imap = new Imap({
+            user: account.user,
+            password: account.password,
+            tlsOptions: { servername: account.host },
+            host: account.host,
+            port: account.port,
+            tls: account.tls,
+        });
+    
+        this.account = account;
+        this.boxes = [];
+
+        /**
+         * IMAP
+         */
+        this.imap.once("ready", () => {
+            DEBUG.log("imap connected")
+            this.imapReady();
+        });
+
+        this.imap.once("error", function (err) {
+            DEBUG.log(err);
+        });
+        
+        this.imap.once("end", function () {
+            DEBUG.log("Connection ended");
+        });
+
+        this.imap.connect();
+    }
+
+    imapReady() {
+        getAllMailboxes(this.account.id).then((mailboxes) => {
+            if (mailboxes.length > 0) {
+                this.boxes.push(new Box(this.imap, mailboxes[0].mailbox_id, mailboxes[0].mailbox_name));
+            } else {
+                this.imap.getBoxes('', (err, boxes) => {
+                    if (err) DEBUG.log(err);
+                    const allBoxName = this.getAllBox(boxes);
+                    registerMailbox(this.account.id, allBoxName).then((mailboxId) => {
+                        this.boxes.push(new Box(this.imap, mailboxId, allBoxName));
+                    });
+                });
+            }
+        });
+    }
+
+    getAllBox(boxes) {
+        // ideally we should get the all box to get all messages
+        let allBox;
+        Object.keys(boxes).forEach(key => {
+            if (key === 'INBOX') return;
+            allBox = key;
+            Object.keys(boxes[key].children).forEach((childBoxes) => {
+                if (boxes[key].children[childBoxes].attribs.includes('\\All')) {
+                    allBox += '/' + childBoxes;
+                }
+            });
+        });
+        return allBox;
+    }
+}
+
+module.exports = {
+    ImapInstance
+}

back/mails/imap/ImapSync.js

@@ -0,0 +1,28 @@
+const { getAllAccounts } = require("../../db/imap/imap");
+const { DEBUG } = require("../../utils/debug");
+const { ImapInstance } = require("./ImapInstance");
+
+class ImapSync {
+    constructor() {
+        this.instances = [];
+    }
+
+    init() {
+        getAllAccounts().then((accounts) => {
+            for (let i = 0; i < accounts.length; i++) {
+                accounts[i].password = accounts[i]?.password.toString().replace(/[\u{0080}-\u{FFFF}]/gu,"");
+                this.addInstance(accounts[i]);
+            }
+        }).catch((err) => {
+            DEBUG.log(err);
+        });
+    }
+
+    addInstance(config) {
+        this.instances.push(new ImapInstance(config));
+    }
+}
+
+module.exports = {
+    ImapSync
+}

back/mails/index.js

@@ -56,7 +56,6 @@ imap.once("ready", function () {
             msg.once("attributes", (attrs) => {
                 // todo find boxId
                 const boxId = 1;
-                console.log(attrs.envelope)
                 // mails.push(attrs);
                 // promises.push(saveMessage(attrs, boxId, imap));
             });

back/mails/storeMessage.js

@@ -38,8 +38,8 @@ function saveMessage(attrs, mailboxId, imap) {
                         });
 
                         stream.once("end", () => {
-                            // save raw data
-                            saveSource(messageId, buffer);
+                            // save raw data todo
+                            // saveSource(messageId, buffer);
 
                             // parse data
                             simpleParser(buffer, async (err, parsed) => {

back/routes/mail.js

@@ -6,56 +6,42 @@ const Ajv = require("ajv");
 const addFormats = require("ajv-formats");
 const ajv = new Ajv({ allErrors: true });
 addFormats(ajv);
-const schema_mailbox = require("../schemas/mailbox_schema.json");
-const { addMailbox } = require("../controllers/addMailbox.js");
-const { getMailboxes } = require("../db/api.js");
+const schema_account = require("../schemas/account_schema.json");
+const { addAccount } = require("../controllers/addAccount.js");
+const { getAccounts } = require("../db/api.js");
 const { rooms } = require("../controllers/rooms.js");
 const { messages } = require("../controllers/messages.js");
 
-const validate_mailbox = ajv.compile(schema_mailbox);
+const validate_account = ajv.compile(schema_account);
 
 /**
  * Return all mailboxes and folders for an user
  */
-router.get("/mailboxes", (req, res) => {
-    getMailboxes().then((data) => {
-        data[0].id = 1; // todo debug
-        res.status(statusCodes.OK).json(data)
+router.get("/accounts", (req, res) => {
+    getAccounts().then((data) => {
+        res.status(statusCodes.OK).json(data);
     });
 });
 
-/**
- * @param {number} mailboxId the id of the mailbox (account_id) from which to fetch the messages, 0 if from all
- * @param {number} offset the offset of the query
- * @param {number} limit the number of message to return
- * @param {string} token the token of the user
- * @return {object} a list of room and their preview (subject)
- */
 router.get("/:mailboxId/rooms", async (req, res) => {
-    const { mailboxId, offset, limit } = req.params;
-    // todo check token
-    // todo use offset
+    // todo use offset limit
     await rooms(req.params, res);
-
 });
 
 router.get("/:roomId/messages", async (req, res) => {
     const { roomId } = req.params;
-    console.log("called")
-    // todo check token
     await messages(req.params, res);
 });
 
 /**
  * Register a new mailbox inside the app
  */
-router.post("/mailbox", async (req, res) => {
-    console.log(req.body)
-    const valid = validate_mailbox(req.body);
+router.post("/account", async (req, res) => {
+    const valid = validate_account(req.body);
     if (!valid) {
-        res.status(statusCodes.NOT_ACCEPTABLE).send({ error: validate_mailbox.errors });
+        res.status(statusCodes.NOT_ACCEPTABLE).send({ error: validate_account.errors });
     } else {
-        await addMailbox(req.body, res);
+        await addAccount(req.body, res);
     }
 });
 

back/server.js

@@ -1,6 +1,8 @@
 const express = require("express");
 const cors = require("cors");
 const app = express();
+const { ImapSync } = require("./mails/imap/ImapSync");
+
 app.use(express.json());
 app.use(
     express.urlencoded({
@@ -11,4 +13,7 @@ app.use(cors());
 app.listen(process.env.PORT || 5500);
 
 const mailRouter = require("./routes/mail");
-app.use("/api/mail", mailRouter);
+app.use("/api/mail", mailRouter);
+
+const imapSync = new ImapSync();
+imapSync.init()

front/src/services/imapAPI.js

@@ -1,11 +1,11 @@
 import API from './API'
 
 export default {
-    registerMailbox(data) {
-        return API().post('/mail/mailbox', data);
+    registerAccount(data) {
+        return API().post('/mail/account', data);
     },
-    getMailboxes() {
-        return API().get('/mail/mailboxes');
+    getAccounts() {
+        return API().get('/mail/accounts');
     },
     getRooms(mailboxId) {
         return API().get(`/mail/${mailboxId}/rooms`);

front/src/store/store.js

@@ -25,6 +25,7 @@ const store = createStore({
             const mailbox = state.mailboxes.find((mailbox) => mailbox.id == payload);
             // todo fetched mailbox all
             if (mailbox?.fetched == false) {
+                console.log(payload)
                 API.getRooms(payload)
                     .then((res) => {
                         // todo add if not exist
@@ -91,7 +92,7 @@ const store = createStore({
     },
     actions: {
         fetchMailboxes: async (context) => {
-            API.getMailboxes()
+            API.getAccounts()
                 .then((res) => {
                     context.commit("addMailboxes", res.data);
                 })

front/src/views/modals/AddMailboxModal.vue

@@ -77,7 +77,7 @@ function addMailboxRequest() {
     tls: true
   };
 
-  API.registerMailbox(data).then((res) => {
+  API.registerAccount(data).then((res) => {
     console.log(res.status);
   }).catch((err) => {
     console.log(err.request.status)