From 62dd43c3d5d41057c8e1af4eafc1e76154689ee1 Mon Sep 17 00:00:00 2001 From: grimhilt Date: Sun, 26 Mar 2023 14:20:16 +0200 Subject: [PATCH] link imap sync to server and show email on front --- .../{addMailbox.js => addAccount.js} | 10 ++- back/db/api.js | 24 +++--- back/db/imap/imap.js | 49 +++++++++++++ back/db/mail.js | 11 +-- back/db/saveMessage.js | 2 +- back/db/structureV2.sql | 11 +-- back/mails/imap/Box.js | 59 +++++++++++++++ back/mails/imap/ImapInstance.js | 73 +++++++++++++++++++ back/mails/imap/ImapSync.js | 28 +++++++ back/mails/index.js | 1 - back/mails/storeMessage.js | 4 +- back/routes/mail.js | 38 +++------- ...ailbox_schema.json => account_schema.json} | 0 back/server.js | 7 +- front/src/services/imapAPI.js | 8 +- front/src/store/store.js | 3 +- front/src/views/modals/AddMailboxModal.vue | 2 +- 17 files changed, 266 insertions(+), 64 deletions(-) rename back/controllers/{addMailbox.js => addAccount.js} (70%) create mode 100644 back/db/imap/imap.js create mode 100644 back/mails/imap/Box.js create mode 100644 back/mails/imap/ImapInstance.js create mode 100644 back/mails/imap/ImapSync.js rename back/schemas/{mailbox_schema.json => account_schema.json} (100%) diff --git a/back/controllers/addMailbox.js b/back/controllers/addAccount.js similarity index 70% rename from back/controllers/addMailbox.js rename to back/controllers/addAccount.js index fd901b9..ab3d90f 100644 --- a/back/controllers/addMailbox.js +++ b/back/controllers/addAccount.js @@ -1,11 +1,11 @@ const statusCode = require("../utils/statusCodes").statusCodes; -const { registerMailbox } = require("../db/api"); +const { registerAccount } = require("../db/api"); const { getAddresseId } = require("../db/mail"); -async function addMailbox(body, res) { +async function addAccount(body, res) { const { email, pwd, xoauth, xoauth2, host, port, tls } = body; getAddresseId(email).then((addressId) => { - registerMailbox(addressId, pwd, xoauth, xoauth2, host, port, tls) + registerAccount(addressId, pwd, xoauth, xoauth2, host, port, tls) .then((mailboxId) => { res.status(statusCode.OK).json({ id: mailboxId }); }) @@ -16,5 +16,7 @@ async function addMailbox(body, res) { } module.exports = { - addMailbox, + addAccount, }; + +// todo change mailbox to account diff --git a/back/db/api.js b/back/db/api.js index 8d49f5f..13b1db5 100644 --- a/back/db/api.js +++ b/back/db/api.js @@ -2,7 +2,7 @@ const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js"); const { queryCcId, queryToId, queryFromId } = require("./utils/addressQueries.js"); const DEBUG = require("../utils/debug").DEBUG; -async function registerMailbox(userId, pwd, xoauth, xoauth2, host, port, tls) { +async function registerAccount(userId, pwd, xoauth, xoauth2, host, port, tls) { const query = ` INSERT INTO app_account (user_id, account_pwd, xoauth, xoauth2, host, port, tls) VALUES (?, ?, ?, ?, ?, ?, ?) @@ -11,13 +11,18 @@ async function registerMailbox(userId, pwd, xoauth, xoauth2, host, port, tls) { return await execQueryAsyncWithId(query, values); } -async function getMailboxes() { +async function getAccounts() { + // todo mailbox or account id ? const query = ` SELECT - app_account.account_id AS id, + mailbox.mailbox_id AS id, address.email - FROM app_account INNER JOIN address - WHERE address.address_id = app_account.user_id + FROM app_account + INNER JOIN address + INNER JOIN mailbox + WHERE + address.address_id = app_account.user_id AND + mailbox.account_id = app_account.account_id `; const values = []; return await execQueryAsync(query, values); @@ -38,12 +43,11 @@ async function getRooms(mailboxId) { INNER JOIN address WHERE message.message_id = app_room.message_id AND - mailbox_message.mailbox_id = 1 AND + mailbox_message.mailbox_id = ? AND mailbox_message.message_id = message.message_id AND address.address_id = app_room.owner_id `; - // todo mailboxId - const values = []; + const values = [mailboxId]; return await execQueryAsync(query, values); } @@ -92,8 +96,8 @@ async function getMessages(roomId) { } module.exports = { - registerMailbox, - getMailboxes, + registerAccount, + getAccounts, getRooms, getMessages }; diff --git a/back/db/imap/imap.js b/back/db/imap/imap.js new file mode 100644 index 0000000..1371af1 --- /dev/null +++ b/back/db/imap/imap.js @@ -0,0 +1,49 @@ +const { execQueryAsyncWithId, execQueryAsync, execQuery } = require("../db"); + +async function getAllAccounts() { + const query = ` + SELECT + app_account.account_id AS id, + address.email AS user, + app_account.account_pwd AS password, + app_account.host AS host, + app_account.port AS port, + app_account.tls AS tls + FROM app_account INNER JOIN address + WHERE address.address_id = app_account.user_id + `; + const values = []; + return await execQueryAsync(query, values); +} + +async function getAllMailboxes(accountId) { + const query = 'SELECT * FROM mailbox WHERE mailbox.account_id = ?'; + const values = [accountId]; + return await execQueryAsync(query, values) +} + +async function registerMailbox(accountId, mailboxName) { + const query = `INSERT INTO mailbox (account_id, mailbox_name) VALUES (?, ?)`; + const values = [accountId, mailboxName]; + return await execQueryAsyncWithId(query, values); +} + +async function getMailbox(mailboxId) { + const query = `SELECT * FROM mailbox WHERE mailbox_id = ?`; + const values = [mailboxId]; + return await execQueryAsync(query, values); +} + +function updateMailbox(mailboxId, uidnext) { + const query = `UPDATE mailbox SET uidnext = ? WHERE mailbox_id = ?`; + const values = [uidnext, mailboxId]; + execQuery(query, values); +} + +module.exports = { + getAllAccounts, + getAllMailboxes, + registerMailbox, + getMailbox, + updateMailbox +} \ No newline at end of file diff --git a/back/db/mail.js b/back/db/mail.js index bba9547..f7434b4 100644 --- a/back/db/mail.js +++ b/back/db/mail.js @@ -1,19 +1,14 @@ -const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js"); +const { execQueryAsync, execQueryAsyncWithId } = require("./db.js"); const DEBUG = require("../utils/debug").DEBUG; -function isValidEmail(email) { - // todo - return true; -} - async function getAddresseId(email, name) { const localpart = email.split("@")[0]; const domain = email.split("@")[1]; const query = `INSERT INTO address (address_name, localpart, domain, email) VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE email = ?, address_id = LAST_INSERT_ID(address_id)`; - const values = [name, localpart, domain, email, email]; + ON DUPLICATE KEY UPDATE address_name = ?, address_id = LAST_INSERT_ID(address_id)`; + const values = [name, localpart, domain, email, name]; return await execQueryAsyncWithId(query, values); } diff --git a/back/db/saveMessage.js b/back/db/saveMessage.js index 8d36646..d493dd0 100644 --- a/back/db/saveMessage.js +++ b/back/db/saveMessage.js @@ -17,7 +17,6 @@ function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, delete (mailbox_id, uid, message_id, modseq, seen, deleted) VALUES (?, ?, ?, ?, ?, ?) `; const values = [mailboxId, uid, messageId, modseq, seen, deleted]; - console.log(values) execQuery(query, values); } @@ -55,6 +54,7 @@ async function saveAddress_fields(messageId, fieldId, addressId, number) { } function saveSource(messageId, content) { + content = Buffer.from(content); const query = ` INSERT INTO source (message_id, content) VALUES (?, ?) ON DUPLICATE KEY UPDATE content = ? diff --git a/back/db/structureV2.sql b/back/db/structureV2.sql index 31f1735..91333c1 100644 --- a/back/db/structureV2.sql +++ b/back/db/structureV2.sql @@ -67,7 +67,7 @@ CREATE TABLE bodypart ( bodypart_id INT AUTO_INCREMENT, bytes INT NOT NULL, hash TEXT NOT NULL, - text TEXT, + text TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci, data BINARY, PRIMARY KEY (bodypart_id) ); @@ -75,10 +75,10 @@ CREATE TABLE bodypart ( -- 7 CREATE TABLE source ( message_id INT NOT NULL, - content TEXT NOT NULL, + content BLOB NOT NULL, PRIMARY KEY (message_id), FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE -); +) -- 8 CREATE TABLE field_name ( @@ -94,8 +94,9 @@ CREATE TABLE header_field ( field_id INT NOT NULL, bodypart_id INT, part VARCHAR(128), - value TEXT, - UNIQUE KEY (message_id, field_id, bodypart_id),-- todo multiple raws + value TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci, + UNIQUE KEY (message_id, field_id, bodypart_id), + UNIQUE KEY (message_id, field_id, part), FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE, FOREIGN KEY (field_id) REFERENCES field_name(field_id) ON DELETE CASCADE, FOREIGN KEY (bodypart_id) REFERENCES bodypart(bodypart_id) diff --git a/back/mails/imap/Box.js b/back/mails/imap/Box.js new file mode 100644 index 0000000..9329148 --- /dev/null +++ b/back/mails/imap/Box.js @@ -0,0 +1,59 @@ +const { getMailbox, updateMailbox } = require("../../db/imap/imap"); +const { DEBUG } = require("../../utils/debug"); +const { registerMessageInApp } = require("../saveMessage"); +const { saveMessage } = require("../storeMessage"); + +class Box { + constructor(_imap, boxId, _boxName) { + this.imap = _imap; + this.boxName = _boxName; + this.id = boxId; + this.box; + this.init(); + } + + async init() { + this.box = (await getMailbox(this.id))[0]; + + const readOnly = true; + this.imap.openBox(this.boxName, readOnly, (err, box) => { + if (err) DEBUG.log(err); + this.sync(this.box.uidnext, box.uidnext); + }); + } + + sync(savedUid, currentUid) { + const promises = []; + const mails = []; + console.log(savedUid, currentUid); + const f = this.imap.seq.fetch(`${savedUid}:${currentUid}`, { + size: true, + envelope: true, + }); + + f.on("message", (msg, seqno) => { + msg.once("attributes", (attrs) => { + mails.push(attrs); + promises.push(saveMessage(attrs, this.id, this.imap)); + }); + }); + + f.once("error", (err) => { + DEBUG.log("Fetch error: " + err); + }); + + f.once("end", async () => { + await Promise.all(promises).then(async (res) => { + for (let i = 0; i < mails.length; i++) { + console.log(i, mails[i].uid) + await registerMessageInApp(res[i], mails[i]); + } + }); + updateMailbox(this.id, currentUid); + }); + } +} + +module.exports = { + Box, +}; diff --git a/back/mails/imap/ImapInstance.js b/back/mails/imap/ImapInstance.js new file mode 100644 index 0000000..e6526fb --- /dev/null +++ b/back/mails/imap/ImapInstance.js @@ -0,0 +1,73 @@ +const Imap = require("imap"); +const { getAllMailboxes, registerMailbox } = require("../../db/imap/imap"); +const { DEBUG } = require("../../utils/debug"); +const { Box } = require("./Box"); + +class ImapInstance { + constructor(account) { + this.imap = new Imap({ + user: account.user, + password: account.password, + tlsOptions: { servername: account.host }, + host: account.host, + port: account.port, + tls: account.tls, + }); + + this.account = account; + this.boxes = []; + + /** + * IMAP + */ + this.imap.once("ready", () => { + DEBUG.log("imap connected") + this.imapReady(); + }); + + this.imap.once("error", function (err) { + DEBUG.log(err); + }); + + this.imap.once("end", function () { + DEBUG.log("Connection ended"); + }); + + this.imap.connect(); + } + + imapReady() { + getAllMailboxes(this.account.id).then((mailboxes) => { + if (mailboxes.length > 0) { + this.boxes.push(new Box(this.imap, mailboxes[0].mailbox_id, mailboxes[0].mailbox_name)); + } else { + this.imap.getBoxes('', (err, boxes) => { + if (err) DEBUG.log(err); + const allBoxName = this.getAllBox(boxes); + registerMailbox(this.account.id, allBoxName).then((mailboxId) => { + this.boxes.push(new Box(this.imap, mailboxId, allBoxName)); + }); + }); + } + }); + } + + getAllBox(boxes) { + // ideally we should get the all box to get all messages + let allBox; + Object.keys(boxes).forEach(key => { + if (key === 'INBOX') return; + allBox = key; + Object.keys(boxes[key].children).forEach((childBoxes) => { + if (boxes[key].children[childBoxes].attribs.includes('\\All')) { + allBox += '/' + childBoxes; + } + }); + }); + return allBox; + } +} + +module.exports = { + ImapInstance +} \ No newline at end of file diff --git a/back/mails/imap/ImapSync.js b/back/mails/imap/ImapSync.js new file mode 100644 index 0000000..71cbc7e --- /dev/null +++ b/back/mails/imap/ImapSync.js @@ -0,0 +1,28 @@ +const { getAllAccounts } = require("../../db/imap/imap"); +const { DEBUG } = require("../../utils/debug"); +const { ImapInstance } = require("./ImapInstance"); + +class ImapSync { + constructor() { + this.instances = []; + } + + init() { + getAllAccounts().then((accounts) => { + for (let i = 0; i < accounts.length; i++) { + accounts[i].password = accounts[i]?.password.toString().replace(/[\u{0080}-\u{FFFF}]/gu,""); + this.addInstance(accounts[i]); + } + }).catch((err) => { + DEBUG.log(err); + }); + } + + addInstance(config) { + this.instances.push(new ImapInstance(config)); + } +} + +module.exports = { + ImapSync +} \ No newline at end of file diff --git a/back/mails/index.js b/back/mails/index.js index 89e6f82..f0ba06f 100644 --- a/back/mails/index.js +++ b/back/mails/index.js @@ -56,7 +56,6 @@ imap.once("ready", function () { msg.once("attributes", (attrs) => { // todo find boxId const boxId = 1; - console.log(attrs.envelope) // mails.push(attrs); // promises.push(saveMessage(attrs, boxId, imap)); }); diff --git a/back/mails/storeMessage.js b/back/mails/storeMessage.js index 7a1ed18..f5d34b0 100644 --- a/back/mails/storeMessage.js +++ b/back/mails/storeMessage.js @@ -38,8 +38,8 @@ function saveMessage(attrs, mailboxId, imap) { }); stream.once("end", () => { - // save raw data - saveSource(messageId, buffer); + // save raw data todo + // saveSource(messageId, buffer); // parse data simpleParser(buffer, async (err, parsed) => { diff --git a/back/routes/mail.js b/back/routes/mail.js index a3130d3..67bceb4 100644 --- a/back/routes/mail.js +++ b/back/routes/mail.js @@ -6,56 +6,42 @@ const Ajv = require("ajv"); const addFormats = require("ajv-formats"); const ajv = new Ajv({ allErrors: true }); addFormats(ajv); -const schema_mailbox = require("../schemas/mailbox_schema.json"); -const { addMailbox } = require("../controllers/addMailbox.js"); -const { getMailboxes } = require("../db/api.js"); +const schema_account = require("../schemas/account_schema.json"); +const { addAccount } = require("../controllers/addAccount.js"); +const { getAccounts } = require("../db/api.js"); const { rooms } = require("../controllers/rooms.js"); const { messages } = require("../controllers/messages.js"); -const validate_mailbox = ajv.compile(schema_mailbox); +const validate_account = ajv.compile(schema_account); /** * Return all mailboxes and folders for an user */ -router.get("/mailboxes", (req, res) => { - getMailboxes().then((data) => { - data[0].id = 1; // todo debug - res.status(statusCodes.OK).json(data) +router.get("/accounts", (req, res) => { + getAccounts().then((data) => { + res.status(statusCodes.OK).json(data); }); }); -/** - * @param {number} mailboxId the id of the mailbox (account_id) from which to fetch the messages, 0 if from all - * @param {number} offset the offset of the query - * @param {number} limit the number of message to return - * @param {string} token the token of the user - * @return {object} a list of room and their preview (subject) - */ router.get("/:mailboxId/rooms", async (req, res) => { - const { mailboxId, offset, limit } = req.params; - // todo check token - // todo use offset + // todo use offset limit await rooms(req.params, res); - }); router.get("/:roomId/messages", async (req, res) => { const { roomId } = req.params; - console.log("called") - // todo check token await messages(req.params, res); }); /** * Register a new mailbox inside the app */ -router.post("/mailbox", async (req, res) => { - console.log(req.body) - const valid = validate_mailbox(req.body); +router.post("/account", async (req, res) => { + const valid = validate_account(req.body); if (!valid) { - res.status(statusCodes.NOT_ACCEPTABLE).send({ error: validate_mailbox.errors }); + res.status(statusCodes.NOT_ACCEPTABLE).send({ error: validate_account.errors }); } else { - await addMailbox(req.body, res); + await addAccount(req.body, res); } }); diff --git a/back/schemas/mailbox_schema.json b/back/schemas/account_schema.json similarity index 100% rename from back/schemas/mailbox_schema.json rename to back/schemas/account_schema.json diff --git a/back/server.js b/back/server.js index 2903944..d4c2139 100644 --- a/back/server.js +++ b/back/server.js @@ -1,6 +1,8 @@ const express = require("express"); const cors = require("cors"); const app = express(); +const { ImapSync } = require("./mails/imap/ImapSync"); + app.use(express.json()); app.use( express.urlencoded({ @@ -11,4 +13,7 @@ app.use(cors()); app.listen(process.env.PORT || 5500); const mailRouter = require("./routes/mail"); -app.use("/api/mail", mailRouter); \ No newline at end of file +app.use("/api/mail", mailRouter); + +const imapSync = new ImapSync(); +imapSync.init() \ No newline at end of file diff --git a/front/src/services/imapAPI.js b/front/src/services/imapAPI.js index 587f1a7..476faec 100644 --- a/front/src/services/imapAPI.js +++ b/front/src/services/imapAPI.js @@ -1,11 +1,11 @@ import API from './API' export default { - registerMailbox(data) { - return API().post('/mail/mailbox', data); + registerAccount(data) { + return API().post('/mail/account', data); }, - getMailboxes() { - return API().get('/mail/mailboxes'); + getAccounts() { + return API().get('/mail/accounts'); }, getRooms(mailboxId) { return API().get(`/mail/${mailboxId}/rooms`); diff --git a/front/src/store/store.js b/front/src/store/store.js index 5e37586..ee15c2c 100644 --- a/front/src/store/store.js +++ b/front/src/store/store.js @@ -25,6 +25,7 @@ const store = createStore({ const mailbox = state.mailboxes.find((mailbox) => mailbox.id == payload); // todo fetched mailbox all if (mailbox?.fetched == false) { + console.log(payload) API.getRooms(payload) .then((res) => { // todo add if not exist @@ -91,7 +92,7 @@ const store = createStore({ }, actions: { fetchMailboxes: async (context) => { - API.getMailboxes() + API.getAccounts() .then((res) => { context.commit("addMailboxes", res.data); }) diff --git a/front/src/views/modals/AddMailboxModal.vue b/front/src/views/modals/AddMailboxModal.vue index 29c583b..da4af93 100644 --- a/front/src/views/modals/AddMailboxModal.vue +++ b/front/src/views/modals/AddMailboxModal.vue @@ -77,7 +77,7 @@ function addMailboxRequest() { tls: true }; - API.registerMailbox(data).then((res) => { + API.registerAccount(data).then((res) => { console.log(res.status); }).catch((err) => { console.log(err.request.status)