diff --git a/back/app/saveMessage.js b/back/app/saveMessage.js new file mode 100644 index 0000000..a84f79e --- /dev/null +++ b/back/app/saveMessage.js @@ -0,0 +1,92 @@ +const { + createRoom, + registerMessageInRoom, + createThread, + registerMessageInThread, + isRoomGroup, + findSpacesFromMessage, + hasSameMembersAsParent, +} = require("../db/saveMessageApp"); + +const { findRoomByOwner, getAddresseId } = require("../db/mail"); + +/** + * take object address and join mailbox and host to return mailbox@host + */ +function createAddress(elt) { + return `${elt.mailbox}@${elt.host}`; +} + +async function registerMessageInApp(messageId, attrs) { + const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify + const envelope = attrs.envelope; + + await getAddresseId(createAddress(envelope.sender[0])).then(async (ownerId) => { + if (envelope.inReplyTo) { + await registerReplyMessage(envelope, messageId, isSeen); + } else { + await findRoomByOwner(ownerId).then(async (res) => { + if (res.length == 0) { + await createRoom(envelope.subject, ownerId).then(async (roomId) => { + await registerMessageInRoom(messageId, roomId, isSeen); + }); + } else { + await registerMessageInRoom(messageId, res[0].room_id, isSeen); + } + }); + } + }); +} + +async function registerReplyMessage(envelope, messageId, isSeen) { + const messageID = envelope.messageId; + await findSpacesFromMessage(messageId).then(async (spaces) => { + // todo sub thread will not be in index 0 so look in all indexes + if (spaces.length == 0) { + // no space, so is a transfer + // todo test if members of transferred message are included + } else if (spaces[0].thread_id) { + await registerMessageInThread(messageId, spaces[0].thread_id, isSeen); + // todo + // if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) { + // // register new message in thread + // // possibly convert to room only if parent is channel + // } else { + // // todo create sub thread + // } + } else if (spaces[0].room_id) { + // message in room and not thread + await isRoomGroup(spaces[0].room_id).then(async (isGroup) => { + if (isGroup) { + const hasSameMembers = await hasSameMembersAsParent(messageID, envelope.inReplyTo); + if (hasSameMembers) { + await registerMessageInRoom(messageId, spaces[0].room_id, isSeen); + } else { + // group and not the same member as the reply + // some recipient has been removed create a thread + const isDm = 0; // todo + await createThread(space[0].room_id, envelope.subject, isSeen, isDm).then(async (threadId) => { + await registerMessageInThread(messageId, threadId, isSeen); + }); + } + } else { + // reply from channel + // todo + // if (messageInRoom == 1) { // was new channel transform to group + // // register new message in group + // } else if (sender == owner) { // correction from the original sender + // // leave in the same channel + // } else { // user response to announcement + // // create new thread + // } + } + }); + } + }); + + // `SELECT app_room_messages.room, app_room_messages.thread FROM app_room_messages INNER JOIN messages WHERE messages.messageID = '${envelope.inReplyTo}' AND app_room_messages.message = messages.id`; +} + +module.exports = { + registerMessageInApp +}; \ No newline at end of file diff --git a/back/sql/bdd.js b/back/db/db.js similarity index 79% rename from back/sql/bdd.js rename to back/db/db.js index 98287da..9f179dd 100644 --- a/back/sql/bdd.js +++ b/back/db/db.js @@ -3,14 +3,14 @@ const MYSQL = require("./config.json").mysql; const DEBUG = require("../utils/debug.js").DEBUG; -const bdd = mysql.createConnection({ +const db = mysql.createConnection({ host: MYSQL.host, user: MYSQL.user, password: MYSQL.pwd, database: MYSQL.database, }); -bdd.connect(function (err) { +db.connect(function (err) { if (err) { DEBUG.log("Impossible de se connecter", err.code); } else { @@ -20,7 +20,7 @@ bdd.connect(function (err) { function execQueryAsync(query, values) { return new Promise((resolve, reject) => { - bdd.query(query, values, (err, results, fields) => { + db.query(query, values, (err, results, fields) => { if (err) { reject(err); } else { @@ -32,7 +32,7 @@ function execQueryAsync(query, values) { function execQueryAsyncWithId(query, values) { return new Promise((resolve, reject) => { - bdd.query(query, values, (err, results, fields) => { + db.query(query, values, (err, results, fields) => { if (err) { reject(err); } else { @@ -43,7 +43,7 @@ function execQueryAsyncWithId(query, values) { } function execQuery(query, values) { - bdd.query(query, values, (err, results, fields) => { + db.query(query, values, (err, results, fields) => { if (err) { DEBUG.log(err); throw (err); @@ -53,7 +53,7 @@ function execQuery(query, values) { } module.exports = { - bdd, // todo remove this + db, // todo remove this execQuery, execQueryAsync, execQueryAsyncWithId diff --git a/back/sql/mail.js b/back/db/mail.js similarity index 94% rename from back/sql/mail.js rename to back/db/mail.js index 20212f4..bba9547 100644 --- a/back/sql/mail.js +++ b/back/db/mail.js @@ -1,4 +1,4 @@ -const { bdd, execQueryAsync, execQueryAsyncWithId } = require("./bdd.js"); +const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js"); const DEBUG = require("../utils/debug").DEBUG; function isValidEmail(email) { diff --git a/back/sql/saveMessage.js b/back/db/saveMessage.js similarity index 50% rename from back/sql/saveMessage.js rename to back/db/saveMessage.js index 7854327..8d9c836 100644 --- a/back/sql/saveMessage.js +++ b/back/db/saveMessage.js @@ -1,25 +1,32 @@ -const { bdd, execQuery, execQueryAsync, execQueryAsyncWithId } = require("./bdd.js"); +const { db, execQuery, execQueryAsync, execQueryAsyncWithId } = require("./db.js"); const DEBUG = require("../utils/debug").DEBUG; async function registerMessage(timestamp, rfc822size, messageId) { - const query = `INSERT INTO message - (idate, messageID, rfc822size) VALUES (?, ?, ?) - ON DUPLICATE KEY UPDATE message_id = LAST_INSERT_ID(message_id)`; + const query = ` + INSERT INTO message + (idate, messageID, rfc822size) VALUES (?, ?, ?) + ON DUPLICATE KEY UPDATE message_id = LAST_INSERT_ID(message_id) + `; const values = [timestamp, messageId, rfc822size]; return await execQueryAsyncWithId(query, values); } function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, deleted) { - const query = `INSERT IGNORE INTO mailbox_message (mailbox_id, uid, message_id, modseq, seen, deleted) VALUES (?, ?, ?, ?, ?, ?)`; + const query = ` + INSERT IGNORE INTO mailbox_message + (mailbox_id, uid, message_id, modseq, seen, deleted) VALUES (?, ?, ?, ?, ?, ?) + `; const values = [mailboxId, uid, messageId, modseq, seen, deleted]; execQuery(query, values); } function registerBodypart(messageId, part, bodypartId, bytes, nbLines) { - const query = `INSERT IGNORE INTO part_number (message_id, part, bodypart_id, bytes, nb_lines) VALUES ('${messageId}', '${part}', '${bodypartId}', '${bytes}', '${nbLines}')`; - bdd.query(query, (err, results, fields) => { - if (err) DEBUG.log(err); - }); + const query = ` + INSERT IGNORE INTO part_number + (message_id, part, bodypart_id, bytes, nb_lines) VALUES (?, ?, ?, ?, ?) + `; + const values = [messageId, part, bodypartId, bytes, nbLines]; + execQuery(query, values); } async function saveBodypart(bytes, hash, text, data) { @@ -29,19 +36,30 @@ async function saveBodypart(bytes, hash, text, data) { } async function saveHeader_fields(messageId, fieldId, bodypartId, part, value) { - const query = `INSERT IGNORE INTO header_field (message_id, field_id, bodypart_id, part, value) VALUES (?, ?, ?, ?, ?)`; + const query = ` + INSERT IGNORE INTO header_field + (message_id, field_id, bodypart_id, part, value) VALUES (?, ?, ?, ?, ?) + `; const values = [messageId, fieldId, bodypartId, part, value]; return await execQueryAsync(query, values); } async function saveAddress_fields(messageId, fieldId, addressId, number) { - const query = `INSERT IGNORE INTO address_field (message_id , field_id, address_id, number) VALUES (?, ?, ?, ?)`; + const query = ` + INSERT IGNORE INTO address_field + (message_id , field_id, address_id, number) VALUES (?, ?, ?, ?) + `; const values = [messageId, fieldId, addressId, number]; - return execQueryAsync(query, values); + return await execQueryAsync(query, values); } function saveSource(messageId, content) { - // todo + const query = ` + INSERT INTO source (message_id, content) VALUES (?, ?) + ON DUPLICATE KEY UPDATE content = ? + `; + const values = [messageId, content, content]; + execQuery(query, values); } module.exports = { diff --git a/back/sql/saveMessageApp.js b/back/db/saveMessageApp.js similarity index 61% rename from back/sql/saveMessageApp.js rename to back/db/saveMessageApp.js index c0aadaa..1a6b38a 100644 --- a/back/sql/saveMessageApp.js +++ b/back/db/saveMessageApp.js @@ -1,4 +1,4 @@ -const { bdd, execQueryAsync, execQueryAsyncWithId } = require("./bdd.js"); +const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js"); const DEBUG = require("../utils/debug").DEBUG; async function createRoom(roomName, ownerId) { @@ -8,11 +8,10 @@ async function createRoom(roomName, ownerId) { // todo add members } -function registerMessageInRoom(messageId, roomId, isSeen) { - const query = `INSERT INTO app_space_message (message_id, room_id) VALUES ('${messageId}', '${roomId}')`; - bdd.query(query, (err, results, fields) => { - if (err) throw err; - }); +async function registerMessageInRoom(messageId, roomId, isSeen) { + const query = `INSERT INTO app_space_message (message_id, room_id) VALUES (?, ?)`; + const values = [messageId, roomId]; + await execQueryAsync(query, values); updateLastUpdateRoom(roomId); @@ -29,31 +28,19 @@ function incrementNotSeenRoom(roomId) { // todo } -function createThread(roomId, threadName, isDm) { - return new Promise((resolve, reject) => { - const query = `INSERT INTO app_thread - (room_id, thread_name, isDm) - VALUES ( - '${roomId}', - '${threadName}', - '${isDm}', - )`; - bdd.query(query, (err, results, fields) => { - if (err) reject(err); - resolve(results.insertId); - }); - }); - +async function createThread(roomId, threadName, isDm) { + const query = `INSERT INTO app_thread (room_id, thread_name, isDm) VALUES (?, ?, ?)`; + const values = [roomId, threadName, isDm]; + return await execQueryAsync(query, values); // todo add members } -function registerMessageInThread(messageId, threadId, isSeen) { +async function registerMessageInThread(messageId, threadId, isSeen) { // todo check if it is still a thread or should be a room const query = `INSERT IGNORE INTO app_space_message - (message_id, thread_id) VALUES ('${messageId}', '${threadId}')`; - bdd.query(query, (err, results, fields) => { - if (err) throw err; - }); + (message_id, thread_id) VALUES (?, ?)`; + const values = [messageId, threadId]; + await execQueryAsync(query, values); updateLastUpdateThread(threadId); if (!isSeen) { @@ -71,10 +58,10 @@ function incrementNotSeenThread(threadId) { // also increment parent room } -function isRoomGroup(roomId) { +async function isRoomGroup(roomId) { return new Promise((resolve, reject) => { const query = `SELECT isGroup FROM app_room WHERE room_id = '${roomId}'`; - bdd.query(query, (err, results, fields) => { + db.query(query, (err, results, fields) => { if (err) reject(err); resolve(results[0].isGroup); }); @@ -98,5 +85,5 @@ module.exports = { createThread, registerMessageInThread, isRoomGroup, - findSpacesFromMessage -}; \ No newline at end of file + findSpacesFromMessage, +}; diff --git a/back/sql/structure b/back/db/structure similarity index 100% rename from back/sql/structure rename to back/db/structure diff --git a/back/sql/structureV2.sql b/back/db/structureV2.sql similarity index 99% rename from back/sql/structureV2.sql rename to back/db/structureV2.sql index f095126..bc791f6 100644 --- a/back/sql/structureV2.sql +++ b/back/db/structureV2.sql @@ -145,6 +145,7 @@ CREATE TABLE app_space_message ( message_id INT NOT NULL, room_id INT, thread_id INT, + UNIQUE KEY (member_id, room_id, thread_id), FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE, FOREIGN KEY (room_id) REFERENCES app_room(room_id) ON DELETE SET NULL, FOREIGN KEY (thread_id) REFERENCES app_thread(thread_id) ON DELETE SET NULL diff --git a/back/imap/index.js b/back/imap/index.js index 96eab36..91def64 100644 --- a/back/imap/index.js +++ b/back/imap/index.js @@ -2,9 +2,11 @@ const Imap = require("imap"); const { simpleParser } = require("mailparser"); const inspect = require("util").inspect; const saveMessage = require("./storeMessage").saveMessage; +const registerMessageInApp = require("../app/saveMessage").registerMessageInApp; const imapConfig = require("./config.json").mail; const fs = require("fs"); +const { DEBUG } = require("../utils/debug"); const imap = new Imap({ user: imapConfig.user, password: imapConfig.password, @@ -20,57 +22,56 @@ let shouldReset = false; // let shouldReset = true; if (shouldReset) { - - - const { execQuery, execQueryAsync } = require("../sql/bdd"); - const query = - "SELECT table_name FROM INFORMATION_SCHEMA.tables WHERE table_schema = 'mail'"; + const { execQuery, execQueryAsync } = require("../db/db"); + const query = "SELECT table_name FROM INFORMATION_SCHEMA.tables WHERE table_schema = 'mail'"; execQueryAsync(query).then((results) => { execQuery("SET FOREIGN_KEY_CHECKS=0"); results.map((table) => { execQuery("DELETE FROM " + table.table_name); - }); }); return; } - imap.once("ready", function () { const readOnly = true; imap.openBox("INBOX", readOnly, (err, box) => { // console.log(box); // uidvalidty uidnext, messages total and new - imap.search(["ALL"], function (err, results) { - console.log(results[results.length - 1]); - }); + // imap.search(["ALL"], function (err, results) { + // console.log(results[results.length - 1]); + // }); - const f = imap.fetch(970, { + // const f = imap.fetch(970, { + // size: true, + // envelope: true, + // }); + const promises = []; + const mails = []; + var f = imap.seq.fetch('1:10', { size: true, - envelope: true, - }); - let messageIDs = []; - // var f = imap.seq.fetch('1:3', { - // bodies: 'HEADER.FIELDS (FROM TO SUBJECT DATE)', - // struct: true - // }); + envelope: true + }); f.on("message", function (msg, seqno) { msg.once("attributes", (attrs) => { // todo find boxId const boxId = 1; - messageIDs.push(attrs.envelope.messageId) - saveMessage(attrs, boxId, imap); + mails.push(attrs); + promises.push(saveMessage(attrs, boxId, imap)); }); }); f.once("error", function (err) { - console.log("Fetch error: " + err); + DEBUG.log("Fetch error: " + err); }); - f.once("end", function () { - console.log(messageIDs) - isDone = true; - console.log("Done fetching all messages!"); + f.once("end", async function () { + Promise.all(promises).then(async (res) => { + DEBUG.log("Done fetching all messages!"); + for (let i = 0; i < mails.length; i++) { + await registerMessageInApp(res[i], mails[i]); + } + }); }); - // imap.end() + // imap.end() }); }); diff --git a/back/imap/storeMessage.js b/back/imap/storeMessage.js index e7e615f..7de687f 100644 --- a/back/imap/storeMessage.js +++ b/back/imap/storeMessage.js @@ -1,4 +1,4 @@ -const { getAddresseId } = require("../sql/mail"); +const { getAddresseId } = require("../db/mail"); const { DEBUG } = require("../utils/debug"); const { simpleParser } = require("mailparser"); const moment = require("moment"); @@ -11,95 +11,84 @@ const { registerBodypart, saveBodypart, saveSource, -} = require("../sql/saveMessage"); +} = require("../db/saveMessage"); -const { - createRoom, - registerMessageInRoom, - createThread, - registerMessageInThread, - isRoomGroup, - findSpacesFromMessage, - hasSameMembersAsParent, -} = require("../sql/saveMessageApp"); -const { getFieldId, findRoomByOwner } = require("../sql/mail"); +const { getFieldId } = require("../db/mail"); function saveMessage(attrs, mailboxId, imap) { const envelope = attrs.envelope; - const ts = moment(new Date(envelope.date).getTime()).format( - "YYYY-MM-DD HH:mm:ss" - ); + const ts = moment(new Date(envelope.date).getTime()).format("YYYY-MM-DD HH:mm:ss"); const rfc822size = attrs.size; const messageID = envelope.messageId; - registerMessage(ts, rfc822size, messageID) - .then((messageId) => { - const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify - const deleted = attrs.flags.includes("Deleted") ? 1 : 0; // todo verify + return new Promise((resolve, reject) => { + registerMessage(ts, rfc822size, messageID) + .then((messageId) => { + const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify + const deleted = attrs.flags.includes("Deleted") ? 1 : 0; // todo verify - registerMailbox_message( - mailboxId, - attrs.uid, - messageId, - attrs.modseq, - isSeen, - deleted - ); - const f = imap.fetch(attrs.uid, { bodies: "" }); - let buffer = ""; + registerMailbox_message(mailboxId, attrs.uid, messageId, attrs.modseq, isSeen, deleted); + const f = imap.fetch(attrs.uid, { bodies: "" }); + let buffer = ""; - f.on("message", function (msg, seqno) { - msg.on("body", function (stream, info) { - stream.on("data", function (chunk) { - buffer += chunk.toString("utf8"); - }); + f.on("message", function (msg, seqno) { + msg.on("body", function (stream, info) { + stream.on("data", function (chunk) { + buffer += chunk.toString("utf8"); + }); - stream.once("end", () => { - // save raw data - saveSource(messageId, buffer); + stream.once("end", () => { + // save raw data + saveSource(messageId, buffer); - // parse data - simpleParser(buffer, async (err, parsed) => { - saveFromParsedData(parsed, messageId); + // parse data + simpleParser(buffer, async (err, parsed) => { + saveFromParsedData(parsed, messageId) + .then(() => { + resolve(messageId); + }) + .catch((err) => { + reject(err); + }); + }); }); }); }); + f.once("error", function (err) { + console.log("Fetch error: " + err); + }); + f.once("end", function () { + DEBUG.log("Done fetching data of "+messageID); + }); + }) + .catch((err) => { + DEBUG.log("Unable to register message: " + err); + reject(err); }); - f.once("error", function (err) { - console.log("Fetch error: " + err); - }); - f.once("end", function () { - console.log("Done fetching all messages!"); - }); - }) - .catch((err) => { - DEBUG.log("Unable to register message: " + err); - }); + }); } -function saveFromParsedData(parsed, messageId) { - fs.writeFileSync("./test.txt", JSON.stringify(parsed)); +async function saveFromParsedData(parsed, messageId) { + const promises = []; Object.keys(parsed).forEach((key) => { if (["from", "to", "cc", "bcc", "reply-to"].includes(key)) { - // save address field - getFieldId(key).then((fieldId) => { - parsed[key].value.forEach((addr, nb) => { - getAddresseId(addr.address, addr.name).then((addressId) => { - saveAddress_fields(messageId, fieldId, addressId, nb); + promises.push( + // save address field + getFieldId(key).then((fieldId) => { + parsed[key].value.forEach((addr, nb) => { + getAddresseId(addr.address, addr.name).then(async (addressId) => { + await saveAddress_fields(messageId, fieldId, addressId, nb); + }); }); - }); - }); + }), + ); } else if (["subject", "inReplyTo"].includes(key)) { // todo : "references" - getFieldId(key).then((fieldId) => { - saveHeader_fields( - messageId, - fieldId, - undefined, - undefined, - parsed[key] - ); - }); + promises.push( + getFieldId(key).then(async (fieldId) => { + await saveHeader_fields(messageId, fieldId, undefined, undefined, parsed[key]); + }), + ); } else if (["html", "text", "textAsHtml"].includes(key)) { const hash = "0"; const size = "0"; @@ -116,9 +105,7 @@ function saveFromParsedData(parsed, messageId) { // }); } else if (key == "attachments") { // todo - } else if ( - ["date", "messageId", "headers", "headerLines"].includes(key) - ) { + } else if (["date", "messageId", "headers", "headerLines"].includes(key)) { // messageId and date are already saved // other field are not improted and can be retrieved in source return; @@ -127,104 +114,10 @@ function saveFromParsedData(parsed, messageId) { return; } }); + return Promise.all(promises); // todo when transfered } -function haveSameReceivers() { - // take cc and to -} - -/** - * take object address and join mailbox and host to return mailbox@host - */ -function createAddress(elt) { - return `${elt.mailbox}@${elt.host}`; -} - -function registerMessageInApp(envelope, messageId, isSeen) { - getAddresseId(createAddress(envelope.sender[0])).then((ownerId) => { - if (envelope.inReplyTo) { - registerReplyMessage(envelope, messageId, isSeen); - } else { - findRoomByOwner(ownerId).then((res) => { - if (res.length == 0) { - createRoom(envelope.subject, ownerId).then((roomId) => { - registerMessageInRoom(messageId, roomId, isSeen); - }); - } else { - registerMessageInRoom(messageId, res[0].room_id, isSeen); - } - }); - } - }); -} - -function registerReplyMessage(envelope, messageId, isSeen) { - const messageID = envelope.messageId; - findSpacesFromMessage(messageId).then((spaces) => { - // todo sub thread will not be in index 0 so look in all indexes - if (spaces.length == 0) { - // no space, so is a transfer - // todo test if members of transferred message are included - } else if (spaces[0].thread_id) { - registerMessageInThread(messageId, spaces[0].thread_id, isSeen); - // todo - // if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) { - // // register new message in thread - // // possibly convert to room only if parent is channel - // } else { - // // todo create sub thread - // } - } else if (spaces[0].room_id) { - // message in room and not thread - isRoomGroup(spaces[0].room_id).then((isGroup) => { - if (isGroup) { - if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) { - registerMessageInRoom( - messageId, - spaces[0].room_id, - isSeen - ); - } else { - // group and not the same member as the reply - // some recipient has been removed create a thread - const isDm = 0; // todo - createThread( - space[0].room_id, - envelope.subject, - isSeen, - isDm - ).then((threadId) => { - registerMessageInThread( - messageId, - threadId, - isSeen - ); - }); - } - } else { - // reply from channel - // todo - // if (messageInRoom == 1) { // was new channel transform to group - // // register new message in group - // } else if (sender == owner) { // correction from the original sender - // // leave in the same channel - // } else { // user response to announcement - // // create new thread - // } - } - }); - } - }); - - // `SELECT app_room_messages.room, app_room_messages.thread FROM app_room_messages INNER JOIN messages WHERE messages.messageID = '${envelope.inReplyTo}' AND app_room_messages.message = messages.id`; -} - -function keyNormalizer(key) { - // todo - return key; -} - module.exports = { saveMessage, }; diff --git a/back/utils/debug.js b/back/utils/debug.js index e7e5599..ff14596 100644 --- a/back/utils/debug.js +++ b/back/utils/debug.js @@ -1,9 +1,9 @@ -const DEBUG = (function() { - const timestamp = function(){}; - timestamp.toString = () => "[DEBUG " + new Date().toLocaleDateString() + "]"; - return { log: console.log.bind(console, "%s", timestamp) }; +const DEBUG = (function () { + const timestamp = function () {}; + timestamp.toString = () => "[" + new Date().toLocaleString() + "]"; + return { log: console.log.bind(console, "%s", timestamp) }; })(); module.exports = { - DEBUG -} \ No newline at end of file + DEBUG, +};