From 0ea7f5865bd3595099b3fd4293f0cebce54212aa Mon Sep 17 00:00:00 2001 From: grimhilt Date: Sat, 25 Mar 2023 16:47:23 +0100 Subject: [PATCH] advancements in tests and storing messages --- .gitignore | 2 +- back/db/api.js | 30 ++----------- back/db/saveMessageApp.js | 62 ++++++++++++++++++++++---- back/db/structureV2.sql | 6 +-- back/db/utils/addressQueries.js | 20 +++++++++ back/{imap => mails}/index.js | 16 ++----- back/{app => mails}/saveMessage.js | 65 +++++++++++++++------------- back/{imap => mails}/storeMessage.js | 6 +++ back/mails/utils/statusUtils.js | 13 ++++++ back/server.js | 2 +- back/test/mail/saveMessage-test.js | 52 ++++++++++++++++++++++ back/test/sql/saveMessageApp-test.js | 19 ++++++++ back/test/sql/test-utilsDb.js | 62 ++++++++++++++++++++++++++ back/utils/array.js | 13 ++++++ front/src/views/room/RoomView.vue | 1 - 15 files changed, 285 insertions(+), 84 deletions(-) create mode 100644 back/db/utils/addressQueries.js rename back/{imap => mails}/index.js (89%) rename back/{app => mails}/saveMessage.js (50%) rename back/{imap => mails}/storeMessage.js (97%) create mode 100644 back/mails/utils/statusUtils.js create mode 100644 back/test/mail/saveMessage-test.js create mode 100644 back/test/sql/saveMessageApp-test.js create mode 100644 back/test/sql/test-utilsDb.js create mode 100644 back/utils/array.js diff --git a/.gitignore b/.gitignore index d9feed9..bfd38a6 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,6 @@ config.json *.txt *.json tmp -*test* +test.* *.png !*/schemas/* \ No newline at end of file diff --git a/back/db/api.js b/back/db/api.js index 46f1ad2..8d49f5f 100644 --- a/back/db/api.js +++ b/back/db/api.js @@ -1,4 +1,5 @@ const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js"); +const { queryCcId, queryToId, queryFromId } = require("./utils/addressQueries.js"); const DEBUG = require("../utils/debug").DEBUG; async function registerMailbox(userId, pwd, xoauth, xoauth2, host, port, tls) { @@ -48,8 +49,6 @@ async function getRooms(mailboxId) { async function getMessages(roomId) { // todo attachements name - // todo html, text, textAsHtml - // todo datetime const query = ` SELECT msg.message_id AS id, @@ -61,30 +60,9 @@ async function getMessages(roomId) { message.idate AS date FROM app_room_message msg - LEFT JOIN ( - SELECT address_field.address_id, address_field.message_id - FROM address_field - INNER JOIN field_name - WHERE - field_name.field_id = address_field.field_id AND - field_name.field_name = 'from' - ) fromT ON msg.message_id = fromT.message_id - LEFT JOIN ( - SELECT address_field.address_id, address_field.message_id - FROM address_field - INNER JOIN field_name - WHERE - field_name.field_id = address_field.field_id AND - field_name.field_name = 'to' - ) toT ON msg.message_id = toT.message_id - LEFT JOIN ( - SELECT address_field.address_id, address_field.message_id - FROM address_field - INNER JOIN field_name - WHERE - field_name.field_id = address_field.field_id AND - field_name.field_name = 'cc' - ) ccT ON msg.message_id = ccT.message_id + ${queryFromId} fromT ON msg.message_id = fromT.message_id + ${queryToId} toT ON msg.message_id = toT.message_id + ${queryCcId} ccT ON msg.message_id = ccT.message_id LEFT JOIN ( SELECT header_field.message_id, header_field.value diff --git a/back/db/saveMessageApp.js b/back/db/saveMessageApp.js index b73bacb..ece6ab4 100644 --- a/back/db/saveMessageApp.js +++ b/back/db/saveMessageApp.js @@ -1,4 +1,5 @@ const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js"); +const { queryFromId, queryToId, queryCcId } = require("./utils/addressQueries.js"); const DEBUG = require("../utils/debug").DEBUG; async function createRoom(roomName, ownerId, messageId) { @@ -28,15 +29,18 @@ function incrementNotSeenRoom(roomId) { // todo } -async function createThread(roomId, threadName, isDm) { - const query = `INSERT INTO app_thread (room_id, thread_name, isDm) VALUES (?, ?, ?)`; - const values = [roomId, threadName, isDm]; +async function createThread(threadName, ownerId, messageId, parentRoomId, isDm) { + const rootRoomId = -1; // todo + const threadId = await createRoom(threadName, ownerId, messageId); + const query = `INSERT INTO app_thread (room_id, parent_room_id, root_room_id, isDm) VALUES (?, ?, ?, ?)`; + const values = [threadId, parentRoomId, rootRoomId, isDm]; return await execQueryAsync(query, values); // todo add members } async function registerMessageInThread(messageId, threadId, isSeen) { // todo check if it is still a thread or should be a room + // todo isdm const query = `INSERT IGNORE INTO app_space_message (message_id, thread_id) VALUES (?, ?)`; const values = [messageId, threadId]; @@ -69,14 +73,53 @@ async function isRoomGroup(roomId) { } async function findRoomsFromMessage(messageId) { - const query = `SELECT room_id FROM app_room_message WHERE message_id = '${messageId}'`; - return await execQueryAsync(query); + const query = `SELECT room_id FROM app_room_message WHERE message_id = ? ORDER BY room_id`; + const values = [messageId]; + return await execQueryAsync(query, values); } -async function hasSameMembersAsParent(messageId, parentID) { - // const query = `SELECT `; - // return await execQueryAsync(query) - // todo +async function hasSameMembersAsParent(messageId, messageID) { + const query1 = ` + SELECT + GROUP_CONCAT(fromT.address_id) AS fromA, + GROUP_CONCAT(toT.address_id) AS toA, + GROUP_CONCAT(ccT.address_id) AS ccA + FROM message msg + ${queryFromId} fromT ON msg.message_id = fromT.message_id + ${queryToId} toT ON msg.message_id = toT.message_id + ${queryCcId} ccT ON msg.message_id = ccT.message_id + WHERE msg.message_id = ? + `; + const values1 = [messageId]; + let addressesMsg1 = await execQueryAsync(query1, values1); + + const query2 = ` + SELECT + GROUP_CONCAT(fromT.address_id) AS fromA, + GROUP_CONCAT(toT.address_id) AS toA, + GROUP_CONCAT(ccT.address_id) AS ccA + FROM message msg + ${queryFromId} fromT ON msg.message_id = fromT.message_id + ${queryToId} toT ON msg.message_id = toT.message_id + ${queryCcId} ccT ON msg.message_id = ccT.message_id + WHERE msg.messageID = ? + `; + const values2 = [messageID]; + let addressesMsg2 = await execQueryAsync(query2, values2); + + addressesMsg1 = addressesMsg1[0]?.fromA + ?.split(",") + .concat(addressesMsg1[0]?.toA?.split(",")) + .concat(addressesMsg1[0]?.ccA?.split(",")); + addressesMsg2 = addressesMsg2[0]?.fromA + ?.split(",") + .concat(addressesMsg2[0]?.toA?.split(",")) + .concat(addressesMsg2[0]?.ccA?.split(",")); + + return ( + addressesMsg1.length == addressesMsg2.length && + addressesMsg1.reduce((a, b) => a && addressesMsg2.includes(b), true) + ); } module.exports = { @@ -86,4 +129,5 @@ module.exports = { registerMessageInThread, isRoomGroup, findRoomsFromMessage, + hasSameMembersAsParent, }; diff --git a/back/db/structureV2.sql b/back/db/structureV2.sql index 4fe13f2..31f1735 100644 --- a/back/db/structureV2.sql +++ b/back/db/structureV2.sql @@ -94,10 +94,10 @@ CREATE TABLE header_field ( field_id INT NOT NULL, bodypart_id INT, part VARCHAR(128), - value TEXT NOT NULL, - UNIQUE KEY (message_id, field_id, bodypart_id), + value TEXT, + UNIQUE KEY (message_id, field_id, bodypart_id),-- todo multiple raws FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE, - FOREIGN KEY (field_id) REFERENCES field_name(field_id), -- todo on delete behavior + FOREIGN KEY (field_id) REFERENCES field_name(field_id) ON DELETE CASCADE, FOREIGN KEY (bodypart_id) REFERENCES bodypart(bodypart_id) ); diff --git a/back/db/utils/addressQueries.js b/back/db/utils/addressQueries.js new file mode 100644 index 0000000..3787764 --- /dev/null +++ b/back/db/utils/addressQueries.js @@ -0,0 +1,20 @@ +const queryAddress = (type) => ` + LEFT JOIN ( + SELECT address_field.address_id, address_field.message_id + FROM address_field + INNER JOIN field_name + WHERE + field_name.field_id = address_field.field_id AND + field_name.field_name = '${type}' + ) +`; + +const queryFromId = queryAddress("from"); +const queryToId = queryAddress("to"); +const queryCcId = queryAddress("cc"); + +module.exports = { + queryFromId, + queryToId, + queryCcId +} \ No newline at end of file diff --git a/back/imap/index.js b/back/mails/index.js similarity index 89% rename from back/imap/index.js rename to back/mails/index.js index ac18268..89e6f82 100644 --- a/back/imap/index.js +++ b/back/mails/index.js @@ -2,7 +2,7 @@ const Imap = require("imap"); const { simpleParser } = require("mailparser"); const inspect = require("util").inspect; const saveMessage = require("./storeMessage").saveMessage; -const registerMessageInApp = require("../app/saveMessage").registerMessageInApp; +const registerMessageInApp = require("./saveMessage").registerMessageInApp; const imapConfig = require("./config.json").mail; const fs = require("fs"); @@ -56,8 +56,9 @@ imap.once("ready", function () { msg.once("attributes", (attrs) => { // todo find boxId const boxId = 1; - mails.push(attrs); - promises.push(saveMessage(attrs, boxId, imap)); + console.log(attrs.envelope) + // mails.push(attrs); + // promises.push(saveMessage(attrs, boxId, imap)); }); }); @@ -85,12 +86,3 @@ imap.once("end", function () { }); imap.connect(); - -function isValidEmail(email) { - // todo - return true; -} - -module.exports = { - isValidEmail, -}; diff --git a/back/app/saveMessage.js b/back/mails/saveMessage.js similarity index 50% rename from back/app/saveMessage.js rename to back/mails/saveMessage.js index 2ca7ce8..9e72ecb 100644 --- a/back/app/saveMessage.js +++ b/back/mails/saveMessage.js @@ -4,11 +4,12 @@ const { createThread, registerMessageInThread, isRoomGroup, - findSpacesFromMessage, + findRoomsFromMessage, hasSameMembersAsParent, } = require("../db/saveMessageApp"); const { findRoomByOwner, getAddresseId } = require("../db/mail"); +const { isDmOnEnvelope } = require("./utils/statusUtils"); /** * take object address and join mailbox and host to return mailbox@host @@ -23,14 +24,16 @@ async function registerMessageInApp(messageId, attrs) { await getAddresseId(createAddress(envelope.sender[0])).then(async (ownerId) => { if (envelope.inReplyTo) { - await registerReplyMessage(envelope, messageId, isSeen); + await registerReplyMessage(envelope, messageId, isSeen, ownerId); } else { await findRoomByOwner(ownerId).then(async (res) => { if (res.length == 0) { + // first message of this sender await createRoom(envelope.subject, ownerId, messageId).then(async (roomId) => { await registerMessageInRoom(messageId, roomId, isSeen); }); } else { + // not a reply, add to the list of message if this sender await registerMessageInRoom(messageId, res[0].room_id, isSeen); } }); @@ -38,55 +41,55 @@ async function registerMessageInApp(messageId, attrs) { }); } -async function registerReplyMessage(envelope, messageId, isSeen) { +async function registerReplyMessage(envelope, messageId, isSeen, ownerId) { const messageID = envelope.messageId; - await findSpacesFromMessage(messageId).then(async (spaces) => { - // todo sub thread will not be in index 0 so look in all indexes - if (spaces.length == 0) { - // no space, so is a transfer + await findRoomsFromMessage(messageId).then(async (rooms) => { + if (rooms.length == 0) { + // no rooms, so is a transfer // todo test if members of transferred message are included - } else if (spaces[0].thread_id) { - await registerMessageInThread(messageId, spaces[0].thread_id, isSeen); - // todo - // if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) { - // // register new message in thread - // // possibly convert to room only if parent is channel - // } else { - // // todo create sub thread - // } - } else if (spaces[0].room_id) { - // message in room and not thread - await isRoomGroup(spaces[0].room_id).then(async (isGroup) => { + } else if (rooms.length === 1) { + // only one room so message is only in a room and not in a thread + // as a thread is associated to a room to begin + await isRoomGroup(rooms[0].room_id).then(async (isGroup) => { if (isGroup) { const hasSameMembers = await hasSameMembersAsParent(messageID, envelope.inReplyTo); if (hasSameMembers) { - await registerMessageInRoom(messageId, spaces[0].room_id, isSeen); + await registerMessageInRoom(messageId, rooms[0].room_id, isSeen); } else { - // group and not the same member as the reply + // is a group and has not the same member as the previous message // some recipient has been removed create a thread - const isDm = 0; // todo - await createThread(space[0].room_id, envelope.subject, isSeen, isDm).then(async (threadId) => { + const isDm = isDmOnEnvelope(envelope); + await createThread(envelope.subject, ownerId, messageId, rooms[0].room_id, isDm).then(async (threadId) => { await registerMessageInThread(messageId, threadId, isSeen); }); } } else { // reply from channel // todo - // if (messageInRoom == 1) { // was new channel transform to group - // // register new message in group - // } else if (sender == owner) { // correction from the original sender + // if (sender == owner) { // correction from the original sender // // leave in the same channel - // } else { // user response to announcement - // // create new thread // } } }); + } else if (rooms.length > 1) { + // get the lowest thread (order by room_id) + const room = rooms[rooms.length-1]; + const hasSameMembers = await hasSameMembersAsParent(messageID, envelope.inReplyTo); + if (hasSameMembers) { + await registerMessageInThread(messageId, room.room_id, isSeen); + } else { + // has not the same members so it is a derivation of this thread + // todo put this in a function and add default message in the reply chain + const isDm = isDmOnEnvelope(envelope); + await createThread(envelope.subject, ownerId, messageId, room.room_id, isDm).then(async (threadId) => { + await registerMessageInThread(messageId, threadId, isSeen); + }); + } } }); - - // `SELECT app_room_messages.room, app_room_messages.thread FROM app_room_messages INNER JOIN messages WHERE messages.messageID = '${envelope.inReplyTo}' AND app_room_messages.message = messages.id`; } module.exports = { registerMessageInApp -}; \ No newline at end of file +}; + diff --git a/back/imap/storeMessage.js b/back/mails/storeMessage.js similarity index 97% rename from back/imap/storeMessage.js rename to back/mails/storeMessage.js index d2fc0b4..7a1ed18 100644 --- a/back/imap/storeMessage.js +++ b/back/mails/storeMessage.js @@ -121,3 +121,9 @@ async function saveFromParsedData(parsed, messageId) { module.exports = { saveMessage, }; + +if (process.env['NODE_DEV'] == 'TEST') { + module.exports = { + saveFromParsedData + }; +} \ No newline at end of file diff --git a/back/mails/utils/statusUtils.js b/back/mails/utils/statusUtils.js new file mode 100644 index 0000000..f21765c --- /dev/null +++ b/back/mails/utils/statusUtils.js @@ -0,0 +1,13 @@ +function isDmOnEnvelope(envelope) { + const members = + envelope.bcc?.length + + envelope.cc?.length + + envelope.to?.length + + envelope.sender?.length + + envelope.from?.length; + return members === 2; +} + +module.exports = { + isDmOnEnvelope, +}; diff --git a/back/server.js b/back/server.js index 99651b1..2903944 100644 --- a/back/server.js +++ b/back/server.js @@ -11,4 +11,4 @@ app.use(cors()); app.listen(process.env.PORT || 5500); const mailRouter = require("./routes/mail"); -app.use("/api/mail", mailRouter); +app.use("/api/mail", mailRouter); \ No newline at end of file diff --git a/back/test/mail/saveMessage-test.js b/back/test/mail/saveMessage-test.js new file mode 100644 index 0000000..808f0ed --- /dev/null +++ b/back/test/mail/saveMessage-test.js @@ -0,0 +1,52 @@ +process.env["NODE_DEV"] = "TEST"; +const { saveFromParsedData } = require("../../mails/storeMessage"); +const { TestDb } = require("../sql/test-utilsDb"); +const MYSQL = require("../../db/config.json").MYSQL; + +let db; +beforeAll(async () => { + const options = { + databaseOptions: { + host: "localhost", + port: 3306, + user: MYSQL.user, + password: MYSQL.pwd, + database: "mail_test", + }, + dbSchema: "../../sql/structureV2.sql", + }; + db = new TestDb(options); + await db.init(); +}); + +afterAll(async () => { + db.cleanTables(); +}); + +describe("saveMessage", async () => { + describe("rooms", async () => { + it("messages not related and from same sender should be in the same room", async () => { + await saveFromParsedData( + { + to: { value: [{ address: "address1@email.com" }] }, + from: { value: [{ address: "address2@email.com" }] }, + messageId: "", + }, + 1, + ); + await saveFromParsedData( + { + to: { value: [{ address: "address1@email.com" }] }, + from: { value: [{ address: "address2@email.com" }] }, + messageId: "", + }, + 2, + ); + // todo call parser + const query = "" + db.execQueryAsync().then((res) => { + expect(res.length).toBe(2); + }) + }); + }); +}); diff --git a/back/test/sql/saveMessageApp-test.js b/back/test/sql/saveMessageApp-test.js new file mode 100644 index 0000000..6f9d12c --- /dev/null +++ b/back/test/sql/saveMessageApp-test.js @@ -0,0 +1,19 @@ +beforeAll(async () => { + // const schema = fs.readFileSync('../../sql/structureV2.sql', 'utf8'); + // await setupDB(mysqlConfig, schema); +}); + +afterAll(async () => { + global.db.query("DROP database mail_test"); +}); + +describe('saveMessageApp', async () => { + + beforeEach(() => { + + }); + + it("", () => { + + }); +}); \ No newline at end of file diff --git a/back/test/sql/test-utilsDb.js b/back/test/sql/test-utilsDb.js new file mode 100644 index 0000000..d63236f --- /dev/null +++ b/back/test/sql/test-utilsDb.js @@ -0,0 +1,62 @@ +const mysql = require("mysql"); + +export class TestDb { + constructor (options) { + this.options = options; + this.db; + } + + async init () { + return new Promise((resolve, reject) => { + this.db = mysql.createConnection({ + host: this.options.databaseOptions.host, + user: this.options.databaseOptions.user, + password: this.options.databaseOptions.pwd, + database: this.options.databaseOptions.database, + }); + + this.db.connect(async function (err) { + if (err) { + reject("Impossible de se connecter", err.code) + } else { + if (this.options.dbSchema) { + const schema = fs.readFileSync(this.options.dbSchema, 'utf8'); + await this.execQueryAsync(schema, []); + resolve("Database successfully connected and created !") + } else { + resolve("Database successfully connected"); + } + } + }); + }); + } + + cleanTables() { + const query = "SELECT table_name FROM INFORMATION_SCHEMA.tables WHERE table_schema = ?"; + const values = [this.options.databaseOptions.database]; + this.execQueryAsync(query, values).then((results) => { + this.execQuery("SET FOREIGN_KEY_CHECKS=0"); + results.map((table) => { + execQuery("DROP TABLE " + table.table_name); + }); + }); + } + + execQuery(query, values) { + db.query(query, values, (err, results, fields) => { + }); + } + + async execQueryAsync(query, values) { + return new Promise((resolve, reject) => { + db.query(query, values, (err, results, fields) => { + if (err) { + reject(err); + } else { + resolve(results); + } + }); + }); + } + +} \ No newline at end of file diff --git a/back/utils/array.js b/back/utils/array.js new file mode 100644 index 0000000..a16c02e --- /dev/null +++ b/back/utils/array.js @@ -0,0 +1,13 @@ +function removeDuplicates(array) { + let unique = []; + for (let i = 0; i < array.length; i++) { + if (!unique.includes(array[i])) { + unique.push(array[i]); + } + } + return unique; +} + +module.exports = { + removeDuplicates, +}; diff --git a/front/src/views/room/RoomView.vue b/front/src/views/room/RoomView.vue index e7cab46..be62c8a 100644 --- a/front/src/views/room/RoomView.vue +++ b/front/src/views/room/RoomView.vue @@ -61,7 +61,6 @@ onBeforeRouteUpdate(async (to, from) => { width: 100%; padding-top: 10px; - /* todo composer */ height: 35px; background-color: red; }