grimhilt/mail
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

advancements in tests and storing messages

Browse Source
This commit is contained in:
grimhilt 2023-03-25 16:47:23 +01:00
parent 4d4ef54bcb
commit 0ea7f5865b
15 changed files with 285 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -32,6 +32,6 @@ config.json
*.txt
*.json
tmp
*test*
test.*
*.png
!*/schemas/*

30
back/db/api.js
View File

@ -1,4 +1,5 @@
const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js");
const { queryCcId, queryToId, queryFromId } = require("./utils/addressQueries.js");
const DEBUG = require("../utils/debug").DEBUG;
async function registerMailbox(userId, pwd, xoauth, xoauth2, host, port, tls) {
@ -48,8 +49,6 @@ async function getRooms(mailboxId) {
async function getMessages(roomId) {
// todo attachements name
// todo html, text, textAsHtml
// todo datetime
const query = `
SELECT
msg.message_id AS id,
@ -61,30 +60,9 @@ async function getMessages(roomId) {
message.idate AS date
FROM app_room_message msg
LEFT JOIN (
SELECT address_field.address_id, address_field.message_id
FROM address_field
INNER JOIN field_name
WHERE
field_name.field_id = address_field.field_id AND
field_name.field_name = 'from'
) fromT ON msg.message_id = fromT.message_id
LEFT JOIN (
SELECT address_field.address_id, address_field.message_id
FROM address_field
INNER JOIN field_name
WHERE
field_name.field_id = address_field.field_id AND
field_name.field_name = 'to'
) toT ON msg.message_id = toT.message_id
LEFT JOIN (
SELECT address_field.address_id, address_field.message_id
FROM address_field
INNER JOIN field_name
WHERE
field_name.field_id = address_field.field_id AND
field_name.field_name = 'cc'
) ccT ON msg.message_id = ccT.message_id
${queryFromId} fromT ON msg.message_id = fromT.message_id
${queryToId} toT ON msg.message_id = toT.message_id
${queryCcId} ccT ON msg.message_id = ccT.message_id
LEFT JOIN (
SELECT header_field.message_id, header_field.value

62
back/db/saveMessageApp.js
View File

@ -1,4 +1,5 @@
const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js");
const { queryFromId, queryToId, queryCcId } = require("./utils/addressQueries.js");
const DEBUG = require("../utils/debug").DEBUG;
async function createRoom(roomName, ownerId, messageId) {
@ -28,15 +29,18 @@ function incrementNotSeenRoom(roomId) {
// todo
}
async function createThread(roomId, threadName, isDm) {
const query = `INSERT INTO app_thread (room_id, thread_name, isDm) VALUES (?, ?, ?)`;
const values = [roomId, threadName, isDm];
async function createThread(threadName, ownerId, messageId, parentRoomId, isDm) {
const rootRoomId = -1; // todo
const threadId = await createRoom(threadName, ownerId, messageId);
const query = `INSERT INTO app_thread (room_id, parent_room_id, root_room_id, isDm) VALUES (?, ?, ?, ?)`;
const values = [threadId, parentRoomId, rootRoomId, isDm];
return await execQueryAsync(query, values);
// todo add members
}
async function registerMessageInThread(messageId, threadId, isSeen) {
// todo check if it is still a thread or should be a room
// todo isdm
const query = `INSERT IGNORE INTO app_space_message
(message_id, thread_id) VALUES (?, ?)`;
const values = [messageId, threadId];
@ -69,14 +73,53 @@ async function isRoomGroup(roomId) {
}
async function findRoomsFromMessage(messageId) {
const query = `SELECT room_id FROM app_room_message WHERE message_id = '${messageId}'`;
return await execQueryAsync(query);
const query = `SELECT room_id FROM app_room_message WHERE message_id = ? ORDER BY room_id`;
const values = [messageId];
return await execQueryAsync(query, values);
}
async function hasSameMembersAsParent(messageId, parentID) {
// const query = `SELECT `;
// return await execQueryAsync(query)
// todo
async function hasSameMembersAsParent(messageId, messageID) {
const query1 = `
SELECT
GROUP_CONCAT(fromT.address_id) AS fromA,
GROUP_CONCAT(toT.address_id) AS toA,
GROUP_CONCAT(ccT.address_id) AS ccA
FROM message msg
${queryFromId} fromT ON msg.message_id = fromT.message_id
${queryToId} toT ON msg.message_id = toT.message_id
${queryCcId} ccT ON msg.message_id = ccT.message_id
WHERE msg.message_id = ?
`;
const values1 = [messageId];
let addressesMsg1 = await execQueryAsync(query1, values1);
const query2 = `
SELECT
GROUP_CONCAT(fromT.address_id) AS fromA,
GROUP_CONCAT(toT.address_id) AS toA,
GROUP_CONCAT(ccT.address_id) AS ccA
FROM message msg
${queryFromId} fromT ON msg.message_id = fromT.message_id
${queryToId} toT ON msg.message_id = toT.message_id
${queryCcId} ccT ON msg.message_id = ccT.message_id
WHERE msg.messageID = ?
`;
const values2 = [messageID];
let addressesMsg2 = await execQueryAsync(query2, values2);
addressesMsg1 = addressesMsg1[0]?.fromA
?.split(",")
.concat(addressesMsg1[0]?.toA?.split(","))
.concat(addressesMsg1[0]?.ccA?.split(","));
addressesMsg2 = addressesMsg2[0]?.fromA
?.split(",")
.concat(addressesMsg2[0]?.toA?.split(","))
.concat(addressesMsg2[0]?.ccA?.split(","));
return (
addressesMsg1.length == addressesMsg2.length &&
addressesMsg1.reduce((a, b) => a && addressesMsg2.includes(b), true)
);
}
module.exports = {
@ -86,4 +129,5 @@ module.exports = {
registerMessageInThread,
isRoomGroup,
findRoomsFromMessage,
hasSameMembersAsParent,
};

6
back/db/structureV2.sql
View File

@ -94,10 +94,10 @@ CREATE TABLE header_field (
field_id INT NOT NULL,
bodypart_id INT,
part VARCHAR(128),
value TEXT NOT NULL,
UNIQUE KEY (message_id, field_id, bodypart_id),
value TEXT,
UNIQUE KEY (message_id, field_id, bodypart_id),-- todo multiple raws
FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE,
FOREIGN KEY (field_id) REFERENCES field_name(field_id), -- todo on delete behavior
FOREIGN KEY (field_id) REFERENCES field_name(field_id) ON DELETE CASCADE,
FOREIGN KEY (bodypart_id) REFERENCES bodypart(bodypart_id)
);

20
back/db/utils/addressQueries.js Normal file
View File

@ -0,0 +1,20 @@
const queryAddress = (type) => `
LEFT JOIN (
SELECT address_field.address_id, address_field.message_id
FROM address_field
INNER JOIN field_name
WHERE
field_name.field_id = address_field.field_id AND
field_name.field_name = '${type}'
)
`;
const queryFromId = queryAddress("from");
const queryToId = queryAddress("to");
const queryCcId = queryAddress("cc");
module.exports = {
queryFromId,
queryToId,
queryCcId
}

16
back/imap/index.js → back/mails/index.js
View File

@ -2,7 +2,7 @@ const Imap = require("imap");
const { simpleParser } = require("mailparser");
const inspect = require("util").inspect;
const saveMessage = require("./storeMessage").saveMessage;
const registerMessageInApp = require("../app/saveMessage").registerMessageInApp;
const registerMessageInApp = require("./saveMessage").registerMessageInApp;
const imapConfig = require("./config.json").mail;
const fs = require("fs");
@ -56,8 +56,9 @@ imap.once("ready", function () {
msg.once("attributes", (attrs) => {
// todo find boxId
const boxId = 1;
mails.push(attrs);
promises.push(saveMessage(attrs, boxId, imap));
console.log(attrs.envelope)
// mails.push(attrs);
// promises.push(saveMessage(attrs, boxId, imap));
});
});
@ -85,12 +86,3 @@ imap.once("end", function () {
});
imap.connect();
function isValidEmail(email) {
// todo
return true;
}
module.exports = {
isValidEmail,
};

65
back/app/saveMessage.js → back/mails/saveMessage.js
View File

@ -4,11 +4,12 @@ const {
createThread,
registerMessageInThread,
isRoomGroup,
findSpacesFromMessage,
findRoomsFromMessage,
hasSameMembersAsParent,
} = require("../db/saveMessageApp");
const { findRoomByOwner, getAddresseId } = require("../db/mail");
const { isDmOnEnvelope } = require("./utils/statusUtils");
/**
* take object address and join mailbox and host to return mailbox@host
@ -23,14 +24,16 @@ async function registerMessageInApp(messageId, attrs) {
await getAddresseId(createAddress(envelope.sender[0])).then(async (ownerId) => {
if (envelope.inReplyTo) {
await registerReplyMessage(envelope, messageId, isSeen);
await registerReplyMessage(envelope, messageId, isSeen, ownerId);
} else {
await findRoomByOwner(ownerId).then(async (res) => {
if (res.length == 0) {
// first message of this sender
await createRoom(envelope.subject, ownerId, messageId).then(async (roomId) => {
await registerMessageInRoom(messageId, roomId, isSeen);
});
} else {
// not a reply, add to the list of message if this sender
await registerMessageInRoom(messageId, res[0].room_id, isSeen);
}
});
@ -38,55 +41,55 @@ async function registerMessageInApp(messageId, attrs) {
});
}
async function registerReplyMessage(envelope, messageId, isSeen) {
async function registerReplyMessage(envelope, messageId, isSeen, ownerId) {
const messageID = envelope.messageId;
await findSpacesFromMessage(messageId).then(async (spaces) => {
// todo sub thread will not be in index 0 so look in all indexes
if (spaces.length == 0) {
// no space, so is a transfer
await findRoomsFromMessage(messageId).then(async (rooms) => {
if (rooms.length == 0) {
// no rooms, so is a transfer
// todo test if members of transferred message are included
} else if (spaces[0].thread_id) {
await registerMessageInThread(messageId, spaces[0].thread_id, isSeen);
// todo
// if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) {
// // register new message in thread
// // possibly convert to room only if parent is channel
// } else {
// // todo create sub thread
// }
} else if (spaces[0].room_id) {
// message in room and not thread
await isRoomGroup(spaces[0].room_id).then(async (isGroup) => {
} else if (rooms.length === 1) {
// only one room so message is only in a room and not in a thread
// as a thread is associated to a room to begin
await isRoomGroup(rooms[0].room_id).then(async (isGroup) => {
if (isGroup) {
const hasSameMembers = await hasSameMembersAsParent(messageID, envelope.inReplyTo);
if (hasSameMembers) {
await registerMessageInRoom(messageId, spaces[0].room_id, isSeen);
await registerMessageInRoom(messageId, rooms[0].room_id, isSeen);
} else {
// group and not the same member as the reply
// is a group and has not the same member as the previous message
// some recipient has been removed create a thread
const isDm = 0; // todo
await createThread(space[0].room_id, envelope.subject, isSeen, isDm).then(async (threadId) => {
const isDm = isDmOnEnvelope(envelope);
await createThread(envelope.subject, ownerId, messageId, rooms[0].room_id, isDm).then(async (threadId) => {
await registerMessageInThread(messageId, threadId, isSeen);
});
}
} else {
// reply from channel
// todo
// if (messageInRoom == 1) { // was new channel transform to group
// // register new message in group
// } else if (sender == owner) { // correction from the original sender
// if (sender == owner) { // correction from the original sender
// // leave in the same channel
// } else { // user response to announcement
// // create new thread
// }
}
});
} else if (rooms.length > 1) {
// get the lowest thread (order by room_id)
const room = rooms[rooms.length-1];
const hasSameMembers = await hasSameMembersAsParent(messageID, envelope.inReplyTo);
if (hasSameMembers) {
await registerMessageInThread(messageId, room.room_id, isSeen);
} else {
// has not the same members so it is a derivation of this thread
// todo put this in a function and add default message in the reply chain
const isDm = isDmOnEnvelope(envelope);
await createThread(envelope.subject, ownerId, messageId, room.room_id, isDm).then(async (threadId) => {
await registerMessageInThread(messageId, threadId, isSeen);
});
}
}
});
// `SELECT app_room_messages.room, app_room_messages.thread FROM app_room_messages INNER JOIN messages WHERE messages.messageID = '${envelope.inReplyTo}' AND app_room_messages.message = messages.id`;
}
module.exports = {
registerMessageInApp
};
};

6
back/imap/storeMessage.js → back/mails/storeMessage.js
View File

@ -121,3 +121,9 @@ async function saveFromParsedData(parsed, messageId) {
module.exports = {
saveMessage,
};
if (process.env['NODE_DEV'] == 'TEST') {
module.exports = {
saveFromParsedData
};
}

13
back/mails/utils/statusUtils.js Normal file
View File

@ -0,0 +1,13 @@
function isDmOnEnvelope(envelope) {
const members =
envelope.bcc?.length +
envelope.cc?.length +
envelope.to?.length +
envelope.sender?.length +
envelope.from?.length;
return members === 2;
}
module.exports = {
isDmOnEnvelope,
};

2
back/server.js
View File

@ -11,4 +11,4 @@ app.use(cors());
app.listen(process.env.PORT || 5500);
const mailRouter = require("./routes/mail");
app.use("/api/mail", mailRouter);
app.use("/api/mail", mailRouter);

52
back/test/mail/saveMessage-test.js Normal file
View File

@ -0,0 +1,52 @@
process.env["NODE_DEV"] = "TEST";
const { saveFromParsedData } = require("../../mails/storeMessage");
const { TestDb } = require("../sql/test-utilsDb");
const MYSQL = require("../../db/config.json").MYSQL;
let db;
beforeAll(async () => {
const options = {
databaseOptions: {
host: "localhost",
port: 3306,
user: MYSQL.user,
password: MYSQL.pwd,
database: "mail_test",
},
dbSchema: "../../sql/structureV2.sql",
};
db = new TestDb(options);
await db.init();
});
afterAll(async () => {
db.cleanTables();
});
describe("saveMessage", async () => {
describe("rooms", async () => {
it("messages not related and from same sender should be in the same room", async () => {
await saveFromParsedData(
{
to: { value: [{ address: "address1@email.com" }] },
from: { value: [{ address: "address2@email.com" }] },
messageId: "<messageId1>",
},
1,
);
await saveFromParsedData(
{
to: { value: [{ address: "address1@email.com" }] },
from: { value: [{ address: "address2@email.com" }] },
messageId: "<messageId2>",
},
2,
);
// todo call parser
const query = ""
db.execQueryAsync().then((res) => {
expect(res.length).toBe(2);
})
});
});
});

19
back/test/sql/saveMessageApp-test.js Normal file
View File

@ -0,0 +1,19 @@
beforeAll(async () => {
// const schema = fs.readFileSync('../../sql/structureV2.sql', 'utf8');
// await setupDB(mysqlConfig, schema);
});
afterAll(async () => {
global.db.query("DROP database mail_test");
});
describe('saveMessageApp', async () => {
beforeEach(() => {
});
it("", () => {
});
});

62
back/test/sql/test-utilsDb.js Normal file
View File

@ -0,0 +1,62 @@
const mysql = require("mysql");
export class TestDb {
constructor (options) {
this.options = options;
this.db;
}
async init () {
return new Promise((resolve, reject) => {
this.db = mysql.createConnection({
host: this.options.databaseOptions.host,
user: this.options.databaseOptions.user,
password: this.options.databaseOptions.pwd,
database: this.options.databaseOptions.database,
});
this.db.connect(async function (err) {
if (err) {
reject("Impossible de se connecter", err.code)
} else {
if (this.options.dbSchema) {
const schema = fs.readFileSync(this.options.dbSchema, 'utf8');
await this.execQueryAsync(schema, []);
resolve("Database successfully connected and created !")
} else {
resolve("Database successfully connected");
}
}
});
});
}
cleanTables() {
const query = "SELECT table_name FROM INFORMATION_SCHEMA.tables WHERE table_schema = ?";
const values = [this.options.databaseOptions.database];
this.execQueryAsync(query, values).then((results) => {
this.execQuery("SET FOREIGN_KEY_CHECKS=0");
results.map((table) => {
execQuery("DROP TABLE " + table.table_name);
});
});
}
execQuery(query, values) {
db.query(query, values, (err, results, fields) => {
});
}
async execQueryAsync(query, values) {
return new Promise((resolve, reject) => {
db.query(query, values, (err, results, fields) => {
if (err) {
reject(err);
} else {
resolve(results);
}
});
});
}
}

13
back/utils/array.js Normal file
View File

@ -0,0 +1,13 @@
function removeDuplicates(array) {
let unique = [];
for (let i = 0; i < array.length; i++) {
if (!unique.includes(array[i])) {
unique.push(array[i]);
}
}
return unique;
}
module.exports = {
removeDuplicates,
};

1
front/src/views/room/RoomView.vue
View File

@ -61,7 +61,6 @@ onBeforeRouteUpdate(async (to, from) => {
width: 100%;
padding-top: 10px;
/* todo composer */
height: 35px;
background-color: red;
}