From 46ed3a1f4165dbe313591884796d873b9c26a645 Mon Sep 17 00:00:00 2001 From: grimhilt Date: Fri, 7 Apr 2023 17:43:08 +0200 Subject: [PATCH] update architecture and types --- back/db/{ => message}/saveMessage-db.ts | 10 +- back/db/{imap => message}/storeMessage-db.ts | 29 ++- back/mails/message/saveMessage.ts | 208 +++++++++++++++++++ back/mails/message/storeMessage.ts | 129 ++++++++++++ back/test/mail/saveMessage-test.ts | 4 +- back/test/test-utils/db/test-saveMessage.ts | 2 +- 6 files changed, 366 insertions(+), 16 deletions(-) rename back/db/{ => message}/saveMessage-db.ts (95%) rename back/db/{imap => message}/storeMessage-db.ts (72%) create mode 100644 back/mails/message/saveMessage.ts create mode 100644 back/mails/message/storeMessage.ts diff --git a/back/db/saveMessage-db.ts b/back/db/message/saveMessage-db.ts similarity index 95% rename from back/db/saveMessage-db.ts rename to back/db/message/saveMessage-db.ts index 0fbe00c..1da7120 100644 --- a/back/db/saveMessage-db.ts +++ b/back/db/message/saveMessage-db.ts @@ -1,8 +1,8 @@ -import { RoomType } from "../mails/saveMessage"; -import { hasSameElements } from "../utils/array"; -import { transformEmojis } from "../utils/string"; -import { execQueryAsync, execQueryAsyncWithId, execQuery } from "./db"; -import { queryFromId, queryToId, queryCcId } from "./utils/addressQueries"; +import { RoomType } from "../../mails/message/saveMessage"; +import { hasSameElements } from "../../utils/array"; +import { transformEmojis } from "../../utils/string"; +import { execQueryAsync, execQueryAsyncWithId, execQuery } from "../db"; +import { queryFromId, queryToId, queryCcId } from "../utils/addressQueries"; export async function getAllMembers(messageId: number) { const query = ` diff --git a/back/db/imap/storeMessage-db.ts b/back/db/message/storeMessage-db.ts similarity index 72% rename from back/db/imap/storeMessage-db.ts rename to back/db/message/storeMessage-db.ts index 64fe6ae..77578d6 100644 --- a/back/db/imap/storeMessage-db.ts +++ b/back/db/message/storeMessage-db.ts @@ -1,17 +1,24 @@ import { transformEmojis } from "../../utils/string"; import { execQuery, execQueryAsync, execQueryAsyncWithId } from "../db"; -export async function registerMessage(timestamp, rfc822size, messageId) { +export async function registerMessage(timestamp: string, rfc822size: number, messageID: string) { const query = ` INSERT INTO message (idate, messageID, rfc822size) VALUES (?, ?, ?) ON DUPLICATE KEY UPDATE message_id = LAST_INSERT_ID(message_id) `; - const values = [timestamp, messageId, rfc822size]; + const values = [timestamp, messageID, rfc822size]; return await execQueryAsyncWithId(query, values); } -export function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, deleted) { +export function registerMailbox_message( + mailboxId: number, + uid: number, + messageId: number, + modseq: number, + seen: boolean, + deleted: boolean, +) { const query = ` INSERT IGNORE INTO mailbox_message (mailbox_id, uid, message_id, modseq, seen, deleted) VALUES (?, ?, ?, ?, ?, ?) @@ -20,7 +27,7 @@ export function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, execQuery(query, values); } -export function registerBodypart(messageId, part, bodypartId, bytes, nbLines) { +export function registerBodypart(messageId: number, part: string, bodypartId: number, bytes: number, nbLines: null) { const query = ` INSERT IGNORE INTO part_number (message_id, part, bodypart_id, bytes, nb_lines) VALUES (?, ?, ?, ?, ?) @@ -36,7 +43,13 @@ export async function saveBodypart(bytes, hash, text, data) { return await execQueryAsyncWithId(query, values); } -export async function saveHeader_fields(messageId, fieldId, bodypartId, part, value) { +export async function saveHeader_fields( + messageId: number, + fieldId: number, + bodypartId: number, + part: string, + value: string, +) { value = transformEmojis(value); const query = ` INSERT IGNORE INTO header_field @@ -46,7 +59,7 @@ export async function saveHeader_fields(messageId, fieldId, bodypartId, part, va return await execQueryAsync(query, values); } -export async function saveAddress_fields(messageId, fieldId, addressId, number) { +export async function saveAddress_fields(messageId: number, fieldId: number, addressId: number, number: number) { const query = ` INSERT IGNORE INTO address_field (message_id , field_id, address_id, number) VALUES (?, ?, ?, ?) @@ -55,7 +68,7 @@ export async function saveAddress_fields(messageId, fieldId, addressId, number) return await execQueryAsync(query, values); } -export function saveSource(messageId, content) { +export function saveSource(messageId: number, content: string) { content = transformEmojis(content); const query = ` INSERT INTO source (message_id, content) VALUES (?, ?) @@ -63,4 +76,4 @@ export function saveSource(messageId, content) { `; const values = [messageId, content, content]; execQuery(query, values); -} \ No newline at end of file +} diff --git a/back/mails/message/saveMessage.ts b/back/mails/message/saveMessage.ts new file mode 100644 index 0000000..9c3e066 --- /dev/null +++ b/back/mails/message/saveMessage.ts @@ -0,0 +1,208 @@ +import { + createRoom, + registerMessageInRoom, + getRoomType, + findRoomsFromMessage, + hasSameMembersAsParent, + registerThread, + registerMember, + getAllMembers, + getThreadInfo, + incrementNotSeenRoom, + getThreadInfoOnId, +} from "../../db/message/saveMessage-db"; + +import { findRoomByOwner, getAddresseId, getUserIdOfMailbox } from "../../db/utils/mail"; +import { nbMembers } from "../utils/envelopeUtils"; +import logger from "../../system/Logger"; +import { Attrs, Envelope, User } from "../../interfaces/mail/attrs.interface"; + +/** + * take object address and join mailbox and host to return mailbox@host + */ +function createAddress(elt: User): string { + return `${elt.mailbox}@${elt.host}`; +} + +export enum RoomType { + ROOM = 0, + CHANNEL = 1, + GROUP = 2, + DM = 3, + THREAD = 4, +} + +export default class RegisterMessageInApp { + messageId: number; + attrs: Attrs; + envelope: Envelope; + messageID?: string; + boxId: number; + isSeen: boolean; + ownerId: number; + userId: number; + inReplyTo: string; + + constructor(_messageId: number, _attrs: Attrs, _boxId: number) { + this.messageId = _messageId; + this.attrs = _attrs; + if (!this.attrs.envelope) throw new Error("Envelope must exist in attributes"); + this.envelope = this.attrs.envelope; + this.messageID = this.envelope?.messageId; + this.boxId = _boxId; + this.isSeen = this.attrs.flags.includes("\\Seen"); + this.ownerId = -1; + this.userId = -1; + this.inReplyTo = ""; + } + + async init() { + if (this.envelope.from) { + this.ownerId = await getAddresseId(createAddress(this.envelope.from[0])); // todo use sender or from ? + } else { + throw new Error("Envelope must have a 'from' field"); + } + } + + isDm = () => nbMembers(this.envelope) == 2; + + async isFromUs() { + if (this.userId == -1) { + await getUserIdOfMailbox(this.boxId).then((res) => { + this.userId = res[0]?.user_id; + }); + } + return this.ownerId == this.userId; + } + + async incrementNotSeen(roomId: number) { + // todo it appears there is an error with notifications + if (!this.isSeen) { + await incrementNotSeenRoom(roomId); + } + } + + async registerMembers(roomId: number) { + getAllMembers(this.messageId).then((res) => { + if (res.lenght == 0) return; + const data = res[0].id.split(","); + data.forEach(async (memberId: number) => { + await registerMember(roomId, memberId); + }); + }); + } + + async initiateRoom(owner: number, roomType: RoomType) { + try { + const roomId = await createRoom(this.envelope.subject, owner, this.messageId, roomType); + await registerMessageInRoom(this.messageId, roomId, this.envelope.date); + await this.incrementNotSeen(roomId); + await this.registerMembers(roomId); + return roomId; + } catch (err) { + logger.err(err); + } + } + + async createOrRegisterOnExistence(owner: number, roomType: RoomType) { + await findRoomByOwner(owner).then(async (res) => { + if (res.length == 0) { + // first message with this sender + await this.initiateRoom(owner, roomType); + } else { + // not a reply, add to the list of message if this sender + await registerMessageInRoom(this.messageId, res[0].room_id, this.envelope.date); + await this.incrementNotSeen(res[0].room_id); + } + }); + } + + async initiateThread() { + await createRoom(this.envelope.subject, this.ownerId, this.messageId, RoomType.THREAD).then( + async (threadId: number) => { + // find parent room infos + let roomId: number; + let root_id: number; + await getThreadInfo(this.inReplyTo).then(async (room) => { + // todo room not lenght, reply to transfer ? + roomId = room[0].room_id; + root_id = room[0].root_id; + if (root_id === undefined) root_id = roomId; + await registerThread(threadId, roomId, root_id); + }); + // impl register previous message or go back + await registerMessageInRoom(this.messageId, threadId, this.envelope.date); + await this.incrementNotSeen(root_id); + await this.incrementNotSeen(threadId); + await this.registerMembers(threadId); + }, + ); + } + + async createOrRegisterOnMembers(roomId: number, isThread?: boolean) { + const hasSameMembers = await hasSameMembersAsParent(this.messageId, this.inReplyTo); + if (hasSameMembers) { + await registerMessageInRoom(this.messageId, roomId, this.envelope.date); + await this.incrementNotSeen(roomId); + if (isThread) { + await getThreadInfoOnId(roomId).then(async (res) => { + let root_id = res[0].root_id; + if (root_id == undefined) root_id = res[0].room_id; + await this.incrementNotSeen(res[0].root_id); + }); + } + } else { + await this.initiateThread(); + } + } + + async save() { + await this.init(); + if (this.envelope.inReplyTo) { + this.inReplyTo = this.envelope.inReplyTo; + await this.saveReply(); + } else { + if (await this.isFromUs()) { + if (this.isDm()) { + // create or add new message to DM + if (!this.envelope.to) throw new Error("Who send a DM and put the recipient in cc ?"); + const userTo = await getAddresseId(createAddress(this.envelope.to[0])); + await this.createOrRegisterOnExistence(userTo, RoomType.DM); + } else { + // it is not a reply and not a dm + // so it is a channel, which can be possibly a group + await this.initiateRoom(this.ownerId, RoomType.ROOM); + } + } else { + await this.createOrRegisterOnExistence(this.ownerId, RoomType.ROOM); + } + } + } + + async saveReply() { + await findRoomsFromMessage(this.inReplyTo).then(async (rooms) => { + if (rooms.length < 1) { + // no rooms, so is a transfer + // todo test if members of transferred message are included + } else if (rooms.length === 1) { + // only one room so message is only in a room and not in a thread + // as a thread is associated to a room + const roomType = (await getRoomType(rooms[0].room_id))[0].room_type; + if (roomType == RoomType.GROUP || roomType == RoomType.THREAD) { + await this.createOrRegisterOnMembers(rooms[0].room_id, roomType == RoomType.THREAD); + } else { + // reply from CHANNEL or DM or ROOM + await this.initiateThread(); + // todo + // if (sender == owner) { // correction from the original sender + // // leave in the same channel + // } + } + } else if (rooms.length > 1) { + // get the lowest thread (order by room_id) + const roomId = rooms[rooms.length - 1].room_id; + await this.createOrRegisterOnMembers(roomId); + } + }); + } +} diff --git a/back/mails/message/storeMessage.ts b/back/mails/message/storeMessage.ts new file mode 100644 index 0000000..a4a3e5d --- /dev/null +++ b/back/mails/message/storeMessage.ts @@ -0,0 +1,129 @@ +import { getAddresseId } from "../../db/utils/mail"; +import {simpleParser} from "mailparser"; +import moment from "moment"; +import Imap from "imap"; +import { + registerMessage, + registerMailbox_message, + saveHeader_fields, + saveAddress_fields, + registerBodypart, + saveBodypart, + saveSource, +} from "../../db/message/storeMessage-db"; + +import { getFieldId } from "../../db/utils/mail"; +import logger from "../../system/Logger"; +import { AttrsWithEnvelope } from "../../interfaces/mail/attrs.interface"; + +export function saveMessage(attrs: AttrsWithEnvelope, mailboxId: number, imap: Imap): Promise { + const envelope = attrs.envelope; + const ts = moment(new Date(envelope.date).getTime()).format("YYYY-MM-DD HH:mm:ss"); + const rfc822size = attrs.size; + const messageID = envelope.messageId; + + return new Promise((resolve, reject) => { + registerMessage(ts, rfc822size, messageID) + .then((messageId) => { + + const isSeen: boolean = attrs.flags.includes("\\Seen"); + const deleted: boolean = attrs.flags.includes("\\Deleted"); + + registerMailbox_message(mailboxId, attrs.uid, messageId, attrs?.modseq || 0, isSeen, deleted); + const f = imap.fetch(attrs.uid, { bodies: "" }); + let buffer = ""; + + f.on("message", function (msg, seqno) { + msg.on("body", function (stream, info) { + stream.on("data", function (chunk) { + buffer += chunk.toString("utf8"); + }); + + stream.once("end", () => { + // save raw data todo + // saveSource(messageId, buffer); + + // parse data + simpleParser(buffer, async (err, parsed) => { + saveFromParsedData(parsed, messageId) + .then(() => { + resolve(messageId); + }) + .catch((err) => { + reject(err); + }); + }); + }); + }); + }); + f.once("error", function (err) { + logger.warn("Fetch error: " + err); + }); + f.once("end", function () { + // logger.log("Done fetching data of " + messageID); // todo + }); + }) + .catch((err) => { + logger.warn("Unable to register message: " + err); + reject(err); + }); + }); +} + +async function saveFromParsedData(parsed, messageId) { + const promises: Promise[] = []; + Object.keys(parsed).forEach((key) => { + if (["from", "to", "cc", "bcc", "replyTo"].includes(key)) { + promises.push( + // save address field + getFieldId(key).then((fieldId) => { + parsed[key].value.forEach((addr, nb) => { + getAddresseId(addr.address, addr.name).then(async (addressId) => { + await saveAddress_fields(messageId, fieldId, addressId, nb); + }); + }); + }), + ); + } else if (["subject", "inReplyTo", "references"].includes(key)) { + // todo : "references" (array) + if (key == "references") return; + promises.push( + getFieldId(key).then(async (fieldId) => { + await saveHeader_fields(messageId, fieldId, undefined, undefined, parsed[key]); + }), + ); + } else if (["html", "text", "textAsHtml"].includes(key)) { + const hash = "0"; + const size = "0"; + let partType = "text/plain"; + if (key == "html") { + partType = "text/html"; + } else if (key == "textAsHtml") { + partType = "text/TexAsHtml"; + } + saveBodypart(size, hash, parsed[key], "").then((bodypartId) => { + getFieldId(key).then((fieldId) => { + saveHeader_fields(messageId, fieldId, bodypartId, partType, undefined); + }); + }); + } else if (key == "attachments") { + // todo + } else if (["date", "messageId", "headers", "headerLines"].includes(key)) { + // messageId and date are already saved + // other field are not important and can be retrieved in source + return; + } else { + logger.warn("doesn't know key: " + key); + return; + } + }); + return Promise.all(promises); + // todo when transfered +} + + +if (process.env["NODE_DEV"] == "TEST") { + module.exports = { + saveFromParsedData, + }; +} diff --git a/back/test/mail/saveMessage-test.ts b/back/test/mail/saveMessage-test.ts index 4f6296c..6974b0b 100644 --- a/back/test/mail/saveMessage-test.ts +++ b/back/test/mail/saveMessage-test.ts @@ -10,7 +10,7 @@ import { generateAttrs, generateUsers, randomInt } from "../test-utils/test-attr import { jest, describe, it, expect } from "@jest/globals"; import { mocked } from "jest-mock"; -import registerMessageInApp, { RoomType } from "../../mails/saveMessage"; +import registerMessageInApp, { RoomType } from "../../mails/message/saveMessage"; const db = new saveMessageDatabase(generateUsers(5)); const ownUser = db.users[0]; @@ -52,7 +52,7 @@ import { getThreadInfo, getThreadInfoOnId, incrementNotSeenRoom, -} from "../../db/saveMessage-db"; +} from "../../db/message/saveMessage-db"; import { AttrsWithEnvelopeTest, createReplyWithSameMembers } from "../test-utils/test-messageUtils"; // todo esbuild // new message from us diff --git a/back/test/test-utils/db/test-saveMessage.ts b/back/test/test-utils/db/test-saveMessage.ts index 5cb08b5..1904f6a 100644 --- a/back/test/test-utils/db/test-saveMessage.ts +++ b/back/test/test-utils/db/test-saveMessage.ts @@ -1,5 +1,5 @@ import { AttrsWithEnvelope, User } from "../../../interfaces/mail/attrs.interface"; -import { RoomType } from "../../../mails/saveMessage"; +import { RoomType } from "../../../mails/message/saveMessage"; import { getMembers } from "../../../mails/utils/envelopeUtils"; import { hasSameElements } from "../../../utils/array"; import { generateUsers, UserTest } from "../test-attrsUtils";