grimhilt/mail
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

save message working without reply

Browse Source
This commit is contained in:
grimhilt 2023-03-16 16:14:25 +01:00
parent 95f39cf53a
commit 14e64c1fc3
10 changed files with 240 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
back/app/saveMessage.js Normal file
View File

@ -0,0 +1,92 @@
const {
createRoom,
registerMessageInRoom,
createThread,
registerMessageInThread,
isRoomGroup,
findSpacesFromMessage,
hasSameMembersAsParent,
} = require("../db/saveMessageApp");
const { findRoomByOwner, getAddresseId } = require("../db/mail");
/**
* take object address and join mailbox and host to return mailbox@host
*/
function createAddress(elt) {
return `${elt.mailbox}@${elt.host}`;
}
async function registerMessageInApp(messageId, attrs) {
const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify
const envelope = attrs.envelope;
await getAddresseId(createAddress(envelope.sender[0])).then(async (ownerId) => {
if (envelope.inReplyTo) {
await registerReplyMessage(envelope, messageId, isSeen);
} else {
await findRoomByOwner(ownerId).then(async (res) => {
if (res.length == 0) {
await createRoom(envelope.subject, ownerId).then(async (roomId) => {
await registerMessageInRoom(messageId, roomId, isSeen);
});
} else {
await registerMessageInRoom(messageId, res[0].room_id, isSeen);
}
});
}
});
}
async function registerReplyMessage(envelope, messageId, isSeen) {
const messageID = envelope.messageId;
await findSpacesFromMessage(messageId).then(async (spaces) => {
// todo sub thread will not be in index 0 so look in all indexes
if (spaces.length == 0) {
// no space, so is a transfer
// todo test if members of transferred message are included
} else if (spaces[0].thread_id) {
await registerMessageInThread(messageId, spaces[0].thread_id, isSeen);
// todo
// if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) {
// // register new message in thread
// // possibly convert to room only if parent is channel
// } else {
// // todo create sub thread
// }
} else if (spaces[0].room_id) {
// message in room and not thread
await isRoomGroup(spaces[0].room_id).then(async (isGroup) => {
if (isGroup) {
const hasSameMembers = await hasSameMembersAsParent(messageID, envelope.inReplyTo);
if (hasSameMembers) {
await registerMessageInRoom(messageId, spaces[0].room_id, isSeen);
} else {
// group and not the same member as the reply
// some recipient has been removed create a thread
const isDm = 0; // todo
await createThread(space[0].room_id, envelope.subject, isSeen, isDm).then(async (threadId) => {
await registerMessageInThread(messageId, threadId, isSeen);
});
}
} else {
// reply from channel
// todo
// if (messageInRoom == 1) { // was new channel transform to group
// // register new message in group
// } else if (sender == owner) { // correction from the original sender
// // leave in the same channel
// } else { // user response to announcement
// // create new thread
// }
}
});
}
});
// `SELECT app_room_messages.room, app_room_messages.thread FROM app_room_messages INNER JOIN messages WHERE messages.messageID = '${envelope.inReplyTo}' AND app_room_messages.message = messages.id`;
}
module.exports = {
registerMessageInApp
};

12
back/sql/bdd.js → back/db/db.js
View File

@ -3,14 +3,14 @@ const MYSQL = require("./config.json").mysql;
const DEBUG = require("../utils/debug.js").DEBUG;
const bdd = mysql.createConnection({
const db = mysql.createConnection({
host: MYSQL.host,
user: MYSQL.user,
password: MYSQL.pwd,
database: MYSQL.database,
});
bdd.connect(function (err) {
db.connect(function (err) {
if (err) {
DEBUG.log("Impossible de se connecter", err.code);
} else {
@ -20,7 +20,7 @@ bdd.connect(function (err) {
function execQueryAsync(query, values) {
return new Promise((resolve, reject) => {
bdd.query(query, values, (err, results, fields) => {
db.query(query, values, (err, results, fields) => {
if (err) {
reject(err);
} else {
@ -32,7 +32,7 @@ function execQueryAsync(query, values) {
function execQueryAsyncWithId(query, values) {
return new Promise((resolve, reject) => {
bdd.query(query, values, (err, results, fields) => {
db.query(query, values, (err, results, fields) => {
if (err) {
reject(err);
} else {
@ -43,7 +43,7 @@ function execQueryAsyncWithId(query, values) {
}
function execQuery(query, values) {
bdd.query(query, values, (err, results, fields) => {
db.query(query, values, (err, results, fields) => {
if (err) {
DEBUG.log(err);
throw (err);
@ -53,7 +53,7 @@ function execQuery(query, values) {
}
module.exports = {
bdd, // todo remove this
db, // todo remove this
execQuery,
execQueryAsync,
execQueryAsyncWithId

2
back/sql/mail.js → back/db/mail.js
View File

@ -1,4 +1,4 @@
const { bdd, execQueryAsync, execQueryAsyncWithId } = require("./bdd.js");
const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js");
const DEBUG = require("../utils/debug").DEBUG;
function isValidEmail(email) {

44
back/sql/saveMessage.js → back/db/saveMessage.js
View File

@ -1,25 +1,32 @@
const { bdd, execQuery, execQueryAsync, execQueryAsyncWithId } = require("./bdd.js");
const { db, execQuery, execQueryAsync, execQueryAsyncWithId } = require("./db.js");
const DEBUG = require("../utils/debug").DEBUG;
async function registerMessage(timestamp, rfc822size, messageId) {
const query = `INSERT INTO message
(idate, messageID, rfc822size) VALUES (?, ?, ?)
ON DUPLICATE KEY UPDATE message_id = LAST_INSERT_ID(message_id)`;
const query = `
INSERT INTO message
(idate, messageID, rfc822size) VALUES (?, ?, ?)
ON DUPLICATE KEY UPDATE message_id = LAST_INSERT_ID(message_id)
`;
const values = [timestamp, messageId, rfc822size];
return await execQueryAsyncWithId(query, values);
}
function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, deleted) {
const query = `INSERT IGNORE INTO mailbox_message (mailbox_id, uid, message_id, modseq, seen, deleted) VALUES (?, ?, ?, ?, ?, ?)`;
const query = `
INSERT IGNORE INTO mailbox_message
(mailbox_id, uid, message_id, modseq, seen, deleted) VALUES (?, ?, ?, ?, ?, ?)
`;
const values = [mailboxId, uid, messageId, modseq, seen, deleted];
execQuery(query, values);
}
function registerBodypart(messageId, part, bodypartId, bytes, nbLines) {
const query = `INSERT IGNORE INTO part_number (message_id, part, bodypart_id, bytes, nb_lines) VALUES ('${messageId}', '${part}', '${bodypartId}', '${bytes}', '${nbLines}')`;
bdd.query(query, (err, results, fields) => {
if (err) DEBUG.log(err);
});
const query = `
INSERT IGNORE INTO part_number
(message_id, part, bodypart_id, bytes, nb_lines) VALUES (?, ?, ?, ?, ?)
`;
const values = [messageId, part, bodypartId, bytes, nbLines];
execQuery(query, values);
}
async function saveBodypart(bytes, hash, text, data) {
@ -29,19 +36,30 @@ async function saveBodypart(bytes, hash, text, data) {
}
async function saveHeader_fields(messageId, fieldId, bodypartId, part, value) {
const query = `INSERT IGNORE INTO header_field (message_id, field_id, bodypart_id, part, value) VALUES (?, ?, ?, ?, ?)`;
const query = `
INSERT IGNORE INTO header_field
(message_id, field_id, bodypart_id, part, value) VALUES (?, ?, ?, ?, ?)
`;
const values = [messageId, fieldId, bodypartId, part, value];
return await execQueryAsync(query, values);
}
async function saveAddress_fields(messageId, fieldId, addressId, number) {
const query = `INSERT IGNORE INTO address_field (message_id , field_id, address_id, number) VALUES (?, ?, ?, ?)`;
const query = `
INSERT IGNORE INTO address_field
(message_id , field_id, address_id, number) VALUES (?, ?, ?, ?)
`;
const values = [messageId, fieldId, addressId, number];
return execQueryAsync(query, values);
return await execQueryAsync(query, values);
}
function saveSource(messageId, content) {
// todo
const query = `
INSERT INTO source (message_id, content) VALUES (?, ?)
ON DUPLICATE KEY UPDATE content = ?
`;
const values = [messageId, content, content];
execQuery(query, values);
}
module.exports = {

47
back/sql/saveMessageApp.js → back/db/saveMessageApp.js
View File

@ -1,4 +1,4 @@
const { bdd, execQueryAsync, execQueryAsyncWithId } = require("./bdd.js");
const { db, execQueryAsync, execQueryAsyncWithId } = require("./db.js");
const DEBUG = require("../utils/debug").DEBUG;
async function createRoom(roomName, ownerId) {
@ -8,11 +8,10 @@ async function createRoom(roomName, ownerId) {
// todo add members
}
function registerMessageInRoom(messageId, roomId, isSeen) {
const query = `INSERT INTO app_space_message (message_id, room_id) VALUES ('${messageId}', '${roomId}')`;
bdd.query(query, (err, results, fields) => {
if (err) throw err;
});
async function registerMessageInRoom(messageId, roomId, isSeen) {
const query = `INSERT INTO app_space_message (message_id, room_id) VALUES (?, ?)`;
const values = [messageId, roomId];
await execQueryAsync(query, values);
updateLastUpdateRoom(roomId);
@ -29,31 +28,19 @@ function incrementNotSeenRoom(roomId) {
// todo
}
function createThread(roomId, threadName, isDm) {
return new Promise((resolve, reject) => {
const query = `INSERT INTO app_thread
(room_id, thread_name, isDm)
VALUES (
'${roomId}',
'${threadName}',
'${isDm}',
)`;
bdd.query(query, (err, results, fields) => {
if (err) reject(err);
resolve(results.insertId);
});
});
async function createThread(roomId, threadName, isDm) {
const query = `INSERT INTO app_thread (room_id, thread_name, isDm) VALUES (?, ?, ?)`;
const values = [roomId, threadName, isDm];
return await execQueryAsync(query, values);
// todo add members
}
function registerMessageInThread(messageId, threadId, isSeen) {
async function registerMessageInThread(messageId, threadId, isSeen) {
// todo check if it is still a thread or should be a room
const query = `INSERT IGNORE INTO app_space_message
(message_id, thread_id) VALUES ('${messageId}', '${threadId}')`;
bdd.query(query, (err, results, fields) => {
if (err) throw err;
});
(message_id, thread_id) VALUES (?, ?)`;
const values = [messageId, threadId];
await execQueryAsync(query, values);
updateLastUpdateThread(threadId);
if (!isSeen) {
@ -71,10 +58,10 @@ function incrementNotSeenThread(threadId) {
// also increment parent room
}
function isRoomGroup(roomId) {
async function isRoomGroup(roomId) {
return new Promise((resolve, reject) => {
const query = `SELECT isGroup FROM app_room WHERE room_id = '${roomId}'`;
bdd.query(query, (err, results, fields) => {
db.query(query, (err, results, fields) => {
if (err) reject(err);
resolve(results[0].isGroup);
});
@ -98,5 +85,5 @@ module.exports = {
createThread,
registerMessageInThread,
isRoomGroup,
findSpacesFromMessage
};
findSpacesFromMessage,
};

0
back/sql/structure → back/db/structure
View File

1
back/sql/structureV2.sql → back/db/structureV2.sql
View File

@ -145,6 +145,7 @@ CREATE TABLE app_space_message (
message_id INT NOT NULL,
room_id INT,
thread_id INT,
UNIQUE KEY (member_id, room_id, thread_id),
FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE,
FOREIGN KEY (room_id) REFERENCES app_room(room_id) ON DELETE SET NULL,
FOREIGN KEY (thread_id) REFERENCES app_thread(thread_id) ON DELETE SET NULL

53
back/imap/index.js
View File

@ -2,9 +2,11 @@ const Imap = require("imap");
const { simpleParser } = require("mailparser");
const inspect = require("util").inspect;
const saveMessage = require("./storeMessage").saveMessage;
const registerMessageInApp = require("../app/saveMessage").registerMessageInApp;
const imapConfig = require("./config.json").mail;
const fs = require("fs");
const { DEBUG } = require("../utils/debug");
const imap = new Imap({
user: imapConfig.user,
password: imapConfig.password,
@ -20,57 +22,56 @@ let shouldReset = false;
// let shouldReset = true;
if (shouldReset) {
const { execQuery, execQueryAsync } = require("../sql/bdd");
const query =
"SELECT table_name FROM INFORMATION_SCHEMA.tables WHERE table_schema = 'mail'";
const { execQuery, execQueryAsync } = require("../db/db");
const query = "SELECT table_name FROM INFORMATION_SCHEMA.tables WHERE table_schema = 'mail'";
execQueryAsync(query).then((results) => {
execQuery("SET FOREIGN_KEY_CHECKS=0");
results.map((table) => {
execQuery("DELETE FROM " + table.table_name);
});
});
return;
}
imap.once("ready", function () {
const readOnly = true;
imap.openBox("INBOX", readOnly, (err, box) => {
// console.log(box); // uidvalidty uidnext, messages total and new
imap.search(["ALL"], function (err, results) {
console.log(results[results.length - 1]);
});
// imap.search(["ALL"], function (err, results) {
// console.log(results[results.length - 1]);
// });
const f = imap.fetch(970, {
// const f = imap.fetch(970, {
// size: true,
// envelope: true,
// });
const promises = [];
const mails = [];
var f = imap.seq.fetch('1:10', {
size: true,
envelope: true,
});
let messageIDs = [];
// var f = imap.seq.fetch('1:3', {
// bodies: 'HEADER.FIELDS (FROM TO SUBJECT DATE)',
// struct: true
// });
envelope: true
});
f.on("message", function (msg, seqno) {
msg.once("attributes", (attrs) => {
// todo find boxId
const boxId = 1;
messageIDs.push(attrs.envelope.messageId)
saveMessage(attrs, boxId, imap);
mails.push(attrs);
promises.push(saveMessage(attrs, boxId, imap));
});
});
f.once("error", function (err) {
console.log("Fetch error: " + err);
DEBUG.log("Fetch error: " + err);
});
f.once("end", function () {
console.log(messageIDs)
isDone = true;
console.log("Done fetching all messages!");
f.once("end", async function () {
Promise.all(promises).then(async (res) => {
DEBUG.log("Done fetching all messages!");
for (let i = 0; i < mails.length; i++) {
await registerMessageInApp(res[i], mails[i]);
}
});
});
// imap.end()
// imap.end()
});
});

225
back/imap/storeMessage.js
View File

@ -1,4 +1,4 @@
const { getAddresseId } = require("../sql/mail");
const { getAddresseId } = require("../db/mail");
const { DEBUG } = require("../utils/debug");
const { simpleParser } = require("mailparser");
const moment = require("moment");
@ -11,95 +11,84 @@ const {
registerBodypart,
saveBodypart,
saveSource,
} = require("../sql/saveMessage");
} = require("../db/saveMessage");
const {
createRoom,
registerMessageInRoom,
createThread,
registerMessageInThread,
isRoomGroup,
findSpacesFromMessage,
hasSameMembersAsParent,
} = require("../sql/saveMessageApp");
const { getFieldId, findRoomByOwner } = require("../sql/mail");
const { getFieldId } = require("../db/mail");
function saveMessage(attrs, mailboxId, imap) {
const envelope = attrs.envelope;
const ts = moment(new Date(envelope.date).getTime()).format(
"YYYY-MM-DD HH:mm:ss"
);
const ts = moment(new Date(envelope.date).getTime()).format("YYYY-MM-DD HH:mm:ss");
const rfc822size = attrs.size;
const messageID = envelope.messageId;
registerMessage(ts, rfc822size, messageID)
.then((messageId) => {
const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify
const deleted = attrs.flags.includes("Deleted") ? 1 : 0; // todo verify
return new Promise((resolve, reject) => {
registerMessage(ts, rfc822size, messageID)
.then((messageId) => {
const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify
const deleted = attrs.flags.includes("Deleted") ? 1 : 0; // todo verify
registerMailbox_message(
mailboxId,
attrs.uid,
messageId,
attrs.modseq,
isSeen,
deleted
);
const f = imap.fetch(attrs.uid, { bodies: "" });
let buffer = "";
registerMailbox_message(mailboxId, attrs.uid, messageId, attrs.modseq, isSeen, deleted);
const f = imap.fetch(attrs.uid, { bodies: "" });
let buffer = "";
f.on("message", function (msg, seqno) {
msg.on("body", function (stream, info) {
stream.on("data", function (chunk) {
buffer += chunk.toString("utf8");
});
f.on("message", function (msg, seqno) {
msg.on("body", function (stream, info) {
stream.on("data", function (chunk) {
buffer += chunk.toString("utf8");
});
stream.once("end", () => {
// save raw data
saveSource(messageId, buffer);
stream.once("end", () => {
// save raw data
saveSource(messageId, buffer);
// parse data
simpleParser(buffer, async (err, parsed) => {
saveFromParsedData(parsed, messageId);
// parse data
simpleParser(buffer, async (err, parsed) => {
saveFromParsedData(parsed, messageId)
.then(() => {
resolve(messageId);
})
.catch((err) => {
reject(err);
});
});
});
});
});
f.once("error", function (err) {
console.log("Fetch error: " + err);
});
f.once("end", function () {
DEBUG.log("Done fetching data of "+messageID);
});
})
.catch((err) => {
DEBUG.log("Unable to register message: " + err);
reject(err);
});
f.once("error", function (err) {
console.log("Fetch error: " + err);
});
f.once("end", function () {
console.log("Done fetching all messages!");
});
})
.catch((err) => {
DEBUG.log("Unable to register message: " + err);
});
});
}
function saveFromParsedData(parsed, messageId) {
fs.writeFileSync("./test.txt", JSON.stringify(parsed));
async function saveFromParsedData(parsed, messageId) {
const promises = [];
Object.keys(parsed).forEach((key) => {
if (["from", "to", "cc", "bcc", "reply-to"].includes(key)) {
// save address field
getFieldId(key).then((fieldId) => {
parsed[key].value.forEach((addr, nb) => {
getAddresseId(addr.address, addr.name).then((addressId) => {
saveAddress_fields(messageId, fieldId, addressId, nb);
promises.push(
// save address field
getFieldId(key).then((fieldId) => {
parsed[key].value.forEach((addr, nb) => {
getAddresseId(addr.address, addr.name).then(async (addressId) => {
await saveAddress_fields(messageId, fieldId, addressId, nb);
});
});
});
});
}),
);
} else if (["subject", "inReplyTo"].includes(key)) {
// todo : "references"
getFieldId(key).then((fieldId) => {
saveHeader_fields(
messageId,
fieldId,
undefined,
undefined,
parsed[key]
);
});
promises.push(
getFieldId(key).then(async (fieldId) => {
await saveHeader_fields(messageId, fieldId, undefined, undefined, parsed[key]);
}),
);
} else if (["html", "text", "textAsHtml"].includes(key)) {
const hash = "0";
const size = "0";
@ -116,9 +105,7 @@ function saveFromParsedData(parsed, messageId) {
// });
} else if (key == "attachments") {
// todo
} else if (
["date", "messageId", "headers", "headerLines"].includes(key)
) {
} else if (["date", "messageId", "headers", "headerLines"].includes(key)) {
// messageId and date are already saved
// other field are not improted and can be retrieved in source
return;
@ -127,104 +114,10 @@ function saveFromParsedData(parsed, messageId) {
return;
}
});
return Promise.all(promises);
// todo when transfered
}
function haveSameReceivers() {
// take cc and to
}
/**
* take object address and join mailbox and host to return mailbox@host
*/
function createAddress(elt) {
return `${elt.mailbox}@${elt.host}`;
}
function registerMessageInApp(envelope, messageId, isSeen) {
getAddresseId(createAddress(envelope.sender[0])).then((ownerId) => {
if (envelope.inReplyTo) {
registerReplyMessage(envelope, messageId, isSeen);
} else {
findRoomByOwner(ownerId).then((res) => {
if (res.length == 0) {
createRoom(envelope.subject, ownerId).then((roomId) => {
registerMessageInRoom(messageId, roomId, isSeen);
});
} else {
registerMessageInRoom(messageId, res[0].room_id, isSeen);
}
});
}
});
}
function registerReplyMessage(envelope, messageId, isSeen) {
const messageID = envelope.messageId;
findSpacesFromMessage(messageId).then((spaces) => {
// todo sub thread will not be in index 0 so look in all indexes
if (spaces.length == 0) {
// no space, so is a transfer
// todo test if members of transferred message are included
} else if (spaces[0].thread_id) {
registerMessageInThread(messageId, spaces[0].thread_id, isSeen);
// todo
// if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) {
// // register new message in thread
// // possibly convert to room only if parent is channel
// } else {
// // todo create sub thread
// }
} else if (spaces[0].room_id) {
// message in room and not thread
isRoomGroup(spaces[0].room_id).then((isGroup) => {
if (isGroup) {
if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) {
registerMessageInRoom(
messageId,
spaces[0].room_id,
isSeen
);
} else {
// group and not the same member as the reply
// some recipient has been removed create a thread
const isDm = 0; // todo
createThread(
space[0].room_id,
envelope.subject,
isSeen,
isDm
).then((threadId) => {
registerMessageInThread(
messageId,
threadId,
isSeen
);
});
}
} else {
// reply from channel
// todo
// if (messageInRoom == 1) { // was new channel transform to group
// // register new message in group
// } else if (sender == owner) { // correction from the original sender
// // leave in the same channel
// } else { // user response to announcement
// // create new thread
// }
}
});
}
});
// `SELECT app_room_messages.room, app_room_messages.thread FROM app_room_messages INNER JOIN messages WHERE messages.messageID = '${envelope.inReplyTo}' AND app_room_messages.message = messages.id`;
}
function keyNormalizer(key) {
// todo
return key;
}
module.exports = {
saveMessage,
};

12
back/utils/debug.js
View File

@ -1,9 +1,9 @@
const DEBUG = (function() {
const timestamp = function(){};
timestamp.toString = () => "[DEBUG " + new Date().toLocaleDateString() + "]";
return { log: console.log.bind(console, "%s", timestamp) };
const DEBUG = (function () {
const timestamp = function () {};
timestamp.toString = () => "[" + new Date().toLocaleString() + "]";
return { log: console.log.bind(console, "%s", timestamp) };
})();
module.exports = {
DEBUG
}
DEBUG,
};