advancement on logic of app

This commit is contained in:
grimhilt 2023-03-13 19:12:57 +01:00
parent aa9a69e17f
commit f9fbab3a21
12 changed files with 8974 additions and 238 deletions

View File

@ -14,6 +14,27 @@ const imap = new Imap({
tls: true,
});
// reset table;
let shouldReset = false;
// let shouldReset = true;
if (shouldReset) {
const { execQuery, execQueryAsync } = require("../sql/bdd");
const query =
"SELECT table_name FROM INFORMATION_SCHEMA.tables WHERE table_schema = 'mail'";
execQueryAsync(query).then((results) => {
execQuery("SET FOREIGN_KEY_CHECKS=0");
results.map((table) => {
execQuery("DELETE FROM " + table.table_name);
});
});
return;
}
imap.once("ready", function () {
const readOnly = true;
imap.openBox("INBOX", readOnly, (err, box) => {
@ -22,7 +43,11 @@ imap.once("ready", function () {
console.log(results[results.length - 1]);
});
const f = imap.fetch(970, { size: true, struct: true, envelope: true });
const f = imap.fetch(970, {
size: true,
struct: true,
envelope: true,
});
// var f = imap.seq.fetch('1:3', {
// bodies: 'HEADER.FIELDS (FROM TO SUBJECT DATE)',
// struct: true
@ -42,7 +67,7 @@ imap.once("ready", function () {
// // console.log(prefix + 'Body');
// // stream.pipe(fs.createWriteStream('msg-' + seqno + '-body.txt'));
// });
msg.once('attributes', attrs => {
msg.once("attributes", (attrs) => {
// todo find boxId
const boxId = 1;
// console.log(attrs)
@ -50,9 +75,6 @@ imap.once("ready", function () {
});
});
f.once("error", function (err) {
console.log("Fetch error: " + err);
});
@ -135,7 +157,6 @@ imap.once("end", function () {
imap.connect();
function isValidEmail(email) {
// todo
return true;

View File

@ -1,7 +1,7 @@
const { getAddresseId } = require("../sql/mail");
const { DEBUG } = require("../utils/debug");
const { simpleParser } = require("mailparser");
const moment = require('moment');
const moment = require("moment");
const {
registerMessage,
registerMailbox_message,
@ -17,18 +17,21 @@ const {
createThread,
registerMessageInThread,
isRoomGroup,
findSpacesFromMessage
findSpacesFromMessage,
hasSameMembersAsParent,
} = require("../sql/saveMessageApp");
const { getFieldId, findRoomByOwner } = require("../sql/mail");
function saveMessage(attrs, mailboxId, imap) {
const envelope = attrs.envelope;
const timestamp = moment(new Date(envelope.date).getTime()).format('YYYY-MM-DD HH:mm:ss');
const timestamp = moment(new Date(envelope.date).getTime()).format(
"YYYY-MM-DD HH:mm:ss"
);
const rfc822size = attrs.size;
const messageID = envelope.messageId;
registerMessage(timestamp, rfc822size, envelope.messageId).then(
(messageId) => {
const seen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify
registerMessage(timestamp, rfc822size, messageID).then((messageId) => {
const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify
const deleted = attrs.flags.includes("Deleted") ? 1 : 0; // todo verify
registerMailbox_message(
@ -36,80 +39,99 @@ function saveMessage(attrs, mailboxId, imap) {
attrs.uid,
messageId,
attrs.modseq,
seen,
isSeen,
deleted
);
const len = attrs.struct.length;
attrs.struct.forEach((part) => {
if (len > 1) part = part[0];
if (len > 1) part = part[0];
// todo should be recursive to take eveyraçpghyrue
// todo attachments
console.log("parr", part)
if (part?.type == "text") {
const registerType = `${part.type}/${part.subtype}`
console.log(attrs.uid, part.partID)
const fetch = imap.fetch(attrs.uid, { bodies: ['', part.partID] });
fetch.on("message", (msg) => {
msg.on("body", (stream, info) => {
simpleParser(stream, async (err, parsed) => {
console.log(part.partID, parsed?.subject);
});
});
msg.once("end", () => {
console.log("Finished fetching message");
});
});
fetch.once("error", (err) => {
console.log(err);
});
fetch.once("end", () => {
console.log("Done fetching all messages");
});
const registerType = `${part.type}/${part.subtype}`;
const hash = "2"; // todo
const text = "1"; // todo
saveBodypart(part.size, hash, text, '').then((bodypartId) => {
registerBodypart(messageId, registerType, bodypartId, part.size, part.lines);
saveBodypart(part.size, hash, text, "").then((bodypartId) => {
registerBodypart(
messageId,
registerType,
bodypartId,
part.size,
part.lines
);
});
}
});
// const fetch = imap.fetch(uid, { bodies: ["HEADER", part.partID] });
// fetch.on("message", (msg) => {
// msg.on("body", (stream, info) => {
// simpleParser(stream, async (err, parsed) => {
// console.log(part.partID, parsed?.subject);
// });
// });
// msg.once("end", () => {
// console.log("Finished fetching message");
// todo when transfered
//The part column records to which MIME part this header field belongs. It's empty for the main header (the one seen above) and nonempty when a multipart message has headers on each part.
const part = ""; // todo ^
// save envelope (header + from, to, subject, date, cc)
// Object.keys(envelope).forEach((key, position) => {
// const newKey = keyNormalizer(key);
// if (isHeader(newKey)) {
// getFieldId(newKey).then((fieldId) => {
// saveHeader_fields(
// messageId,
// part,
// position,
// fieldId,
// envelope[key]
// );
// });
// } else {
// getFieldId(newKey).then((fieldId) => {
// if (!envelope[key]) {
// return;
// }
// envelope[key].forEach((elt, index) => {
// getAddresseId(createAddress(elt)).then((addressId) => {
// saveAddress_fields(
// messageId,
// part,
// position,
// fieldId,
// index,
// addressId
// );
// });
// });
// fetch.once("error", (err) => {
// console.log(err);
// });
// fetch.once("end", () => {
// console.log("Done fetching all messages");
// });
// }
// });
// todo when transfered
//The part column records to which MIME part this header field belongs. It's empty for the main header (the one seen above) and nonempty when a multipart message has headers on each part.
const part = ''; // todo ^
const position = 2; // todo
// save envelope (header + from, to, subject, date, cc)
Object.keys(envelope).forEach(key => {
const newKey = keyNormalizer(key);
if (isHeader(newKey)) {
getFieldId(newKey).then((fieldId) => {
saveHeader_fields(messageId, part, position, fieldId, envelope[key]);
});
} else {
getFieldId(newKey).then((fieldId) => {
if (envelope[key]) {
envelope[key].forEach((elt, index) => {
getAddresseId(createAddress(elt)).then((addressId) => {
saveAddress_fields(messageId, part, position, fieldId, index, addressId);
});
});
}
});
}
});
// todo check for different provider name of inreplyto
// registerMessageInApp(envelope, messageId);
console.log(envelope)
}
);
// registerMessageInApp(envelope, messageId, isSeen);
}).catch((err) => {
DEBUG.log("Unable to register message: "+err);
});
}
function haveSameReceivers() {
@ -120,46 +142,73 @@ function haveSameReceivers() {
* take object address and join mailbox and host to return mailbox@host
*/
function createAddress(elt) {
return `${elt.mailbox}@${elt.host}`
return `${elt.mailbox}@${elt.host}`;
}
function registerMessageInApp(envelope, messageId) {
function registerMessageInApp(envelope, messageId, isSeen) {
getAddresseId(createAddress(envelope.sender[0])).then((ownerId) => {
if (envelope.inReplyTo) {
registerReplyMessage();
registerReplyMessage(envelope, messageId, isSeen);
} else {
findRoomByOwner(ownerId).then((res) => {
if (res.length == 0) {
registerMessageInRoom(messageId, res[0].id);
} else {
createRoom(envelope.subject, ownerId, envelope.flags.includes["seen"]).then((roomId) => {
registerMessageInRoom(messageId, roomId);
console.log(res, ownerId)
createRoom(envelope.subject, ownerId).then((roomId) => {
registerMessageInRoom(messageId, roomId, isSeen);
});
} else {
registerMessageInRoom(messageId, res[0].room_id, isSeen);
}
});
}
});
}
function registerReplyMessage(envelope, messageId) {
function registerReplyMessage(envelope, messageId, isSeen) {
const messageID = envelope.messageId;
findSpacesFromMessage(messageId).then((spaces) => {
if(spaces.length == 0) { // no space, so is a transfer
// todo sub thread will not be in index 0 so look in all indexes
if (spaces.length == 0) {
// no space, so is a transfer
// todo test if members of transferred message are included
} else if (spaces[0].room) { // message in room
} else if (spaces[0].thread_id) {
registerMessageInThread(messageId, spaces[0].thread_id, isSeen);
// todo
// if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) {
// // register new message in thread
// // possibly convert to room only if parent is channel
// } else {
// // todo create sub thread
// }
} else if (spaces[0].room_id) {
// message in room and not thread
isRoomGroup(spaces[0].room_id).then((isGroup) => {
if (isGroup) {
if (hasSameMembersAsParent(messageId)) {
registerMessageInRoom(messageId, spaces[0].room_id);
if (hasSameMembersAsParent(messageID, envelope.inReplyTo)) {
registerMessageInRoom(
messageId,
spaces[0].room_id,
isSeen
);
} else {
// group and not the same member as the reply
// some recipient has been removed create a thread
const notSeen = 0; // todo
const isDm = 0; // todo
createThread(space[0].room_id, envelope.subject, notSeen, isDm).then((threadId) => {
registerMessageInThread(messageId, threadId);
createThread(
space[0].room_id,
envelope.subject,
isSeen,
isDm
).then((threadId) => {
registerMessageInThread(
messageId,
threadId,
isSeen
);
});
}
} else { // reply from channel
} else {
// reply from channel
// todo
// if (messageInRoom == 1) { // was new channel transform to group
// // register new message in group
@ -170,32 +219,23 @@ function registerReplyMessage(envelope, messageId) {
// }
}
});
} else if (spaces[0].thread) { // message in thread
// todo
// if (hasSameMembersAsParent(messageId)) {
// // register new message in thread
// // possibly convert to room only if parent is channel
// } else {
// // create sub thread
// }
}
});
// `SELECT app_room_messages.room, app_room_messages.thread FROM app_room_messages INNER JOIN messages WHERE messages.messageID = '${envelope.inReplyTo}' AND app_room_messages.message = messages.id`;
}
function findTextPart(struct) {
for (var i = 0, len = struct.length, r; i < len; ++i) {
if (Array.isArray(struct[i])) {
if (r = findTextPart(struct[i]))
return r;
} else if (struct[i].type === 'text'
&& (struct[i].subtype === 'plain'
|| struct[i].subtype === 'html'))
return [struct[i].partID, struct[i].type + '/' + struct[i].subtype];
}
if ((r = findTextPart(struct[i]))) return r;
} else if (
struct[i].type === "text" &&
(struct[i].subtype === "plain" || struct[i].subtype === "html")
)
return [struct[i].partID, struct[i].type + "/" + struct[i].subtype];
}
}
function isHeader(key) {
switch (key) {

14
back/jest-mysql-config.js Normal file
View File

@ -0,0 +1,14 @@
const MYSQL = require("./sql/config.json").mysql;
module.exports = {
databaseOptions: {
host: "localhost",
port: 3306,
user: MYSQL.user,
password: MYSQL.pwd,
database: "mail_test",
},
createDatabase: true,
dbSchema: "./sql/structureV2.sql",
truncateDatabase: true,
};

6077
back/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -6,5 +6,15 @@
"moment": "^2.29.4",
"mysql": "^2.18.1",
"vue-router": "^4.1.6"
},
"devDependencies": {
"jest": "^29.5.0",
"jest-mysql": "^2.0.0"
},
"jest": {
"preset": "jest-mysql",
"testMatch": [
"<rootDir>/test//**/*-test.[jt]s?(x)"
]
}
}

View File

@ -18,9 +18,9 @@ bdd.connect(function (err) {
}
});
function execQueryAsync(query) {
function execQueryAsync(query, values) {
return new Promise((resolve, reject) => {
bdd.query(query, (err, results, fields) => {
bdd.query(query, values, (err, results, fields) => {
if (err) {
reject(err);
} else {
@ -30,9 +30,9 @@ function execQueryAsync(query) {
});
}
function execQueryAsyncWithId(query) {
function execQueryAsyncWithId(query, values) {
return new Promise((resolve, reject) => {
bdd.query(query, (err, results, fields) => {
bdd.query(query, values, (err, results, fields) => {
if (err) {
reject(err);
} else {
@ -42,10 +42,13 @@ function execQueryAsyncWithId(query) {
});
}
function execQuery(query) {
bdd.query(query, (err, results, fields) => {
if (err) throw (err);
return results
function execQuery(query, values) {
bdd.query(query, values, (err, results, fields) => {
if (err) {
DEBUG.log(err);
throw (err);
}
return results;
});
}

View File

@ -1,4 +1,4 @@
const { bdd, execQueryAsyncWithId } = require("./bdd.js");
const { bdd, execQueryAsync, execQueryAsyncWithId } = require("./bdd.js");
const DEBUG = require("../utils/debug").DEBUG;
function isValidEmail(email) {
@ -10,9 +10,11 @@ function isValidEmail(email) {
async function getAddresseId(email, name) {
const localpart = email.split("@")[0];
const domain = email.split("@")[1];
const query = `INSERT INTO address (address_name, localpart, domain, email) VALUES ('${name}', '${localpart}', '${domain}', '${email}')
ON DUPLICATE KEY UPDATE email = '${email}', address_id = LAST_INSERT_ID(address_id)`;
return await execQueryAsyncWithId(query);
const query = `INSERT INTO address
(address_name, localpart, domain, email) VALUES (?, ?, ?, ?)
ON DUPLICATE KEY UPDATE email = ?, address_id = LAST_INSERT_ID(address_id)`;
const values = [name, localpart, domain, email, email];
return await execQueryAsyncWithId(query, values);
}
function getMailboxId(email) {
@ -23,18 +25,15 @@ function getMailboxId(email) {
}
async function getFieldId(field) {
const query = `INSERT INTO field_name (field_name) VALUES ('${field}') ON DUPLICATE KEY UPDATE field_id=LAST_INSERT_ID(field_id)`;
return await execQueryAsyncWithId(query);
const query = `INSERT INTO field_name (field_name) VALUES (?) ON DUPLICATE KEY UPDATE field_id=LAST_INSERT_ID(field_id)`;
const values = [field]
return await execQueryAsyncWithId(query, values);
}
function findRoomByOwner(ownerId) {
return new Promise((resolve, reject) => {
const query = `SELECT room_id FROM app_room WHERE owner_id = '${ownerId}'`;
bdd.query(query, (err, results, fields) => {
if (err) reject(err);
resolve(results);
});
});
async function findRoomByOwner(ownerId) {
const query = `SELECT room_id FROM app_room WHERE owner_id = ?`;
const values = [ownerId];
return await execQueryAsync(query, values);
}
module.exports = {

View File

@ -1,15 +1,12 @@
const {bdd} = require("./bdd.js");
const { bdd, execQuery, execQueryAsync, execQueryAsyncWithId } = require("./bdd.js");
const DEBUG = require("../utils/debug").DEBUG;
function registerMessage(timestamp, rfc822size, messageId) {
return new Promise((resolve, reject) => {
const query = `INSERT INTO message (idate, messageID, rfc822size) VALUES ('${timestamp}', '${messageId}', ${rfc822size})
async function registerMessage(timestamp, rfc822size, messageId) {
const query = `INSERT INTO message
(idate, messageID, rfc822size) VALUES (?, ?, ?)
ON DUPLICATE KEY UPDATE message_id = LAST_INSERT_ID(message_id)`;
bdd.query(query, (err, results, fields) => {
if (err) reject(err);
resolve(results.insertId);
});
});
const values = [timestamp, messageId, rfc822size];
return await execQueryAsyncWithId(query, values);
}
function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, deleted) {
@ -38,17 +35,15 @@ function saveBodypart(bytes, hash, text, data) {
}
function saveHeader_fields(message, part, position, field, value) {
const query = `INSERT IGNORE INTO header_field (message_id, part, position, field_id, value) VALUES ('${message}', '${part}', '${position}', '${field}', '${value}')`;
bdd.query(query, (err, results, fields) => {
if (err) throw err;
});
const query = `INSERT INTO header_field (message_id, part, position, field_id, value) VALUES (?, ?, ?, ?, ?)`;
const values = [message, part, position, field, value];
execQuery(query, values);
}
function saveAddress_fields(message, part, position, field, number, address) {
const query = `INSERT IGNORE INTO address_field (message_id , part, position, field_id, number, address_id) VALUES ('${message}', '${part}', '${position}', '${field}', '${number}', '${address}')`;
bdd.query(query, (err, results, fields) => {
if (err) throw err;
});
const query = `INSERT INTO address_field (message_id , part, position, field_id, number, address_id) VALUES (?, ?, ?, ?, ?, ?)`;
const values = [message, part, position, field, number, address];
execQuery(query, values);
}
module.exports = {

View File

@ -1,18 +1,15 @@
const bdd = require("./bdd.js").bdd;
const { bdd, execQueryAsync, execQueryAsyncWithId } = require("./bdd.js");
const DEBUG = require("../utils/debug").DEBUG;
function createRoom(roomName, ownerId, notSeen) {
return new Promise((resolve, reject) => {
const query = `INSERT INTO app_room (room_name, owner_id, notSeen) VALUES ('${roomName}', '${ownerId}', '${notSeen}')`;
bdd.query(query, (err, results, fields) => {
if (err) reject(err);
resolve(results.insertId);
});
});
async function createRoom(roomName, ownerId) {
const query = `INSERT INTO app_room (room_name, owner_id) VALUES (?, ?)`;
const values = [roomName.substring(0, 255), ownerId];
return await execQueryAsyncWithId(query, values);
// todo add members
}
function registerMessageInRoom(messageId, roomId, isSeen) {
const query = `INSERT INTO app_room_message (message_id, room_id) VALUES ('${messageId}', '${roomId}')`;
const query = `INSERT INTO app_space_message (message_id, room_id) VALUES ('${messageId}', '${roomId}')`;
bdd.query(query, (err, results, fields) => {
if (err) throw err;
});
@ -32,14 +29,13 @@ function incrementNotSeenRoom(roomId) {
// todo
}
function createThread(roomId, threadName, notSeen, isDm) {
function createThread(roomId, threadName, isDm) {
return new Promise((resolve, reject) => {
const query = `INSERT INTO app_thread
(room_id, thread_name, notSeen, isDm)
(room_id, thread_name, isDm)
VALUES (
'${roomId}',
'${threadName}',
'${notSeen}',
'${isDm}',
)`;
bdd.query(query, (err, results, fields) => {
@ -47,11 +43,13 @@ function createThread(roomId, threadName, notSeen, isDm) {
resolve(results.insertId);
});
});
// todo add members
}
function registerMessageInThread(messageId, threadId, isSeen) {
// todo check if it is still a thread or should be a room
const query = `INSERT IGNORE INTO app_room_messages
const query = `INSERT IGNORE INTO app_space_message
(message_id, thread_id) VALUES ('${messageId}', '${threadId}')`;
bdd.query(query, (err, results, fields) => {
if (err) throw err;
@ -83,8 +81,15 @@ function isRoomGroup(roomId) {
});
}
function findSpacesFromMessage(messageId) {
const query = ``;
async function findSpacesFromMessage(messageId) {
const query = `SELECT room_id, thread_id FROM app_space_message WHERE message_id = '${messageId}'`;
return await execQueryAsync(query);
}
async function hasSameMembersAsParent(messageId, parentID) {
// const query = `SELECT `;
// return await execQueryAsync(query)
// todo
}
module.exports = {

View File

@ -1,6 +1,4 @@
/**
* Mail storage
*/
-- Mail storage
-- 1
CREATE TABLE address (
@ -13,6 +11,20 @@ CREATE TABLE address (
UNIQUE KEY (email)
);
-- 2 app
CREATE TABLE app_account (
account_id INT AUTO_INCREMENT,
user_id INT NOT NULL,
account_pwd BINARY(22),
xoauth VARCHAR(116),
xoauth2 VARCHAR(116),
host VARCHAR(255) NOT NULL DEFAULT 'localhost',
port INT(5) NOT NULL DEFAULT 143,
tls BOOLEAN NOT NULL DEFAULT true,
PRIMARY KEY (account_id),
FOREIGN KEY (user_id) REFERENCES address(address_id) ON DELETE CASCADE
);
-- 3
CREATE TABLE mailbox (
mailbox_id INT AUTO_INCREMENT,
@ -44,14 +56,24 @@ CREATE TABLE mailbox_message (
uid INT,
message_id INT,
modseq BIGINT NOT NULL,
seen BIT(1) NOT NULL DEFAULT 0,
deleted BIT(1) NOT NULL DEFAULT 0,
seen BOOLEAN NOT NULL DEFAULT false,
deleted BOOLEAN NOT NULL DEFAULT false,
PRIMARY KEY (uid, message_id),
FOREIGN KEY (mailbox_id) REFERENCES mailbox(mailbox_id) ON DELETE CASCADE,
FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE
);
-- 6
CREATE TABLE bodypart (
bodypart_id INT AUTO_INCREMENT,
bytes INT NOT NULL,
hash TEXT NOT NULL,
text TEXT,
data BINARY,
PRIMARY KEY (bodypart_id)
);
-- 7
CREATE TABLE part_number (
message_id INT NOT NULL,
part VARCHAR(128) NOT NULL,
@ -63,16 +85,6 @@ CREATE TABLE part_number (
FOREIGN KEY (bodypart_id) REFERENCES bodypart(bodypart_id) ON DELETE CASCADE
);
-- 7
CREATE TABLE bodypart (
bodypart_id INT AUTO_INCREMENT,
bytes INT NOT NULL,
hash TEXT NOT NULL,
text TEXT,
data BINARY,
PRIMARY KEY (bodypart_id)
);
-- 8
CREATE TABLE field_name (
field_id INT AUTO_INCREMENT,
@ -106,30 +118,14 @@ CREATE TABLE address_field (
FOREIGN KEY (address_id) REFERENCES address(address_id)
);
/**
* App table
*/
-- 2
CREATE TABLE app_account (
account_id INT AUTO_INCREMENT,
user_id INT NOT NULL,
account_pwd BINARY(22),
xoauth VARCHAR(116),
xoauth2 VARCHAR(116),
host VARCHAR(255) NOT NULL DEFAULT 'localhost',
port INT(5) NOT NULL DEFAULT 143,
tls BIT(1) NOT NULL DEFAULT 1,
PRIMARY KEY (account_id),
FOREIGN KEY (user_id) REFERENCES address(address_id) ON DELETE CASCADE
);
-- App table
-- 11
CREATE TABLE app_room (
room_id INT AUTO_INCREMENT,
room_name VARCHAR(30) NOT NULL,
room_name VARCHAR(255) NOT NULL,
owner_id INT NOT NULL,
isGroup BIT(1) NOT NULL DEFAULT 0,
isGroup BOOLEAN NOT NULL DEFAULT false,
notSeen INT NOT NULL DEFAULT 0,
lastUpdate TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP(),
PRIMARY KEY (room_id),
@ -140,16 +136,16 @@ CREATE TABLE app_room (
CREATE TABLE app_thread (
thread_id INT AUTO_INCREMENT,
room_id INT NOT NULL,
thread_name VARCHAR(30) NOT NULL,
thread_name VARCHAR(255) NOT NULL,
notSeen INT NOT NULL DEFAULT 0,
lastUpdate TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP(),
isDm BIT(1) NOT NULL DEFAULT 0,
isDm BOOLEAN NOT NULL DEFAULT false,
PRIMARY KEY (thread_id),
FOREIGN KEY (room_id) REFERENCES app_room(room_id) ON DELETE CASCADE
);
-- 13
CREATE TABLE app_room_message (
CREATE TABLE app_space_message (
message_id INT NOT NULL,
room_id INT,
thread_id INT,

View File

@ -5,5 +5,5 @@ const DEBUG = (function() {
})();
module.exports = {
DEBUG: DEBUG
DEBUG
}

2606
back/yarn.lock Normal file

File diff suppressed because it is too large Load Diff