grimhilt/mail
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

save message sync

Browse Source
This commit is contained in:
grimhilt 2023-03-15 14:48:15 +01:00
parent f9fbab3a21
commit 95f39cf53a
5 changed files with 145 additions and 244 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
back/imap/index.js
View File

@ -35,6 +35,7 @@ if (shouldReset) {
return; return;
} }
imap.once("ready", function () { imap.once("ready", function () {
const readOnly = true; const readOnly = true;
imap.openBox("INBOX", readOnly, (err, box) => { imap.openBox("INBOX", readOnly, (err, box) => {
@ -45,32 +46,18 @@ imap.once("ready", function () {
const f = imap.fetch(970, { const f = imap.fetch(970, {
size: true, size: true,
struct: true,
envelope: true, envelope: true,
}); });
let messageIDs = [];
// var f = imap.seq.fetch('1:3', { // var f = imap.seq.fetch('1:3', {
// bodies: 'HEADER.FIELDS (FROM TO SUBJECT DATE)', // bodies: 'HEADER.FIELDS (FROM TO SUBJECT DATE)',
// struct: true // struct: true
// }); // });
f.on("message", function (msg, seqno) { f.on("message", function (msg, seqno) {
// console.log("Message #%d", seqno);
// var prefix = "(#" + seqno + ") ";
// msg.on("body", function (stream, info) {
// simpleParser(stream, async (err, parsed) => {
// // find box id;
// // console.log(parsed)
// const boxId = 1;
// saveMessage(parsed, boxId);
// // console.log(parsed.subject);
// // fs.writeFileSync("./test.txt", JSON.stringify(parsed));
// });
// // console.log(prefix + 'Body');
// // stream.pipe(fs.createWriteStream('msg-' + seqno + '-body.txt'));
// });
msg.once("attributes", (attrs) => { msg.once("attributes", (attrs) => {
// todo find boxId // todo find boxId
const boxId = 1; const boxId = 1;
// console.log(attrs) messageIDs.push(attrs.envelope.messageId)
saveMessage(attrs, boxId, imap); saveMessage(attrs, boxId, imap);
}); });
}); });
@ -79,71 +66,11 @@ imap.once("ready", function () {
console.log("Fetch error: " + err); console.log("Fetch error: " + err);
}); });
f.once("end", function () { f.once("end", function () {
console.log(messageIDs)
isDone = true;
console.log("Done fetching all messages!"); console.log("Done fetching all messages!");
// imap.end();
}); });
// }); // imap.end()
return;
// if (err) throw err;
// const f = imap.seq.fetch('2:2', {
// bodies: ['HEADER.FIELDS (FROM)','TEXT'],
// struct: true,
// envelope: true,
// extensions: true
// });
// f.on('message', function(msg, seqno) {
// // console.log('Message #%d', seqno);
// var prefix = '(#' + seqno + ') ';
// let attributes = undefined;
// let body = undefined;
// msg.on('body', function(stream, info) {
// simpleParser(stream, async (err, parsed) => {
// body = parsed;
// // console.log(body)
// if (attributes) {
// saveMessage(body, attributes);
// };
// // console.log(parsed.headers)
// // const {from, subject, textAsHtml, text} = parsed;
// // console.log(parsed.attachments)
// // console.log(prefix + parsed.text)
// // console.log(parsed.from.value);
// // console.log(parsed.subject);
// // console.log(parsed.date)
// // console.log(parsed.replyTo.value);
// // console.log(parsed.messageId);
// // console.log(parsed.html);
// // console.log(parsed.text);
// // console.log(parsed.textAsHtml);
// });
// });
// msg.once('attributes', attrs => {
// attributes = attrs;
// console.log(attributes)
// if (body) {
// saveMessage(body, attributes);
// };
// // console.log(prefix + 'Attributes: %s', inspect(attrs, false, 8));
// });
// msg.once('end', function() {
// console.log(prefix + 'Finished');
// });
// });
// f.once('error', function(err) {
// console.log('Fetch error: ' + err);
// });
// f.once('end', function() {
// console.log('Done fetching all messages!');
// imap.end();
// });
}); });
}); });

217
back/imap/storeMessage.js
View File

@ -2,6 +2,7 @@ const { getAddresseId } = require("../sql/mail");
const { DEBUG } = require("../utils/debug"); const { DEBUG } = require("../utils/debug");
const { simpleParser } = require("mailparser"); const { simpleParser } = require("mailparser");
const moment = require("moment"); const moment = require("moment");
const fs = require("fs");
const { const {
registerMessage, registerMessage,
registerMailbox_message, registerMailbox_message,
@ -9,6 +10,7 @@ const {
saveAddress_fields, saveAddress_fields,
registerBodypart, registerBodypart,
saveBodypart, saveBodypart,
saveSource,
} = require("../sql/saveMessage"); } = require("../sql/saveMessage");
const { const {
@ -24,114 +26,108 @@ const { getFieldId, findRoomByOwner } = require("../sql/mail");
function saveMessage(attrs, mailboxId, imap) { function saveMessage(attrs, mailboxId, imap) {
const envelope = attrs.envelope; const envelope = attrs.envelope;
const timestamp = moment(new Date(envelope.date).getTime()).format( const ts = moment(new Date(envelope.date).getTime()).format(
"YYYY-MM-DD HH:mm:ss" "YYYY-MM-DD HH:mm:ss"
); );
const rfc822size = attrs.size; const rfc822size = attrs.size;
const messageID = envelope.messageId; const messageID = envelope.messageId;
registerMessage(timestamp, rfc822size, messageID).then((messageId) => { registerMessage(ts, rfc822size, messageID)
const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify .then((messageId) => {
const deleted = attrs.flags.includes("Deleted") ? 1 : 0; // todo verify const isSeen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify
const deleted = attrs.flags.includes("Deleted") ? 1 : 0; // todo verify
registerMailbox_message( registerMailbox_message(
mailboxId, mailboxId,
attrs.uid, attrs.uid,
messageId, messageId,
attrs.modseq, attrs.modseq,
isSeen, isSeen,
deleted deleted
); );
const f = imap.fetch(attrs.uid, { bodies: "" });
let buffer = "";
const len = attrs.struct.length; f.on("message", function (msg, seqno) {
attrs.struct.forEach((part) => { msg.on("body", function (stream, info) {
stream.on("data", function (chunk) {
if (len > 1) part = part[0]; buffer += chunk.toString("utf8");
// todo should be recursive to take eveyraçpghyrue });
// todo attachments
console.log("parr", part)
if (part?.type == "text") {
console.log(attrs.uid, part.partID)
const fetch = imap.fetch(attrs.uid, { bodies: ['', part.partID] });
fetch.on("message", (msg) => { stream.once("end", () => {
msg.on("body", (stream, info) => { // save raw data
simpleParser(stream, async (err, parsed) => { saveSource(messageId, buffer);
console.log(part.partID, parsed?.subject);
// parse data
simpleParser(buffer, async (err, parsed) => {
saveFromParsedData(parsed, messageId);
}); });
}); });
msg.once("end", () => {
console.log("Finished fetching message");
});
}); });
fetch.once("error", (err) => { });
console.log(err); f.once("error", function (err) {
}); console.log("Fetch error: " + err);
});
fetch.once("end", () => { f.once("end", function () {
console.log("Done fetching all messages"); console.log("Done fetching all messages!");
}); });
const registerType = `${part.type}/${part.subtype}`; })
const hash = "2"; // todo .catch((err) => {
const text = "1"; // todo DEBUG.log("Unable to register message: " + err);
saveBodypart(part.size, hash, text, "").then((bodypartId) => {
registerBodypart(
messageId,
registerType,
bodypartId,
part.size,
part.lines
);
});
}
}); });
}
function saveFromParsedData(parsed, messageId) {
fs.writeFileSync("./test.txt", JSON.stringify(parsed));
// todo when transfered Object.keys(parsed).forEach((key) => {
//The part column records to which MIME part this header field belongs. It's empty for the main header (the one seen above) and nonempty when a multipart message has headers on each part. if (["from", "to", "cc", "bcc", "reply-to"].includes(key)) {
const part = ""; // todo ^ // save address field
// save envelope (header + from, to, subject, date, cc) getFieldId(key).then((fieldId) => {
// Object.keys(envelope).forEach((key, position) => { parsed[key].value.forEach((addr, nb) => {
// const newKey = keyNormalizer(key); getAddresseId(addr.address, addr.name).then((addressId) => {
// if (isHeader(newKey)) { saveAddress_fields(messageId, fieldId, addressId, nb);
// getFieldId(newKey).then((fieldId) => { });
// saveHeader_fields( });
// messageId, });
// part, } else if (["subject", "inReplyTo"].includes(key)) {
// position, // todo : "references"
// fieldId, getFieldId(key).then((fieldId) => {
// envelope[key] saveHeader_fields(
// ); messageId,
// }); fieldId,
// } else { undefined,
// getFieldId(newKey).then((fieldId) => { undefined,
// if (!envelope[key]) { parsed[key]
// return; );
// } });
// envelope[key].forEach((elt, index) => { } else if (["html", "text", "textAsHtml"].includes(key)) {
// getAddresseId(createAddress(elt)).then((addressId) => { const hash = "0";
// saveAddress_fields( const size = "0";
// messageId, // saveBodypart(size, hash, parsed[key], "").then((bodypartId) => {
// part, // getFieldId(key).then((fieldId) => {
// position, // saveHeader_fields(
// fieldId, // messageId,
// index, // fieldId,
// addressId // bodypartId,
// ); // undefined, // todo ?
// }); // undefined
// }); // );
// }); // });
// } // });
// }); } else if (key == "attachments") {
// todo
// todo check for different provider name of inreplyto } else if (
// registerMessageInApp(envelope, messageId, isSeen); ["date", "messageId", "headers", "headerLines"].includes(key)
}).catch((err) => { ) {
DEBUG.log("Unable to register message: "+err); // messageId and date are already saved
// other field are not improted and can be retrieved in source
return;
} else {
DEBUG.log("doesn't know key: " + key);
return;
}
}); });
// todo when transfered
} }
function haveSameReceivers() { function haveSameReceivers() {
@ -152,7 +148,6 @@ function registerMessageInApp(envelope, messageId, isSeen) {
} else { } else {
findRoomByOwner(ownerId).then((res) => { findRoomByOwner(ownerId).then((res) => {
if (res.length == 0) { if (res.length == 0) {
console.log(res, ownerId)
createRoom(envelope.subject, ownerId).then((roomId) => { createRoom(envelope.subject, ownerId).then((roomId) => {
registerMessageInRoom(messageId, roomId, isSeen); registerMessageInRoom(messageId, roomId, isSeen);
}); });
@ -225,38 +220,6 @@ function registerReplyMessage(envelope, messageId, isSeen) {
// `SELECT app_room_messages.room, app_room_messages.thread FROM app_room_messages INNER JOIN messages WHERE messages.messageID = '${envelope.inReplyTo}' AND app_room_messages.message = messages.id`; // `SELECT app_room_messages.room, app_room_messages.thread FROM app_room_messages INNER JOIN messages WHERE messages.messageID = '${envelope.inReplyTo}' AND app_room_messages.message = messages.id`;
} }
function findTextPart(struct) {
for (var i = 0, len = struct.length, r; i < len; ++i) {
if (Array.isArray(struct[i])) {
if ((r = findTextPart(struct[i]))) return r;
} else if (
struct[i].type === "text" &&
(struct[i].subtype === "plain" || struct[i].subtype === "html")
)
return [struct[i].partID, struct[i].type + "/" + struct[i].subtype];
}
}
function isHeader(key) {
switch (key) {
case "date":
case "subject":
case "messageId":
case "inReplyTo": // when transfer or reply to message
return true;
case "from":
case "sender":
case "replyTo":
case "to":
case "cc":
case "bcc":
return false;
default:
DEBUG.log("Unknown header key: " + key);
return true;
}
}
function keyNormalizer(key) { function keyNormalizer(key) {
// todo // todo
return key; return key;

41
back/sql/saveMessage.js
View File

@ -10,10 +10,9 @@ async function registerMessage(timestamp, rfc822size, messageId) {
} }
function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, deleted) { function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, deleted) {
const query = `INSERT IGNORE INTO mailbox_message (mailbox_id, uid, message_id, modseq, seen, deleted) VALUES ('${mailboxId}', '${uid}', '${messageId}', '${modseq}', '${seen}', '${deleted}')`; const query = `INSERT IGNORE INTO mailbox_message (mailbox_id, uid, message_id, modseq, seen, deleted) VALUES (?, ?, ?, ?, ?, ?)`;
bdd.query(query, (err, results, fields) => { const values = [mailboxId, uid, messageId, modseq, seen, deleted];
if (err) DEBUG.log(err); execQuery(query, values);
});
} }
function registerBodypart(messageId, part, bodypartId, bytes, nbLines) { function registerBodypart(messageId, part, bodypartId, bytes, nbLines) {
@ -23,27 +22,26 @@ function registerBodypart(messageId, part, bodypartId, bytes, nbLines) {
}); });
} }
async function saveBodypart(bytes, hash, text, data) {
function saveBodypart(bytes, hash, text, data) { const query = `INSERT IGNORE INTO bodypart (bytes, hash, text, data) VALUES (?, ?, ?,)`;
return new Promise((resolve, reject) => { const values = [bytes, hash, text, data];
const query = `INSERT IGNORE INTO bodypart (bytes, hash, text, data) VALUES ('${bytes}', '${hash}', '${text}', '${data}')`; return await execQueryAsyncWithId(query, values);
bdd.query(query, (err, results, fields) => {
if (err) reject(err);
resolve(results.insertId);
});
});
} }
function saveHeader_fields(message, part, position, field, value) { async function saveHeader_fields(messageId, fieldId, bodypartId, part, value) {
const query = `INSERT INTO header_field (message_id, part, position, field_id, value) VALUES (?, ?, ?, ?, ?)`; const query = `INSERT IGNORE INTO header_field (message_id, field_id, bodypart_id, part, value) VALUES (?, ?, ?, ?, ?)`;
const values = [message, part, position, field, value]; const values = [messageId, fieldId, bodypartId, part, value];
execQuery(query, values); return await execQueryAsync(query, values);
} }
function saveAddress_fields(message, part, position, field, number, address) { async function saveAddress_fields(messageId, fieldId, addressId, number) {
const query = `INSERT INTO address_field (message_id , part, position, field_id, number, address_id) VALUES (?, ?, ?, ?, ?, ?)`; const query = `INSERT IGNORE INTO address_field (message_id , field_id, address_id, number) VALUES (?, ?, ?, ?)`;
const values = [message, part, position, field, number, address]; const values = [messageId, fieldId, addressId, number];
execQuery(query, values); return execQueryAsync(query, values);
}
function saveSource(messageId, content) {
// todo
} }
module.exports = { module.exports = {
@ -53,4 +51,5 @@ module.exports = {
saveAddress_fields, saveAddress_fields,
registerBodypart, registerBodypart,
saveBodypart, saveBodypart,
saveSource
} }

16
back/sql/structure
View File

@ -139,3 +139,19 @@ create table app_accounts (
tls int(1) not null default 0, tls int(1) not null default 0,
primary key (id) primary key (id)
); );
create table app_rooms (
id int not null auto_increment,
name text not null,
owner int not null,
isGroup BIT(1) not null default 0,
notSeen int not null default 0,
lastUpdate timestamp not null,
primary key (id)
);
create table app_room_messages (
message int [not null]
room int,
primary key()
)

30
back/sql/structureV2.sql
View File

@ -74,15 +74,11 @@ CREATE TABLE bodypart (
); );
-- 7 -- 7
CREATE TABLE part_number ( CREATE TABLE source (
message_id INT NOT NULL, message_id INT NOT NULL,
part VARCHAR(128) NOT NULL, content TEXT NOT NULL,
bodypart_id INT NOT NULL, PRIMARY KEY (message_id),
bytes INT, FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE
nb_lines INT,
PRIMARY KEY (message_id, part),
FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE,
FOREIGN KEY (bodypart_id) REFERENCES bodypart(bodypart_id) ON DELETE CASCADE
); );
-- 8 -- 8
@ -96,24 +92,24 @@ CREATE TABLE field_name (
-- 9 -- 9
CREATE TABLE header_field ( CREATE TABLE header_field (
message_id INT NOT NULL, message_id INT NOT NULL,
part VARCHAR(128) NOT NULL,
position INT NOT NULL,
field_id INT NOT NULL, field_id INT NOT NULL,
bodypart_id INT,
part VARCHAR(128),
value TEXT NOT NULL, value TEXT NOT NULL,
UNIQUE KEY (message_id, part, position, field_id), UNIQUE KEY (message_id, field_id, bodypart_id),
FOREIGN KEY (message_id, part) REFERENCES part_number(message_id, part) ON DELETE CASCADE, FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE,
FOREIGN KEY (field_id) REFERENCES field_name(field_id) FOREIGN KEY (field_id) REFERENCES field_name(field_id), -- todo on delete behavior
FOREIGN KEY (bodypart_id) REFERENCES bodypart(bodypart_id)
); );
-- 10 -- 10
CREATE TABLE address_field ( CREATE TABLE address_field (
message_id INT NOT NULL, message_id INT NOT NULL,
part VARCHAR(128) NOT NULL,
position INT NOT NULL,
field_id INT NOT NULL, field_id INT NOT NULL,
number INT,
address_id INT NOT NULL, address_id INT NOT NULL,
FOREIGN KEY (message_id, part) REFERENCES part_number(message_id, part) ON DELETE CASCADE, number INT,
UNIQUE KEY (message_id, field_id, address_id),
FOREIGN KEY (message_id) REFERENCES message(message_id) ON DELETE CASCADE,
FOREIGN KEY (field_id) REFERENCES field_name(field_id), FOREIGN KEY (field_id) REFERENCES field_name(field_id),
FOREIGN KEY (address_id) REFERENCES address(address_id) FOREIGN KEY (address_id) REFERENCES address(address_id)
); );