grimhilt/mail
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

gloablly save messages

Browse Source
This commit is contained in:
grimhilt 2023-03-10 17:07:05 +01:00
parent df69a7dbd9
commit 29bf4bbdbd
4 changed files with 123 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
back/imap/index.js
View File

@ -18,49 +18,40 @@ imap.once("ready", function () {
const readOnly = true; const readOnly = true;
imap.openBox("INBOX", readOnly, (err, box) => { imap.openBox("INBOX", readOnly, (err, box) => {
// console.log(box); // uidvalidty uidnext, messages total and new // console.log(box); // uidvalidty uidnext, messages total and new
// imap.search(["ALL"], function (err, results) { imap.search(["ALL"], function (err, results) {
// console.log(results[results.length - 1]); console.log(results[results.length - 1]);
// }); });
const fetchOptions = { const f = imap.fetch(969, { size: true, struct: true, envelope: true });
bodies: ["HEADER.FIELDS (FROM TO SUBJECT DATE)", "TEXT"],
struct: true,
};
const f = imap.fetch(969, fetchOptions);
// var f = imap.seq.fetch('1:3', { // var f = imap.seq.fetch('1:3', {
// bodies: 'HEADER.FIELDS (FROM TO SUBJECT DATE)', // bodies: 'HEADER.FIELDS (FROM TO SUBJECT DATE)',
// struct: true // struct: true
// }); // });
f.on("message", function (msg, seqno) { f.on("message", function (msg, seqno) {
const emailData = {}; // console.log("Message #%d", seqno);
msg.on("body", (stream, info) => { // var prefix = "(#" + seqno + ") ";
let buffer = ""; // msg.on("body", function (stream, info) {
stream.on("data", (chunk) => { // simpleParser(stream, async (err, parsed) => {
buffer += chunk.toString("utf8"); // // find box id;
}); // console.log(parsed)
stream.on("end", () => { // const boxId = 1;
if (info.which === "TEXT") { // // saveMessage(parsed, boxId);
emailData.text = buffer; // console.log(parsed.subject);
} else { // fs.writeFileSync("./test.json", JSON.stringify(parsed));
const parsedHeader = Imap.parseHeader(buffer); // });
emailData.from = parsedHeader.from[0]; // // console.log(prefix + 'Body');
emailData.to = parsedHeader.to[0]; // // stream.pipe(fs.createWriteStream('msg-' + seqno + '-body.txt'));
emailData.subject = parsedHeader.subject[0]; // });
emailData.date = parsedHeader.date[0]; msg.once('attributes', attrs => {
} // todo find boxId
}); const boxId = 1;
}); saveMessage(attrs, boxId, imap);
msg.once("attributes", (attrs) => {
emailData.uid = attrs.uid;
emailData.flags = attrs.flags;
emailData.struct = attrs.struct;
});
msg.once("end", () => {
console.log("Email data:", inspect(emailData));
}); });
}); });
f.once("error", function (err) { f.once("error", function (err) {
console.log("Fetch error: " + err); console.log("Fetch error: " + err);
}); });

146
back/imap/storeMessage.js
View File

@ -7,15 +7,16 @@ const {
registerMailbox_message, registerMailbox_message,
saveHeader_fields, saveHeader_fields,
saveAddress_fields, saveAddress_fields,
registerBodypart,
saveBodypart
} = require("../sql/saveMessage"); } = require("../sql/saveMessage");
const { getMailboxId, getField } = require("../sql/mail"); const { getFieldId } = require("../sql/mail");
function saveMessage(attrs, mailboxId, imap) { function saveMessage(attrs, mailboxId, imap) {
console.log("SAVE MSG")
// console.log(attrs.struct);
const envelope = attrs.envelope; const envelope = attrs.envelope;
const timestamp = new Date(envelope.date).getTime(); const timestamp = new Date(envelope.date).getTime();
const rfc822size = attrs.size; const rfc822size = attrs.size;
registerMessage(timestamp, rfc822size, envelope.messageId).then( registerMessage(timestamp, rfc822size, envelope.messageId).then(
(messageId) => { (messageId) => {
const seen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify const seen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify
@ -29,90 +30,102 @@ function saveMessage(attrs, mailboxId, imap) {
seen, seen,
deleted deleted
); );
// // attributes.struct.forEach(part => {
// // // saveBodyparts().then((bodypartId) => {
// // // const partText = undefined;
// // // savePart_numbers(messageId, partText, bodypartId, part.size, part.lines)
// // // });
// // });
const len = attrs.struct.length; const len = attrs.struct.length;
console.log(attrs.struct); attrs.struct.forEach((part) => {
// // attrs.struct.forEach((part) => { if (len > 1) part = part[0];
// // console.log(part)
// console.log("----------");
// if (len > 1) part = part[0];
// if (part?.type == "text") {
// console.log(part.partID);
// console.log(attrs.uid);
// const uid = attrs.uid;
// const bodies = ["HEADER", "1.TEXT", "2.TEXT", "2.HTML"];
// const fetchOptions = { // todo should be recursive to take eveyraçpghyrue
// bodies: bodies, // todo attachments
// }; if (part?.type == "text") {
const registerType = `${part.type}/${part.subtype}`
const hash = "2"; // todo
const text = "1"; // todo
saveBodypart(part.size, hash, text, '').then((bodypartId) => {
registerBodypart(messageId, registerType, bodypartId, part.size, part.lines);
});
}
});
const fetch = imap.fetch(attrs.uid, { bodies: ["HEADER.FIELDS (FROM)", 2] }); // const fetch = imap.fetch(uid, { bodies: ["HEADER", part.partID] });
fetch.on("message", (msg) => { // fetch.on("message", (msg) => {
msg.on("body", (stream, info) => { // msg.on("body", (stream, info) => {
simpleParser(stream, async (err, parsed) => { // simpleParser(stream, async (err, parsed) => {
console.log(parsed); // console.log(part.partID, parsed?.subject);
}); // });
}); // });
msg.once("end", () => { // msg.once("end", () => {
console.log("Finished fetching message"); // console.log("Finished fetching message");
}); // });
}); // });
fetch.once("error", (err) => { // fetch.once("error", (err) => {
console.log(err); // console.log(err);
}); // });
fetch.once("end", () => { // fetch.once("end", () => {
console.log("Done fetching all messages"); // console.log("Done fetching all messages");
}); // });
// } // }
// }); // });
// // const part = ''; // todo when transfered // todo when transfered
//The part column records to which MIME part this header field belongs. It's empty for the main header (the one seen above) and nonempty when a multipart message has headers on each part.
const part = ''; // todo ^
const position = 2; // todo
// save envelope (header + from, to, subject, date, cc)
Object.keys(envelope).forEach(key => {
const newKey = keyNormalizer(key);
if (isHeader(newKey)) {
getFieldId(newKey).then((fieldId) => {
saveHeader_fields(messageId, part, position, fieldId, envelope[key]);
});
} else {
getFieldId(newKey).then((fieldId) => {
if (envelope[key]) {
envelope[key].forEach((elt, index) => {
getAddresseId(`${elt.mailbox}@${elt.host}`).then((addressId) => {
saveAddress_fields(messageId, part, position, fieldId, index, addressId);
});
});
}
});
}
});
// // save envelope (header + from, to, subject, date, cc) // save more header
// Object.keys(attributes.envelope).forEach(key => { // message.headerLines.forEach(elt => {
// const newKey = keyNormalizer(key); // const newKey = keyNormalizer(elt.key);
// if (isHeader(newKey)) { // getFieldId(newKey).then((fieldId) => {
// getField(newKey).then((fieldId) => { // saveHeader_fields(messageId, part, 2, fieldId, elt[line]);
// saveHeader_fields(messageId, part, 2, fieldId, attributes.envelope[key]);
// });
// } else {
// getField(newKey).then((fieldId) => {
// if (attributes.envelope[key]) {
// attributes.envelope[key].forEach((elt, index) => {
// saveAddress_fields(messageId, part, fieldId, index, getAddresseId(`${elt.mailbox}@${elt.host}`, elt.name));
// });
// }
// });
// }
// }); // });
// });
// // // save more header // // todo add date field
// // message.headerLines.forEach(elt => {
// // const newKey = keyNormalizer(elt.key);
// // getField(newKey).then((fieldId) => {
// // saveHeader_fields(messageId, part, 2, fieldId, elt[line]);
// // });
// // });
// // // todo add date field
} }
); );
} }
function findTextPart(struct) {
for (var i = 0, len = struct.length, r; i < len; ++i) {
if (Array.isArray(struct[i])) {
if (r = findTextPart(struct[i]))
return r;
} else if (struct[i].type === 'text'
&& (struct[i].subtype === 'plain'
|| struct[i].subtype === 'html'))
return [struct[i].partID, struct[i].type + '/' + struct[i].subtype];
}
}
function isHeader(key) { function isHeader(key) {
switch (key) { switch (key) {
case "date": case "date":
case "subject": case "subject":
case "messageId": case "messageId":
case "inReplyTo": // when transfer or reply to message
return true; return true;
case "from": case "from":
case "sender": case "sender":
@ -120,7 +133,6 @@ function isHeader(key) {
case "to": case "to":
case "cc": case "cc":
case "bcc": case "bcc":
case "inReplyTo":
return false; return false;
default: default:
DEBUG.log("Unknown header key: " + key); DEBUG.log("Unknown header key: " + key);

0
back/routers/mails.js → back/routers/mail.js
View File

20
back/sql/saveMessage.js
View File

@ -19,7 +19,23 @@ function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, delete
}); });
} }
function saveBodyparts() {} function registerBodypart(messageId, part, bodypartId, bytes, nbLines) {
const query = `INSERT IGNORE INTO part_numbers (message, part, bodypart, bytes, nbLines) VALUES ('${messageId}', '${part}', '${bodypartId}', '${bytes}', '${nbLines}')`;
bdd.query(query, (err, results, fields) => {
if (err) DEBUG.log(err);
});
}
function saveBodypart(bytes, hash, text, data) {
return new Promise((resolve, reject) => {
const query = `INSERT IGNORE INTO bodyparts (bytes, hash, text, data) VALUES ('${bytes}', '${hash}', '${text}', '${data}')`;
bdd.query(query, (err, results, fields) => {
if (err) reject(err);
resolve(results.insertId);
});
});
}
function saveHeader_fields(message, part, position, field, value) { function saveHeader_fields(message, part, position, field, value) {
const query = `INSERT IGNORE INTO header_fields (message, part, position, field, value) VALUES ('${message}', '${part}', '${position}', '${field}', '${value}')`; const query = `INSERT IGNORE INTO header_fields (message, part, position, field, value) VALUES ('${message}', '${part}', '${position}', '${field}', '${value}')`;
@ -40,4 +56,6 @@ module.exports = {
registerMailbox_message, registerMailbox_message,
saveHeader_fields, saveHeader_fields,
saveAddress_fields, saveAddress_fields,
registerBodypart,
saveBodypart
} }