From 29bf4bbdbda32cd4453754353cfb47a622171156 Mon Sep 17 00:00:00 2001 From: grimhilt Date: Fri, 10 Mar 2023 17:07:05 +0100 Subject: [PATCH] gloablly save messages --- back/imap/index.js | 59 +++++------- back/imap/storeMessage.js | 146 ++++++++++++++++------------- back/routers/{mails.js => mail.js} | 0 back/sql/saveMessage.js | 20 +++- 4 files changed, 123 insertions(+), 102 deletions(-) rename back/routers/{mails.js => mail.js} (100%) diff --git a/back/imap/index.js b/back/imap/index.js index 38a39a5..5c537fa 100644 --- a/back/imap/index.js +++ b/back/imap/index.js @@ -18,49 +18,40 @@ imap.once("ready", function () { const readOnly = true; imap.openBox("INBOX", readOnly, (err, box) => { // console.log(box); // uidvalidty uidnext, messages total and new - // imap.search(["ALL"], function (err, results) { - // console.log(results[results.length - 1]); - // }); + imap.search(["ALL"], function (err, results) { + console.log(results[results.length - 1]); + }); - const fetchOptions = { - bodies: ["HEADER.FIELDS (FROM TO SUBJECT DATE)", "TEXT"], - struct: true, - }; - - const f = imap.fetch(969, fetchOptions); + const f = imap.fetch(969, { size: true, struct: true, envelope: true }); // var f = imap.seq.fetch('1:3', { // bodies: 'HEADER.FIELDS (FROM TO SUBJECT DATE)', // struct: true // }); f.on("message", function (msg, seqno) { - const emailData = {}; - msg.on("body", (stream, info) => { - let buffer = ""; - stream.on("data", (chunk) => { - buffer += chunk.toString("utf8"); - }); - stream.on("end", () => { - if (info.which === "TEXT") { - emailData.text = buffer; - } else { - const parsedHeader = Imap.parseHeader(buffer); - emailData.from = parsedHeader.from[0]; - emailData.to = parsedHeader.to[0]; - emailData.subject = parsedHeader.subject[0]; - emailData.date = parsedHeader.date[0]; - } - }); - }); - msg.once("attributes", (attrs) => { - emailData.uid = attrs.uid; - emailData.flags = attrs.flags; - emailData.struct = attrs.struct; - }); - msg.once("end", () => { - console.log("Email data:", inspect(emailData)); + // console.log("Message #%d", seqno); + // var prefix = "(#" + seqno + ") "; + // msg.on("body", function (stream, info) { + // simpleParser(stream, async (err, parsed) => { + // // find box id; + // console.log(parsed) + // const boxId = 1; + // // saveMessage(parsed, boxId); + // console.log(parsed.subject); + // fs.writeFileSync("./test.json", JSON.stringify(parsed)); + // }); + // // console.log(prefix + 'Body'); + // // stream.pipe(fs.createWriteStream('msg-' + seqno + '-body.txt')); + // }); + msg.once('attributes', attrs => { + // todo find boxId + const boxId = 1; + saveMessage(attrs, boxId, imap); }); }); + + + f.once("error", function (err) { console.log("Fetch error: " + err); }); diff --git a/back/imap/storeMessage.js b/back/imap/storeMessage.js index 49d2d92..6de75e1 100644 --- a/back/imap/storeMessage.js +++ b/back/imap/storeMessage.js @@ -7,15 +7,16 @@ const { registerMailbox_message, saveHeader_fields, saveAddress_fields, + registerBodypart, + saveBodypart } = require("../sql/saveMessage"); -const { getMailboxId, getField } = require("../sql/mail"); +const { getFieldId } = require("../sql/mail"); function saveMessage(attrs, mailboxId, imap) { - console.log("SAVE MSG") - // console.log(attrs.struct); const envelope = attrs.envelope; const timestamp = new Date(envelope.date).getTime(); const rfc822size = attrs.size; + registerMessage(timestamp, rfc822size, envelope.messageId).then( (messageId) => { const seen = attrs.flags.includes("Seen") ? 1 : 0; // todo verify @@ -29,90 +30,102 @@ function saveMessage(attrs, mailboxId, imap) { seen, deleted ); - // // attributes.struct.forEach(part => { - // // // saveBodyparts().then((bodypartId) => { - // // // const partText = undefined; - // // // savePart_numbers(messageId, partText, bodypartId, part.size, part.lines) - // // // }); - // // }); + const len = attrs.struct.length; - console.log(attrs.struct); - // // attrs.struct.forEach((part) => { - // // console.log(part) - // console.log("----------"); - // if (len > 1) part = part[0]; - // if (part?.type == "text") { - // console.log(part.partID); - // console.log(attrs.uid); - // const uid = attrs.uid; - // const bodies = ["HEADER", "1.TEXT", "2.TEXT", "2.HTML"]; + attrs.struct.forEach((part) => { + if (len > 1) part = part[0]; - // const fetchOptions = { - // bodies: bodies, - // }; + // todo should be recursive to take eveyraçpghyrue + // todo attachments + if (part?.type == "text") { + const registerType = `${part.type}/${part.subtype}` + const hash = "2"; // todo + const text = "1"; // todo + saveBodypart(part.size, hash, text, '').then((bodypartId) => { + registerBodypart(messageId, registerType, bodypartId, part.size, part.lines); + }); + } + }); - const fetch = imap.fetch(attrs.uid, { bodies: ["HEADER.FIELDS (FROM)", 2] }); + // const fetch = imap.fetch(uid, { bodies: ["HEADER", part.partID] }); - fetch.on("message", (msg) => { - msg.on("body", (stream, info) => { - simpleParser(stream, async (err, parsed) => { - console.log(parsed); - }); - }); + // fetch.on("message", (msg) => { + // msg.on("body", (stream, info) => { + // simpleParser(stream, async (err, parsed) => { + // console.log(part.partID, parsed?.subject); + // }); + // }); - msg.once("end", () => { - console.log("Finished fetching message"); - }); - }); + // msg.once("end", () => { + // console.log("Finished fetching message"); + // }); + // }); - fetch.once("error", (err) => { - console.log(err); - }); + // fetch.once("error", (err) => { + // console.log(err); + // }); - fetch.once("end", () => { - console.log("Done fetching all messages"); - }); - // } + // fetch.once("end", () => { + // console.log("Done fetching all messages"); + // }); + // } // }); - // // const part = ''; // todo when transfered + // todo when transfered + //The part column records to which MIME part this header field belongs. It's empty for the main header (the one seen above) and nonempty when a multipart message has headers on each part. + const part = ''; // todo ^ + const position = 2; // todo + // save envelope (header + from, to, subject, date, cc) + Object.keys(envelope).forEach(key => { + const newKey = keyNormalizer(key); + if (isHeader(newKey)) { + getFieldId(newKey).then((fieldId) => { + saveHeader_fields(messageId, part, position, fieldId, envelope[key]); + }); + } else { + getFieldId(newKey).then((fieldId) => { + if (envelope[key]) { + envelope[key].forEach((elt, index) => { + getAddresseId(`${elt.mailbox}@${elt.host}`).then((addressId) => { + saveAddress_fields(messageId, part, position, fieldId, index, addressId); + }); + }); + } + }); + } + }); - // // save envelope (header + from, to, subject, date, cc) - // Object.keys(attributes.envelope).forEach(key => { - // const newKey = keyNormalizer(key); - // if (isHeader(newKey)) { - // getField(newKey).then((fieldId) => { - // saveHeader_fields(messageId, part, 2, fieldId, attributes.envelope[key]); - // }); - // } else { - // getField(newKey).then((fieldId) => { - // if (attributes.envelope[key]) { - // attributes.envelope[key].forEach((elt, index) => { - // saveAddress_fields(messageId, part, fieldId, index, getAddresseId(`${elt.mailbox}@${elt.host}`, elt.name)); - // }); - // } - // }); - // } + // save more header + // message.headerLines.forEach(elt => { + // const newKey = keyNormalizer(elt.key); + // getFieldId(newKey).then((fieldId) => { + // saveHeader_fields(messageId, part, 2, fieldId, elt[line]); // }); + // }); - // // // save more header - // // message.headerLines.forEach(elt => { - // // const newKey = keyNormalizer(elt.key); - // // getField(newKey).then((fieldId) => { - // // saveHeader_fields(messageId, part, 2, fieldId, elt[line]); - // // }); - // // }); - - // // // todo add date field + // // todo add date field } ); } +function findTextPart(struct) { + for (var i = 0, len = struct.length, r; i < len; ++i) { + if (Array.isArray(struct[i])) { + if (r = findTextPart(struct[i])) + return r; + } else if (struct[i].type === 'text' + && (struct[i].subtype === 'plain' + || struct[i].subtype === 'html')) + return [struct[i].partID, struct[i].type + '/' + struct[i].subtype]; + } + } + function isHeader(key) { switch (key) { case "date": case "subject": case "messageId": + case "inReplyTo": // when transfer or reply to message return true; case "from": case "sender": @@ -120,7 +133,6 @@ function isHeader(key) { case "to": case "cc": case "bcc": - case "inReplyTo": return false; default: DEBUG.log("Unknown header key: " + key); diff --git a/back/routers/mails.js b/back/routers/mail.js similarity index 100% rename from back/routers/mails.js rename to back/routers/mail.js diff --git a/back/sql/saveMessage.js b/back/sql/saveMessage.js index 4f86c36..d8dd097 100644 --- a/back/sql/saveMessage.js +++ b/back/sql/saveMessage.js @@ -19,7 +19,23 @@ function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, delete }); } -function saveBodyparts() {} +function registerBodypart(messageId, part, bodypartId, bytes, nbLines) { + const query = `INSERT IGNORE INTO part_numbers (message, part, bodypart, bytes, nbLines) VALUES ('${messageId}', '${part}', '${bodypartId}', '${bytes}', '${nbLines}')`; + bdd.query(query, (err, results, fields) => { + if (err) DEBUG.log(err); + }); +} + + +function saveBodypart(bytes, hash, text, data) { + return new Promise((resolve, reject) => { + const query = `INSERT IGNORE INTO bodyparts (bytes, hash, text, data) VALUES ('${bytes}', '${hash}', '${text}', '${data}')`; + bdd.query(query, (err, results, fields) => { + if (err) reject(err); + resolve(results.insertId); + }); + }); +} function saveHeader_fields(message, part, position, field, value) { const query = `INSERT IGNORE INTO header_fields (message, part, position, field, value) VALUES ('${message}', '${part}', '${position}', '${field}', '${value}')`; @@ -40,4 +56,6 @@ module.exports = { registerMailbox_message, saveHeader_fields, saveAddress_fields, + registerBodypart, + saveBodypart } \ No newline at end of file