grimhilt/mail
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

start to save mail from imap

Browse Source
This commit is contained in:
grimhilt 2023-02-26 17:23:52 +01:00
parent 0692f8caa5
commit b0a0fe2f83
11 changed files with 1895 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
back/api/mails.js Normal file
View File

@ -0,0 +1,13 @@
const statusCodes = require("../utils/statusCodes.js").statusCodes;
/**
* Return all mailboxes and folders for an user
*/
function getMailboxes(req, res) {
const {token} = req.params;
const query = ``;
}
module.exports = {
getFolders,
}

162
back/imap/index.js Normal file
View File

@ -0,0 +1,162 @@
const Imap = require('imap');
const {simpleParser} = require('mailparser');
const inspect = require('util').inspect;
const saveMessage = require('./storeMessage').saveMessage;
const imap = new Imap({
user: '***REMOVED***',
password: '***REMOVED***',
tlsOptions: {servername: "imap.gmail.com"},
host: 'imap.gmail.com',
port: 993,
tls: true
});
imap.once('ready', function() {
const readOnly = true;
imap.openBox('INBOX', readOnly, (err, box) => {
// console.log(box); // uidvalidty uidnext, messages total and new
if (err) throw err;
const f = imap.seq.fetch('2:2', {
bodies: ['HEADER.FIELDS (FROM)','TEXT'],
struct: true,
envelope: true,
extensions: true
});
f.on('message', function(msg, seqno) {
// console.log('Message #%d', seqno);
var prefix = '(#' + seqno + ') ';
let attributes = undefined;
let body = undefined;
msg.on('body', function(stream, info) {
simpleParser(stream, async (err, parsed) => {
body = parsed;
if (attributes) {
saveMessage(body, attributes);
};
// console.log(parsed.headers)
// const {from, subject, textAsHtml, text} = parsed;
// console.log(parsed.attachments)
// console.log(prefix + parsed.text)
// console.log(parsed.from.value);
// console.log(parsed.subject);
// console.log(parsed.date)
// console.log(parsed.replyTo.value);
// console.log(parsed.messageId);
// console.log(parsed.html);
// console.log(parsed.text);
// console.log(parsed.textAsHtml);
});
});
msg.once('attributes', attrs => {
attributes = attrs;
if (body) {
saveMessage(body, attributes);
};
// console.log(prefix + 'Attributes: %s', inspect(attrs, false, 8));
});
msg.once('end', function() {
console.log(prefix + 'Finished');
});
});
f.once('error', function(err) {
console.log('Fetch error: ' + err);
});
f.once('end', function() {
console.log('Done fetching all messages!');
imap.end();
});
});
});
imap.once('error', function(err) {
console.log(err);
});
imap.once('end', function() {
console.log('Connection ended');
});
imap.connect();
// const getEmails = () => {
// imap.once('ready', () => {
// imap.openBox('INBOX', false, () => {
// imap.search(['UNSEEN'], (err, results) => {
// const f = imap.fetch(results, {bodies: ''});
// f.on('message', msg => {
// msg.on('body', stream => {
// simpleParser(stream, async (err, parsed) => {
// // const {from, subject, textAsHtml, text} = parsed;
// // console.log(parsed.from.value);
// // console.log(parsed.subject);
// // console.log(parsed.date)
// // console.log(parsed.replyTo.value);
// // console.log(parsed.messageId);
// // console.log(parsed.html);
// // console.log(parsed.text);
// // console.log(parsed.textAsHtml);
// // 'x-emsg-mtaselection' => 'prod_5_emailing.carrefour.fr',
// // 'message-id' => '<emsg.6584.7d09.15dd1ec64c1@ukmme02.em.unica.net>',
// // 'feedback-id' => 'emailing.carrefour.fr:10070-6584:emsg-x',
// // 'list' => { unsubscribe: [Object] },
// // 'mime-version' => '1.0',
// // 'content-type' => { value: 'text/html', params: [Object] },
// // 'content-transfer-encoding' => 'quoted-printable'
// });
// });
// msg.once('attributes', attrs => {
// const {uid} = attrs;
// console.log(uid)
// // imap.addFlags(uid, ['\\Seen'], () => {
// // // Mark the email as read after reading it
// // console.log('Marked as read!');
// // });
// });
// });
// f.once('error', ex => {
// return Promise.reject(ex);
// });
// f.once('end', () => {
// console.log('Done fetching all messages!');
// imap.end();
// });
// });
// });
// });
// imap.once('error', err => {
// console.log(err);
// });
// imap.once('end', () => {
// console.log('Connection ended');
// });
// imap.connect();
// };
// getEmails();
function isValidEmail(email) {
// todo
return true;
}
module.exports = {
isValidEmail
}

72
back/imap/storeMessage.js Normal file
View File

@ -0,0 +1,72 @@
const { getAddresseId } = require("../sql/mail");
const { DEBUG } = require("../utils/debug");
const { registerMessage, registerMailbox_message, saveHeader_fields, saveAddress_fields } = require('../sql/saveMessage');
const { getMailboxId, getField } = require('../sql/mail');
function saveMessage(message, attributes, mailbox) {
const timestamp = new Date(attributes.envelope.date).getTime();
const rfc822size = 0; // todo
registerMessage(timestamp, rfc822size).then((messageId) => {
getMailboxId(mailbox).then((mailboxId) => {
const seen = attributes.flags.includes('Seen') ? 1 : 0; // todo verify
const deleted = attributes.flags.includes('Deleted') ? 1 : 0; // todo verify
registerMailbox_message(mailboxId, attributes.uid, messageId, attributes.modseq, seen, deleted).then(() => {
attributes.struct.forEach(part => {
// saveBodyparts().then((bodypartId) => {
// const partText = undefined;
// savePart_numbers(messageId, partText, bodypartId, part.size, part.lines)
// });
});
const part = ''; // todo
Object.keys(attributes.envelope).forEach(key => {
const newKey = keyNormalizer(key);
if (isHeader(newKey)) {
getField(newKey).then((fieldId) => {
saveHeader_fields(messageId, part, 2, fieldId, attributes.envelope[key]);
});
} else {
getField(newKey).then((fieldId) => {
if (attributes.envelope[key]) {
attributes.envelope[key].forEach((elt, index) => {
saveAddress_fields(messageId, part, fieldId, index, getAddresseId(`${elt.mailbox}@${elt.host}`, elt.name));
});
}
});
}
});
// todo add date field
});
});
});
}
function isHeader(key) {
switch (key) {
case 'date':
case 'subject':
case 'messageId':
return true;
case 'from':
case 'sender':
case 'replyTo':
case 'to':
case 'cc':
case 'bcc':
case 'inReplyTo':
return false;
default:
DEBUG.log("Unknown header key: "+key);
return true;
}
}
function keyNormalizer(key) {
// todo
return key;
}
module.exports = {
saveMessage
}

1334
back/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

9
back/package.json Normal file
View File

@ -0,0 +1,9 @@
{
"dependencies": {
"imap": "^0.8.19",
"imap-simple": "^5.1.0",
"mailparser": "^3.6.3",
"mysql": "^2.18.1",
"vue-router": "^4.1.6"
}
}

20
back/server.js Normal file
View File

@ -0,0 +1,20 @@
const mails = require("./api/mails.js");
const express = require('express');
const cors = require('cors')
const app = express();
app.use(express.json());
app.use(
express.urlencoded({
extended: true,
})
);
app.use(cors());
app.listen(process.env.PORT || 5500);
// anecdote
app.get("/api/mails/mailboxes", mails.getMailboxes);

23
back/sql/bdd.js Normal file
View File

@ -0,0 +1,23 @@
const mysql = require("mysql");
const MYSQL = require("./config.json").mysql;
const DEBUG = require("../utils/debug.js").DEBUG;
const bdd = mysql.createConnection({
host: MYSQL.host,
user: MYSQL.user,
password: MYSQL.pwd,
database: MYSQL.database,
});
bdd.connect(function (err) {
if (err) {
DEBUG.log("Impossible de se connecter", err.code);
} else {
DEBUG.log("Database successfully connected");
}
});
module.exports = {
bdd: bdd,
};

48
back/sql/saveMessage.js Normal file
View File

@ -0,0 +1,48 @@
const bdd = require("./bdd.js").bdd;
const DEBUG = require("../utils/debug").DEBUG;
function registerMessage(timestamp, rfc822size) {
return new Promise((resolve, reject) => {
resolve(0);
//todo
const query = `INSERT INTO messages (idate, rfc822size) VALUES (UNIX_TIMESTAMP('${timestamp}'), '${rfc822size}')`;
bdd.query(query, (err, results, fields) => {
if (err) reject(err);
// resolve(results.insertId);
});
});
}
function registerMailbox_message(mailboxId, uid, messageId, modseq, seen, deleted) {
return new Promise((resolve, reject) => {
const query = `INSERT IGNORE INTO mailbox_messages (mailbox, uid, message, modseq, seen, deleted) VALUES ('${mailboxId}', '${uid}', '${messageId}', '${modseq}', '${seen}', '${deleted}')`;
bdd.query(query, (err, results, fields) => {
if (err) reject(err);
resolve();
// resolve(results.insertId);
});
});
}
function saveBodyparts() {}
function saveHeader_fields(message, part, position, field, value) {
const query = `INSERT IGNORE INTO header_fields (message, part, position, field, value) VALUES ('${message}', '${part}', '${position}', '${field}', '${value}')`;
bdd.query(query, (err, results, fields) => {
if (err) throw err;
});
}
function saveAddress_fields(message, part, position, field, number, address) {
const query = `INSERT IGNORE INTO address_fields (message, part, position, field, number, address) VALUES ('${message}', '${part}', '${position}', '${field}', '${number}', '${address}')`;
bdd.query(query, (err, results, fields) => {
if (err) throw err;
});
}
module.exports = {
registerMessage,
registerMailbox_message,
saveHeader_fields,
saveAddress_fields,
}

141
back/sql/structure Normal file
View File

@ -0,0 +1,141 @@
create table addresses (
id int not null auto_increment,
name text,
localpart text not null,
domain text not null,
email text not null,
primary key (id),
unique key (email)
);
create table mailboxes (
-- Grant: select, insert, update
id serial primary key,
name text not null unique,
-- owner int references users(id), todo
-- The UID that will be assigned to the next delivered message.
-- Incremented after each successful delivery.
uidnext int not null default 1,
-- The next modsequence value for this mailbox.
nextmodseq bigint not null default 1,
-- The UID of the first message that should be marked \Recent.
-- Set to uidnext when each new IMAP session is created.
first_recent int not null default 1,
-- The IMAP mailbox UIDVALIDITY value, which, along with a message UID,
-- is forever guaranteed to uniquely refer to a single message.
uidvalidity int not null default 1,
-- When a mailbox is deleted, its entry is marked (not removed), so
-- that its UIDVALIDITY can be incremented if it is ever re-created.
deleted boolean not null default false
);
/**
* Store message
*/
create table messages (
-- Grant: select, insert
id int primary key auto_increment,
-- the time the message was added to the mailbox, as a unix time_t
idate timestamp not null,
-- the size of the message in RFC 822 format, in bytes
rfc822size int
);
-- todo make this work
-- foreign key (mailbox) references mailboxes(id),
-- foreign key (uid) references messages(id),
create table mailbox_messages (
-- Grant: select, insert, update
mailbox int not null,
-- the message's number in the mailbox
uid int not null,
message int not null,
modseq bigint not null,
seen boolean not null default false,
deleted boolean not null default false,
primary key (mailbox, uid)
);
-- tood should not be auto increment but nextval('bodypart_ids')
create table bodyparts (
-- Grant: select, insert
id int primary key auto_increment,
-- the size of either text or data, whichever is used
bytes int not null,
-- MD5 hash of either text or data
hash text not null,
text text,
data binary
);
create table part_numbers (
-- Grant: select, insert
message int references messages(id) on delete cascade,
part text not null,
bodypart int references bodyparts(id),
bytes int,
nbLines int,
primary key (message)
);
create table field_names (
-- Grant: select, insert
id serial primary key,
name text unique
);
-- add foreign key (message, part)
-- references part_numbers(message, part)
--on delete cascade at the end
-- references field_names(id) to field
create table header_fields (
-- Grant: select, insert
id serial primary key,
message int not null,
part text not null,
position int not null,
field int not null,
value text,
unique (message, part, position, field)
);
create table address_fields (
-- Grant: select, insert
message int not null,
part text not null,
position int not null,
field int not null,
number int,
address int not null-- references addresses(id),
-- foreign key (message, part)
-- references part_numbers(message, part)
-- on delete cascade
);
create table date_fields (
-- Grant: select, insert
message int not null references messages(id)
on delete cascade,
value timestamp --with time zone
);
/**
* APP tables
*/
create table app_accounts (
id int not null auto_increment,
user varchar(255) not null,
password binary(20),
xoauth varchar(116),
xoauth2 varchar(116),
host varchar(255) not null default 'localhost',
port int(5) not null default 143,
tls int(1) not null default 0,
primary key (id)
);

9
back/utils/debug.js Normal file
View File

@ -0,0 +1,9 @@
const DEBUG = (function() {
const timestamp = function(){};
timestamp.toString = () => "[DEBUG " + new Date().toLocaleDateString() + "]";
return { log: console.log.bind(console, "%s", timestamp) };
})();
module.exports = {
DEBUG: DEBUG
}

64
back/utils/statusCodes.js Normal file
View File

@ -0,0 +1,64 @@
// from https://github.com/prettymuchbryce/http-status-codes/blob/master/src/utils.ts
const statusCodes = {
"CONTINUE": 100,
"SWITCHING_PROTOCOLS": 101,
"PROCESSING": 102,
"OK": 200,
"CREATED": 201,
"ACCEPTED": 202,
"NON_AUTHORITATIVE_INFORMATION": 203,
"NO_CONTENT": 204,
"RESET_CONTENT": 205,
"PARTIAL_CONTENT": 206,
"MULTI_STATUS": 207,
"MULTIPLE_CHOICES": 300,
"MOVED_PERMANENTLY": 301,
"MOVED_TEMPORARILY": 302,
"SEE_OTHER": 303,
"NOT_MODIFIED": 304,
"USE_PROXY": 305,
"TEMPORARY_REDIRECT": 307,
"PERMANENT_REDIRECT": 308,
"BAD_REQUEST": 400,
"UNAUTHORIZED": 401,
"PAYMENT_REQUIRED": 402,
"FORBIDDEN": 403,
"NOT_FOUND": 404,
"METHOD_NOT_ALLOWED": 405,
"NOT_ACCEPTABLE": 406,
"PROXY_AUTHENTICATION_REQUIRED": 407,
"REQUEST_TIMEOUT": 408,
"CONFLICT": 409,
"GONE": 410,
"LENGTH_REQUIRED": 411,
"PRECONDITION_FAILED": 412,
"REQUEST_TOO_LONG": 413,
"REQUEST_URI_TOO_LONG": 414,
"UNSUPPORTED_MEDIA_TYPE": 415,
"REQUESTED_RANGE_NOT_SATISFIABLE": 416,
"EXPECTATION_FAILED": 417,
"IM_A_TEAPOT": 418,
"INSUFFICIENT_SPACE_ON_RESOURCE": 419,
"METHOD_FAILURE": 420,
"MISDIRECTED_REQUEST": 421,
"UNPROCESSABLE_ENTITY": 422,
"LOCKED": 423,
"FAILED_DEPENDENCY": 424,
"PRECONDITION_REQUIRED": 428,
"TOO_MANY_REQUESTS": 429,
"REQUEST_HEADER_FIELDS_TOO_LARGE": 431,
"UNAVAILABLE_FOR_LEGAL_REASONS": 451,
"INTERNAL_SERVER_ERROR": 500,
"NOT_IMPLEMENTED": 501,
"BAD_GATEWAY": 502,
"SERVICE_UNAVAILABLE": 503,
"GATEWAY_TIMEOUT": 504,
"HTTP_VERSION_NOT_SUPPORTED": 505,
"INSUFFICIENT_STORAGE": 507,
"NETWORK_AUTHENTICATION_REQUIRED": 511,
}
module.exports = {
statusCodes: statusCodes
};