back/controllers/addAccount.ts

@@ -1,6 +1,6 @@
 import statusCode from "../utils/statusCodes";
 import { registerAccount } from "../db/api";
-import { getAddresseId } from "../db/utils/mail";
+import { getAddresseId } from "../db/mail";
 
 export async function addAccount(body, res) {
     const { email, pwd, xoauth, xoauth2, host, port, tls } = body;

back/db/database.dart

@@ -0,0 +1,140 @@
+Table "addresses" {
+  "id" int [pk, not null, increment]
+  "name" text
+  "localpart" text [not null]
+  "domain" text [not null]
+  "email" text [not null]
+
+  Indexes {
+    email [unique]
+  }
+}
+
+Table "mailboxes" {
+  "id" int [pk, not null, increment]
+  "name" text [not null]
+  "uidnext" int [not null, default: 1]
+  "nextmodseq" bigint [not null, default: 1]
+  "first_recent" int [not null, default: 1]
+  "uidvalidity" int [not null, default: 1]
+
+  Indexes {
+    name [unique]
+  }
+}
+
+Table "messages" {
+  "id" int [pk, not null, increment]
+  "messageID" text [pk, not null]
+  "idate" timestamp [not null]
+  "rfc822size" int
+}
+
+Table "mailbox_messages" {
+  "mailbox" int [not null]
+  "uid" int [pk, not null]
+  "message" int [pk, not null]
+  "modseq" bigint [not null]
+  "seen" boolean [not null, default: false]
+  "deleted" boolean [not null, default: false]
+}
+
+Ref: mailbox_messages.mailbox > mailboxes.id
+Ref: mailbox_messages.message - messages.id
+
+Table "bodyparts" {
+  "id" int [pk, not null, increment]
+  "bytes" int [not null]
+  "hash" text [not null]
+  "text" text
+  "data" binary
+}
+
+Table "part_numbers" {
+  "message" int [pk, not null]
+  "part" text [not null]
+  "bodypart" int [not null]
+  "bytes" int
+  "nb_lines" int
+}
+
+// todo on delete cascade
+Ref: part_numbers.message > messages.id
+Ref: part_numbers.bodypart - bodyparts.id
+
+Table "field_names" {
+  "id" int [pk, not null, increment]
+  "name" text [not null]
+
+  Indexes {
+    name [unique]
+  }
+}
+
+Table "header_fields" {
+  "id" int [pk, not null, increment]
+  "message" int [pk, not null]
+  "part" text [not null]
+  "position" int [not null]
+  "field" int [not null]
+  "value" text
+
+  Indexes {
+    message [unique]
+    part [unique]
+    position [unique]
+    field [unique]
+  }
+}
+
+Ref: header_fields.message > messages.id
+Ref: header_fields.part > part_numbers.part
+Ref: header_fields.field > field_names.id
+
+Table "address_fields" {
+  "message" int [not null]
+  "part" text [not null]
+  "position" int [not null]
+  "field" int [not null]
+  "number" int
+  "address" int [not null]
+}
+
+Ref: address_fields.message > messages.id
+Ref: address_fields.part > part_numbers.part
+Ref: address_fields.field > field_names.id
+Ref: address_fields.address > addresses.id
+
+// app table
+Table "front_threads" {
+  "id" int [pk, not null, increment]
+  "room" int [not null]
+  "name" text
+  "notSeen" int [not null, default: true]
+  "lastUpdate" timestamp [not null]
+  "isDm" bool [not null, default: true]
+}
+
+Ref: front_threads.room > front_rooms.id
+
+Table "front_rooms" {
+  "id" int [pk, not null, increment]
+  "name" text
+  "isGroup" bool [not null, default: false]
+  "notSeen" int [not null]
+  "lastUpdate" timestamp [not null]
+}
+
+Table "front_room_messages" {
+  "room" int [not null]
+  "thread" int [not null]
+  "message" int [not null]
+}
+
+Ref: front_room_messages.room > front_rooms.id
+Ref: front_room_messages.message - messages.id
+Ref: front_room_messages.thread > front_threads.id
+
+
+
+

back/db/database.sql

@@ -0,0 +1,108 @@
+CREATE TABLE `addresses` (
+  `id` int PRIMARY KEY NOT NULL AUTO_INCREMENT,
+  `name` text,
+  `localpart` text NOT NULL,
+  `domain` text NOT NULL,
+  `email` text NOT NULL
+);
+
+CREATE TABLE `mailboxes` (
+  `id` int PRIMARY KEY NOT NULL AUTO_INCREMENT,
+  `name` text NOT NULL,
+  `uidnext` int NOT NULL DEFAULT 1,
+  `nextmodseq` bigint NOT NULL DEFAULT 1,
+  `first_recent` int NOT NULL DEFAULT 1,
+  `uidvalidity` int NOT NULL DEFAULT 1
+);
+
+CREATE TABLE `messages` (
+  `id` int PRIMARY KEY NOT NULL AUTO_INCREMENT,
+  `idate` timestamp NOT NULL,
+  `rfc822size` int
+);
+
+CREATE TABLE `mailbox_messages` (
+  `mailbox` int NOT NULL,
+  `uid` int NOT NULL,
+  `message` int NOT NULL,
+  `modseq` bigint NOT NULL,
+  `seen` boolean NOT NULL DEFAULT false,
+  `deleted` boolean NOT NULL DEFAULT false,
+  PRIMARY KEY (`uid`, `message`)
+);
+
+CREATE TABLE `bodyparts` (
+  `id` int PRIMARY KEY NOT NULL AUTO_INCREMENT,
+  `bytes` int NOT NULL,
+  `hash` text NOT NULL,
+  `text` text,
+  `data` binary
+);
+
+CREATE TABLE `part_numbers` (
+  `message` int PRIMARY KEY NOT NULL,
+  `part` text NOT NULL,
+  `bodypart` int NOT NULL,
+  `bytes` int,
+  `nb_lines` int
+);
+
+CREATE TABLE `field_names` (
+  `id` int PRIMARY KEY NOT NULL AUTO_INCREMENT,
+  `name` text NOT NULL
+);
+
+CREATE TABLE `header_fields` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `message` int NOT NULL,
+  `part` text NOT NULL,
+  `position` int NOT NULL,
+  `field` int NOT NULL,
+  `value` text,
+  PRIMARY KEY (`id`, `message`)
+);
+
+CREATE TABLE `address_fields` (
+  `message` int NOT NULL,
+  `part` text NOT NULL,
+  `position` int NOT NULL,
+  `field` int NOT NULL,
+  `number` int,
+  `address` int NOT NULL
+);
+
+CREATE UNIQUE INDEX `addresses_index_0` ON `addresses` (`email`);
+
+CREATE UNIQUE INDEX `mailboxes_index_1` ON `mailboxes` (`name`);
+
+CREATE UNIQUE INDEX `field_names_index_2` ON `field_names` (`name`);
+
+CREATE UNIQUE INDEX `header_fields_index_3` ON `header_fields` (`message`);
+
+CREATE UNIQUE INDEX `header_fields_index_4` ON `header_fields` (`part`);
+
+CREATE UNIQUE INDEX `header_fields_index_5` ON `header_fields` (`position`);
+
+CREATE UNIQUE INDEX `header_fields_index_6` ON `header_fields` (`field`);
+
+ALTER TABLE `mailbox_messages` ADD FOREIGN KEY (`mailbox`) REFERENCES `mailboxes` (`id`);
+
+ALTER TABLE `messages` ADD FOREIGN KEY (`id`) REFERENCES `mailbox_messages` (`message`);
+
+ALTER TABLE `part_numbers` ADD FOREIGN KEY (`message`) REFERENCES `messages` (`id`);
+
+ALTER TABLE `bodyparts` ADD FOREIGN KEY (`id`) REFERENCES `part_numbers` (`bodypart`);
+
+ALTER TABLE `header_fields` ADD FOREIGN KEY (`message`) REFERENCES `messages` (`id`);
+
+ALTER TABLE `header_fields` ADD FOREIGN KEY (`part`) REFERENCES `part_numbers` (`part`);
+
+ALTER TABLE `header_fields` ADD FOREIGN KEY (`field`) REFERENCES `field_names` (`id`);
+
+ALTER TABLE `address_fields` ADD FOREIGN KEY (`message`) REFERENCES `messages` (`id`);
+
+ALTER TABLE `address_fields` ADD FOREIGN KEY (`part`) REFERENCES `part_numbers` (`part`);
+
+ALTER TABLE `address_fields` ADD FOREIGN KEY (`field`) REFERENCES `field_names` (`id`);
+
+ALTER TABLE `address_fields` ADD FOREIGN KEY (`address`) REFERENCES `addresses` (`id`);

back/db/db.ts

@@ -2,11 +2,12 @@ import mysql from "mysql";
 import logger from "../system/Logger";
 require("dotenv").config();
 
-const db = mysql.createConnection({
+// todo remove export
+export const db = mysql.createConnection({
     host: process.env.HOST_DB,
     user: process.env.USER_DB,
     password: process.env.PASSWORD_DB,
-    database: (process.env.NODE_ENV === "test") ? process.env.NAME_DB_TEST : process.env.NAME_DB,
+    database: process.env.NAME_DB,
 });
 
 db.connect(function (err) {

back/db/mail.ts

@@ -1,4 +1,4 @@
-import { execQueryAsync, execQueryAsyncWithId } from "../db";
+import { execQueryAsync, execQueryAsyncWithId } from "./db";
 
 export async function getAddresseId(email: string, name?: string): Promise<number> {
     const localpart = email.split("@")[0];

back/db/saveMessage.ts

@@ -1,5 +1,5 @@
-import { transformEmojis } from "../../utils/string";
+import { transformEmojis } from "../utils/string";
-import { execQuery, execQueryAsync, execQueryAsyncWithId } from "../db";
+import { execQuery, execQueryAsync, execQueryAsyncWithId } from "./db";
 
 export async function registerMessage(timestamp, rfc822size, messageId) {
     const query = `

back/db/saveMessageApp.ts

@@ -1,5 +1,5 @@
 import { transformEmojis } from "../utils/string";
-import { execQueryAsync, execQueryAsyncWithId, execQuery } from "./db";
+import { db, execQueryAsync, execQueryAsyncWithId, execQuery } from "./db";
 import { queryFromId, queryToId, queryCcId } from "./utils/addressQueries";
 
 export async function getAllMembers(messageId: number) {
@@ -85,10 +85,11 @@ export async function registerThread(roomId: number, parentId: number, rootId: n
 
 export async function isRoomGroup(roomId: number): Promise<boolean> {
     return new Promise((resolve, reject) => {
-        const query = `SELECT isGroup FROM app_room WHERE room_id = ?`;
+        const query = `SELECT isGroup FROM app_room WHERE room_id = '${roomId}'`;
-        const values = [roomId];
+        db.query(query, (err, results, fields) => {
-        const results = execQueryAsync(query, values);
+            if (err) reject(err);
-        return results[0].isGroup;
+            resolve(results[0].isGroup);
+        });
     });
 }
 

back/db/structure

@@ -0,0 +1,157 @@
+create table addresses (
+    id          int not null auto_increment,
+    name        text,
+    localpart   text not null,
+    domain      text not null,
+    email       text not null,
+    primary key (id),
+    unique key (email)
+);
+
+create table mailboxes (
+    -- Grant: select, insert, update
+    id          serial primary key,
+    name        text not null unique,
+    -- owner       int references users(id), todo
+
+    -- The UID that will be assigned to the next delivered message.
+    -- Incremented after each successful delivery.
+    uidnext     int not null default 1,
+
+    -- The next modsequence value for this mailbox.
+    nextmodseq  bigint not null default 1,
+
+    -- The UID of the first message that should be marked \Recent.
+    -- Set to uidnext when each new IMAP session is created.
+    first_recent int not null default 1,
+
+    -- The IMAP mailbox UIDVALIDITY value, which, along with a message UID,
+    -- is forever guaranteed to uniquely refer to a single message.
+    uidvalidity int not null default 1,
+
+    -- When a mailbox is deleted, its entry is marked (not removed), so
+    -- that its UIDVALIDITY can be incremented if it is ever re-created.
+    deleted     boolean not null default false
+);
+
+/**
+ * Store message
+ */
+
+create table messages (
+    -- Grant: select, insert
+    id          int primary key auto_increment,
+    -- the time the message was added to the mailbox, as a unix time_t
+    idate       timestamp not null,
+    -- the size of the message in RFC 822 format, in bytes
+    rfc822size  int
+);
+
+-- todo make this work
+--     foreign key (mailbox) references mailboxes(id),
+--    foreign key (uid) references messages(id),
+create table mailbox_messages (
+    -- Grant: select, insert, update
+    mailbox     int not null,
+    -- the message's number in the mailbox
+    uid         int not null,
+    message     int not null,
+    modseq      bigint not null,
+    seen        boolean not null default false,
+    deleted     boolean not null default false,
+    primary key (mailbox, uid)
+);
+
+-- tood should not be auto increment but nextval('bodypart_ids')
+create table bodyparts (
+    -- Grant: select, insert
+    id          int primary key auto_increment,
+    -- the size of either text or data, whichever is used
+    bytes       int not null,
+    -- MD5 hash of either text or data
+    hash        text not null,
+    text        text,
+    data        binary
+);
+
+create table part_numbers (
+    -- Grant: select, insert
+    message     int references messages(id) on delete cascade,
+    part        text not null,
+    bodypart    int references bodyparts(id),
+    bytes       int,
+    nbLines       int,
+    primary key (message)
+);
+
+create table field_names (
+    -- Grant: select, insert
+    id          serial primary key,
+    name        text unique
+);
+
+-- add     foreign key (message, part)
+               -- references part_numbers(message, part)
+                --on delete cascade at the end
+--  references field_names(id) to field
+create table header_fields (
+    -- Grant: select, insert
+    id          serial primary key,
+    message     int not null,
+    part        text not null,
+    position    int not null,
+    field       int not null,
+    value       text,
+    unique (message, part, position, field)
+);
+
+create table address_fields (
+    -- Grant: select, insert
+    message     int not null,
+    part        text not null,
+    position    int not null,
+    field       int not null,
+    number      int,
+    address     int not null-- references addresses(id),
+--    foreign key (message, part)
+              --  references part_numbers(message, part)
+               -- on delete cascade
+);
+
+create table date_fields (
+    -- Grant: select, insert
+    message     int not null references messages(id)
+                on delete cascade,
+    value       timestamp --with time zone
+);
+
+/**
+ * APP tables
+ */
+create table app_accounts (
+    id          int not null auto_increment,
+    user        varchar(255) not null,
+    password    binary(20),
+    xoauth      varchar(116),
+    xoauth2     varchar(116),
+    host        varchar(255) not null default 'localhost',
+    port        int(5) not null default 143,
+    tls         int(1) not null default 0,
+    primary key (id)
+);
+
+create table app_rooms (
+    id          int not null auto_increment,
+    name        text not null,
+    owner       int not null,
+    isGroup     BIT(1) not null default 0,
+    notSeen     int not null default 0,
+    lastUpdate  timestamp not null,
+    primary key (id)
+);
+
+create table app_room_messages (
+    message     int [not null]
+    room        int,
+    primary key()
+)

back/mails/imap/Box.ts

@@ -1,5 +1,5 @@
 import Imap, { ImapMessageAttributes, MailBoxes } from "imap";
-import { getMailbox, updateMailbox } from "../../db/imap/imap-db";
+import { getMailbox, updateMailbox } from "../../db/imap/imap";
 import { Attrs, AttrsWithEnvelope } from "../../interfaces/mail/attrs.interface";
 import logger from "../../system/Logger";
 import RegisterMessageInApp from "../saveMessage";

back/mails/imap/ImapInstance.ts

@@ -1,7 +1,7 @@
 import { Account } from "./ImapSync";
 
 import Imap from "imap";
-import { getAllMailboxes, registerMailbox } from "../../db/imap/imap-db";
+import { getAllMailboxes, registerMailbox } from "../../db/imap/imap";
 import logger from "../../system/Logger";
 import Box from "./Box";
 

back/mails/imap/ImapSync.ts

@@ -1,4 +1,4 @@
-import { getAllAccounts } from "../../db/imap/imap-db";
+import { getAllAccounts } from "../../db/imap/imap";
 import logger from "../../system/Logger";
 import { ImapInstance } from "./ImapInstance";
 

back/mails/saveMessage.ts

@@ -8,11 +8,12 @@ import {
     registerMember,
     getAllMembers,
     getRoomInfo,
-} from "../db/saveMessage-db";
+} from "../db/saveMessageApp";
 
-import { findRoomByOwner, getAddresseId, getUserIdOfMailbox } from "../db/utils/mail";
+import { findRoomByOwner, getAddresseId, getUserIdOfMailbox } from "../db/mail";
 import { nbMembers } from "./utils/envelopeUtils";
 import logger from "../system/Logger";
+import { ImapMessageAttributes } from "imap";
 import { Attrs, Envelope, User } from "../interfaces/mail/attrs.interface";
 
 /**

back/mails/storeMessage.ts

@@ -1,4 +1,4 @@
-import { getAddresseId } from "../db/utils/mail";
+import { getAddresseId } from "../db/mail";
 import {simpleParser} from "mailparser";
 import moment from "moment";
 import Imap from "imap";
@@ -10,9 +10,9 @@ import {
     registerBodypart,
     saveBodypart,
     saveSource,
-} from "../db/imap/storeMessage-db";
+} from "../db/saveMessage";
 
-import { getFieldId } from "../db/utils/mail";
+import { getFieldId } from "../db/mail";
 import logger from "../system/Logger";
 import { AttrsWithEnvelope } from "../interfaces/mail/attrs.interface";
 

back/package.json

@@ -29,21 +29,15 @@
     "@types/node": "^18.15.11",
     "concurrently": "^8.0.1",
     "jest": "^29.5.0",
-    "sql-mysql": "^1.2.0",
-    "sqlite3": "^5.1.6",
-    "sqlparser": "^0.1.7",
     "ts-jest": "^29.0.5",
     "ts-node": "^10.9.1",
     "typescript": "^4.9.5"
   },
   "jest": {
     "preset": "ts-jest",
-    "setupFiles": [
-      "<rootDir>/test/.jest/setEnvVars.js"
-    ],
     "testEnvironment": "node",
     "testMatch": [
       "<rootDir>/test//**/*-test.[jt]s?(x)"
     ]
   }
-}
+}

back/test/.jest/setEnvVars.js

@@ -1 +0,0 @@
-process.env.NODE_ENV = "test";

back/test/mail/saveMessage-test.ts

@@ -1,18 +1,11 @@
-import mysql from "mysql";
-
-jest.mock("mysql");
-mysql.createConnection = jest.fn();
-mysql.createConnection.mockImplementation(() => {
-    return { connect: () => new Promise((resolve, rejects) => resolve(true)) };
-});
-
 import { generateAttrs, generateUsers } from "../test-utils/test-attrsUtils";
 import registerMessageInApp, { roomType } from "../../mails/saveMessage";
 import { jest, describe, it, expect } from "@jest/globals";
 
-import { getAddresseId, getUserIdOfMailbox } from "../../db/utils/mail";
+import { getAddresseId, getUserIdOfMailbox } from "../../db/mail";
 // todo esbuild
 // todo mock db
+
 // new message from us
 //     to multiple people -> room
 //         if response has same member => group
@@ -27,12 +20,13 @@ import { getAddresseId, getUserIdOfMailbox } from "../../db/utils/mail";
 //         // make it better
 //         if multiple members reply -> group
 //         if only me reply -> channel
+
 const users = generateUsers(5);
 const ownUser = users[0];
 const messageId = 1;
 const boxId = 1;
 
-jest.mock("../../db/utils/mail", () => ({
+jest.mock("../../db/mail", () => ({
     getAddresseId: jest.fn().mockImplementation((email) => {
         const match = users.find((user) => user.user.mailbox + "@" + user.user.host == email);
         return new Promise((resolve, reject) => resolve(match?.id));
@@ -42,10 +36,6 @@ jest.mock("../../db/utils/mail", () => ({
     }),
 }));
 
-beforeAll(async () => {
-
-});
-
 describe("saveMessage", () => {
     describe("functions", () => {
         it("isFromUs", async () => {

back/test/test-utils/db/test-db.ts

@@ -1,35 +0,0 @@
-const sqlite3 = require("sqlite3").verbose();
-const sqlMysql = require("sql-mysql");
-export const db = new sqlite3.Database(":memory:");
-var Parser = require("sqlparser");
-import fs from "fs";
-// const mysqlQuery = 'SELECT * FROM users WHERE age > 18';
-// const sqliteQuery = sqlMysql.toSqlite(mysqlQuery);
-// console.log(sqliteQuery)
-export function initDatabase() {
-    return new Promise((resolve, reject) => {
-        const pathSQL = __dirname + "/../../../db/structureV2.sql";
-        // fs.readdir(__dirname + "/../../../db/structureV2.sql", (err, files) => {
-        //     if (err) console.log(err);
-        //     else {
-        //         console.log("\nCurrent directory filenames:");
-        //         files.forEach((file) => {
-        //             console.log(file);
-        //         });
-        //     }
-        // });
-        const sqlTables = fs.readFileSync(pathSQL, "utf8");
-        // const sqliteTables = sqlMysql.toSqlite(sqlTables);
-        var sqliteTables = Parser.parse(sqlTables);
-
-        db.serialize(() => {
-            db.run(sqliteTables, (error) => {
-                if (error) {
-                    reject(error);
-                } else {
-                    resolve(true);
-                }
-            });
-        });
-    });
-}