migrate to typescript, finish initial /accounts/add /accounts/list and /accounts/user, fix dep param, update readme, add nightly ci

.env.example

@@ -0,0 +1,2 @@
+MIGRATE_TXT="migrate.txt"
+DB_FILE_NAME="db.sqlite3"

.gitea/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: Build and Push Nightly CI Image
+
+on:
+  schedule:
+  - cron: '0 0 * * *'
+  push:
+    branches:
+    - main
+
+jobs:
+  build_and_push:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v3
+
+    - name: Log in to Gitea Package Registry
+      run: echo "${{ secrets.PACKAGE_TOKEN }}" | docker login $SERVER_URL -u $USERNAME --password-stdin
+      env:
+        SERVER_URL: ${{ secrets.SERVER_URL }}
+        USERNAME: ${{ secrets.USERNAME }}
+        PACKAGE_TOKEN: ${{ secrets.PACKAGE_TOKEN }}
+
+    - name: Build Docker Image (ci tag)
+      run: docker build -t git.pontusmail.org/librecloud/mail-connect:ci .
+
+    - name: Push Docker Image (ci tag)
+      run: docker push git.pontusmail.org/librecloud/mail-connect:ci

.gitignore

@@ -132,5 +132,8 @@ dist
 
 # extra
 .idea/
+drizzle/
 bun.lockb
+migrate.txt
+db.sqlite3
 

Dockerfile

@@ -1,7 +1,8 @@
 FROM oven/bun:latest
 WORKDIR /app
 COPY package*.json /app/
+RUN apt-get update && apt-get install -y python3 build-essential
 RUN bun install
 COPY . /app
 EXPOSE 3000
-CMD [ "bun", "run", "server.js" ]
+CMD [ "bun", "run", "src/server.ts" ]

README.md

@@ -1,5 +1,7 @@
 # mail-connect
 
+[![Build Status](https://git.pontusmail.org/librecloud/web/actions/workflows/docker.yaml/badge.svg)](https://git.pontusmail.org/librecloud/web/actions/workflows/docker.yml)
+
 API bridge for docker-mailserver
 
 *mail-connect is still in early beta*
@@ -8,17 +10,51 @@ API bridge for docker-mailserver
 
 mail-connect aims to connect your `docker-mailserver` to *anything* you can imagine, through the power of an API. Despite being used as a core component of LibreCloud, you can still implement mail-connect any way you wish!
 
-We provide an extendable API which interacts with the `setup` utility via a Docker socket. While this offers advantages, mail-connect is still slow on some functions (such as listing accounts), as it's merely executing pre-made commands and parsing the output.
+We provide an extendable API which interacts with the `setup` utility via a Docker socket. We have implemented a SQLite database with Drizzle ORM for faster polling of users, with strategic caching and updating.
 
 ## Features
 
 All features marked with an **E** are extended features, and are not a part of the original `setup` utility.
 
+## Self-Host with Docker
+
+1. **Clone the repository**
+
+   ```bash
+   git clone https://git.pontusmail.org/librecloud/mail-connect.git
+   ```
+
+2. **Initialize database**
+
+   ```bash
+   bunx drizzle-kit generate
+   bunx drizzle-kit migrate
+   ```
+
+3. **Copy/modify necessary files**
+
+   ```bash
+   touch migrate.txt # put emails (one per line) which already exist on the server which users can claim
+   cp .env.example .env # you don't need to change anything here
+   vim ratelimit.json # optional, customize to your liking
+   ```
+
+4. **Build and run the container**
+
+   ```bash
+   docker-compose up -d --build
+   ```
+
+   Pre-built images are provided in the [Packages](https://git.pontusmail.org/librecloud/mail-connect/packages) tab
+
+   Your server will now be running on port `6723` by default (change this in `docker-compose.yml`)!
+
 ### Email
 
 - [X] Create email
 - [X] List emails
-- [ ] **E** Create email from file
+- [X] **E** View individual user details
+- [X] **E** Create email from file
 - [ ] Change password
 - [ ] Delete email
 - [ ] Restrict email
@@ -54,7 +90,7 @@ All features marked with an **E** are extended features, and are not a part of t
 ### Fail2Ban
 
 - [ ] Ban IP
-- [ ] Unban IP
+- [ ] Un-ban IP
 - [ ] Ban log
 - [ ] Fail2Ban status
 
@@ -71,3 +107,6 @@ I plan to implement a *much* more powerful API, when everything else has settled
 Since `docker-mailserver` is built on Dovecot and Postfix, I am confident we can improve this API to be speedy and efficient as ever.
 
 ## To-Do
+
+- [ ] Implement aforementioned features
+- [ ] Swagger support

docker-compose.yml

@@ -2,7 +2,9 @@ services:
   mail-connect:
     build: .
     container_name: mail-connect
+    env_file: ".env"
     ports:
       - "6723:3000"
     volumes:
-      - '/var/run/docker.sock:/var/run/docker.sock'
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - ".:/app"

drizzle.config.ts

@@ -0,0 +1,11 @@
+import 'dotenv/config';
+import { defineConfig } from 'drizzle-kit';
+
+export default defineConfig({
+  out: './drizzle',
+  schema: './src/db/schema.ts',
+  dialect: 'sqlite',
+  dbCredentials: {
+    url: process.env.DB_FILE_NAME!,
+  },
+});

package.json

@@ -1,22 +1,34 @@
 {
   "name": "mail-connect",
-  "module": "server.js",
+  "module": "src/server.ts",
   "type": "module",
   "devDependencies": {
-    "@types/bun": "latest"
+    "@types/bun": "latest",
+    "@types/dockerode": "^3.3.34",
+    "@types/express": "^5.0.0",
+    "@types/node": "^22.13.4",
+    "@types/validator": "^13.12.2",
+    "drizzle-kit": "^0.30.4"
   },
   "peerDependencies": {
-    "typescript": "^5.0.0"
+    "typescript": "^5.7.3"
   },
   "dependencies": {
+    "@types/better-sqlite3": "^7.6.12",
+    "better-sqlite3": "^11.8.1",
     "child_process": "^1.0.2",
     "dockerode": "^4.0.4",
+    "dotenv": "^16.4.7",
+    "drizzle-orm": "^0.39.3",
     "express": "^4.21.2",
     "express-rate-limit": "^7.5.0",
     "password-validator": "^5.3.0",
     "validator": "^13.12.0"
   },
   "trustedDependencies": [
-    "protobufjs"
+    "better-sqlite3",
+    "cpu-features",
+    "protobufjs",
+    "ssh2"
   ]
 }

ratelimit.json

@@ -1,10 +1,10 @@
 {
   "/list": {
     "windowMs": 60000,
-    "max": 20
+    "limit": 20
   },
   "/add": {
     "windowMs": 60000,
-    "max": 10
+    "limit": 10
   }
 }

server.js

@@ -1,153 +0,0 @@
-const express = require('express');
-const fs = require('fs');
-const rateLimit = require('express-rate-limit');
-const Docker = require('dockerode');
-const validator = require('validator');
-
-const docker = new Docker({ socketPath: '/var/run/docker.sock' });
-const app = express();
-app.use(express.json());
-
-// Rate limiting
-
-let rateLimitConfig = {};
-try {
-  const data = fs.readFileSync('./ratelimit.json', 'utf8');
-  rateLimitConfig = JSON.parse(data);
-} catch (err) {
-  console.error('Error loading rate limit config:', err);
-}
-
-function createLimiter(options) {
-  return rateLimit({
-    windowMs: options.windowMs,
-    max: options.max,
-    message: "Too many requests, please try again another time.",
-  });
-}
-
-if (rateLimitConfig['/list']) {
-  app.use('/list', createLimiter(rateLimitConfig['/list']));
-}
-if (rateLimitConfig['/add']) {
-  app.use('/add', createLimiter(rateLimitConfig['/add']));
-}
-
-// Utility fxns
-
-function validateEmail(email) {
-  return typeof email === 'string' && validator.isEmail(email);
-}
-
-const PasswordValidator = require('password-validator');
-const passwordSchema = new PasswordValidator();
-passwordSchema
-  .is().min(8)
-  .is().max(64)
-  .has().letters()
-  .has().digits()
-  .has().not().spaces();
-
-function listAccounts() {
-  return new Promise((resolve, reject) => {      
-    const container = docker.getContainer('mailserver');
-    container.exec({
-      Cmd: ['setup', 'email', 'list'],
-      AttachStdout: true,
-      AttachStderr: true,
-    }, (err, exec) => {
-      if (err) {
-        return reject(err);
-      }
-      
-      exec.start((err, stream) => {
-        if (err) {
-          return reject(err);
-        }
-        
-        let output = '';
-        stream.on('data', (chunk) => {
-          output += chunk.toString();
-        });
-        
-        stream.on('end', () => {
-          // Remove control characters
-          const cleanOutput = output.replace(/[\u0000-\u001F]+/g, '');
-          const regex = /\*\s*(\S+)\s*\(\s*([^\s\/]+)\s*\/\s*([^)]+)\s*\)\s*\[(\d+)%]/g;
-          const accounts = [];
-          
-          for (const match of cleanOutput.matchAll(regex)) {
-            accounts.push({
-              email: match[1],
-              used: match[2].trim() === '~' ? 'Unlimited' : match[2].trim(),
-              capacity: match[3].trim() === '~' ? 'Unlimited' : match[3].trim(),
-              percentage: match[4]
-            });
-          }
-          
-          resolve(accounts);
-        });
-      });
-    });
-  });
-}
-
-// Routes
-
-app.get('/list', (req, res) => {
-  listAccounts()
-    .then(accounts => res.json({ accounts }))
-    .catch(err => res.status(500).json({ error: err.message }));
-});
-
-app.post('/add', (req, res) => {
-  const { email, password } = req.body;
-  if (!email || !validateEmail(email) || !password) {
-    return res.status(400).json({ error: "A valid email and password is required." });
-  }
-  
-  const container = docker.getContainer('mailserver');
-  container.exec({
-    Cmd: ['setup', 'email', 'add', email, password],
-    AttachStdout: true,
-    AttachStderr: true,
-  }, (err, exec) => {
-    if (err) {
-      return res.status(500).json({ error: err.message });
-    }
-    
-    exec.start((err, stream) => {
-      if (err) {
-        return res.status(500).json({ error: err.message });
-      }
-      
-      let output = '';
-      stream.on('data', (chunk) => {
-        output += chunk.toString();
-      });
-      
-      stream.on('end', async () => {
-        const cleanOutput = output.replace(/[\u0000-\u001F]+/g, '').trim();
-        if (cleanOutput === '') {
-          return res.json({ success: true });
-        }
-        
-        try {
-          const accounts = await listAccounts();
-          const accountFound = accounts.find(acc => acc.email === email);
-          if (accountFound) {
-            return res.json({ success: true });
-          } else {
-            return res.json({ success: false, message: "Account creation failed" });
-          }
-        } catch (error) {
-          return res.status(500).json({ error: error.message });
-        }
-      });
-    });
-  });
-});
-
-app.listen(3000, () => {
-  console.log(`API listening on port 3000`);
-});

src/db/schema.ts

@@ -0,0 +1,15 @@
+import { sqliteTable, text, integer } from "drizzle-orm/sqlite-core"
+
+export const accounts = sqliteTable("accounts", {
+  id: integer("id").primaryKey(),
+  email: text("email").notNull().unique(),
+  used: text("used").notNull(),
+  capacity: text("capacity").notNull(),
+  percentage: text("percentage").notNull(),
+})
+
+export const cacheInfo = sqliteTable("cache_info", {
+  id: integer("id").primaryKey(),
+  lastUpdated: integer("last_updated").notNull(),
+})
+

src/server.ts

@@ -0,0 +1,312 @@
+import express from "express"
+import fs from "fs/promises"
+import rateLimit from "express-rate-limit"
+import Docker from "dockerode"
+import validator from "validator"
+import PasswordValidator from "password-validator"
+import { accounts, cacheInfo } from "./db/schema"
+import { eq } from "drizzle-orm"
+import type { Request, Response } from "express"
+import { drizzle } from 'drizzle-orm/bun-sqlite';
+
+interface RateLimitOptions {
+  windowMs: number
+  limit: number
+  message?: string
+}
+
+interface RateLimitConfig {
+  [endpoint: string]: RateLimitOptions
+}
+
+interface Account {
+  email: string
+  used: string
+  capacity: string
+  percentage: string
+}
+
+const docker = new Docker({ socketPath: "/var/run/docker.sock" })
+const app = express()
+const db = drizzle(process.env.DB_FILE_NAME!);
+const container = docker.getContainer("mailserver");
+app.use(express.json())
+
+let rateLimitConfig: RateLimitConfig = {}
+;(async () => {
+  try {
+    const data = await fs.readFile("/app/ratelimit.json", "utf8")
+    rateLimitConfig = JSON.parse(data)
+  } catch (err) {
+    console.error("Error loading rate limit config:", err)
+  }
+})()
+
+const createLimiter = (options: RateLimitOptions) =>
+  rateLimit({
+    windowMs: options.windowMs,
+    limit: options.limit,
+    message: options.message || "Too many requests, please try again later.",
+  })
+
+Object.entries(rateLimitConfig).forEach(([route, options]) => {
+  app.use(route, createLimiter(options))
+})
+
+const validateEmail = (email: string): boolean => validator.isEmail(email)
+
+const passwordSchema = new PasswordValidator()
+passwordSchema.is().min(8).is().max(64).has().letters().has().digits().has().not().spaces()
+
+const listAccountsFromDocker = async (): Promise<Account[]> => {
+  return new Promise((resolve, reject) => {
+    const container = docker.getContainer("mailserver")
+    container.exec(
+      {
+        Cmd: ["setup", "email", "list"],
+        AttachStdout: true,
+        AttachStderr: true,
+      },
+      (err, exec) => {
+        if (err || !exec) return reject(err || new Error("Exec is undefined"))
+        exec.start({}, (err, stream) => {
+          if (err || !stream) return reject(err || new Error("Exec stream is undefined"))
+          let output = ""
+          stream.on("data", (chunk: Buffer) => (output += chunk.toString()))
+          stream.on("end", () => {
+            const regex = /\*\s*(\S+)\s*\(\s*([^\/]+?)\s*\/\s*([^)]+?)\s*\)\s*\[(\d+)%]/g
+            const accounts: Account[] = [...output.matchAll(regex)].map((match) => ({
+              email: match[1],
+              used: match[2].trim() === "~" ? "Unlimited" : match[2].trim(),
+              capacity: match[3].trim() === "~" ? "Unlimited" : match[3].trim(),
+              percentage: match[4],
+            }))
+            resolve(accounts)
+          })
+        })
+      },
+    )
+  })
+}
+
+const updateAccountsCache = async () => {
+  const dockerAccounts = await listAccountsFromDocker();
+  await db.delete(accounts);
+  if (dockerAccounts.length > 0) {
+    await db.insert(accounts).values(dockerAccounts);
+  }
+  await db
+    .insert(cacheInfo)
+    .values({ lastUpdated: Date.now() })
+    .onConflictDoUpdate({ target: cacheInfo.id, set: { lastUpdated: Date.now() } });
+};
+
+app.get("/accounts/list", async (_req, res) => {
+  try {
+    const cacheData = await db.select().from(cacheInfo).limit(1)
+    const lastUpdated = cacheData[0]?.lastUpdated || 0
+    const currentTime = Date.now()
+
+    if (currentTime - lastUpdated > 30 * 60 * 1000) {
+      // 30 minutes
+      await updateAccountsCache()
+    }
+
+    const accountsList = await db.select().from(accounts)
+    res.json({ accounts: accountsList })
+  } catch (err) {
+    res.status(500).json({ error: (err as Error).message })
+  }
+})
+
+app.post("/accounts/user", async (req: Request, res: Response): Promise<void> => {
+  const { email } = req.body;
+  if (!validateEmail(email)) {
+    res.status(400).json({ error: "Invalid email format" });
+    return;
+  }
+
+  try {
+    const account = await db
+      .select()
+      .from(accounts)
+      .where(eq(accounts.email, email))
+      .limit(1);
+
+    if (account.length > 0) {
+      res.json({ account: account[0] });
+    } else {
+      res.status(404).json({ error: "Account not found" });
+    }
+  } catch (err) {
+    res.status(500).json({ error: (err as Error).message });
+  }
+});
+
+// TODO: The wait upon account creation needs to be more adaptive
+app.post("/accounts/add", async (req: Request, res: Response): Promise<void> => {
+  const { email, password, migrate } = req.body;
+  let failureReason = "";
+
+  if (!validateEmail(email)) {
+    failureReason = "Invalid email format";
+    console.log(`Error adding account: ${failureReason}`);
+    res.status(400).json({ error: failureReason });
+    return;
+  }
+
+  let finalPassword = password;
+
+  if (migrate) {
+    try {
+      const data = await fs.readFile("/app/migrate.txt", "utf8");
+      const line = data.split("\n").find(l => l.trim() === email);
+      if (!line) {
+        failureReason = "Account not eligible";
+        console.log(`Error adding account (migrate): ${failureReason}`);
+        res.status(500).json({ error: "Backend error" });
+        return;
+      } else {
+        const newData = data.replace(line, "");
+        await fs.writeFile("/app/migrate.txt", newData);
+        const exec = await container.exec({
+          Cmd: ["setup", "email", "update", email, finalPassword],
+          AttachStdout: true,
+          AttachStderr: true,
+        });
+        const stream = await new Promise<NodeJS.ReadableStream>((resolve, reject) => {
+          exec.start({}, (err, stream) => {
+            if (err || !stream) {
+              reject(err || new Error("Exec stream is undefined"));
+            } else {
+              resolve(stream);
+            }
+          });
+        });
+        let output = "";
+        stream.on("data", (chunk: Buffer) => {
+          output += chunk.toString();
+        });
+        await new Promise<void>((resolve) => stream.on("end", resolve));
+        console.log("Docker output (migrate update):\n", output);
+        // detect errors
+        if (/ERROR/i.test(output)) {
+          failureReason = "Migration failed";
+          console.log(`Error during migration: ${failureReason}`);
+          res.status(500).json({ error: failureReason });
+          return;
+        } else {
+          // force refresh
+          await updateAccountsCache();
+          const addedAccount = await db
+            .select()
+            .from(accounts)
+            .where(eq(accounts.email, email))
+            .limit(1);
+          if (addedAccount.length > 0) {
+            console.log(`Added account (via migrate): ${email}`);
+            res.json({ success: true });
+            return;
+          } else {
+            failureReason = "Migration failed";
+            console.log(`Failed to migrate account: ${failureReason}`);
+            res.status(500).json({ error: failureReason });
+            return;
+          }
+        }
+      }
+    } catch (error) {
+      failureReason = "Backend error";
+      console.log(`Error adding account (migrate branch): ${failureReason}`, error);
+      res.status(500).json({ error: failureReason });
+      return;
+    }
+  } else {
+    try {
+      const account = await db
+        .select()
+        .from(accounts)
+        .where(eq(accounts.email, email))
+        .limit(1);
+      if (account.length > 0) {
+        console.log(`Account already exists: ${email}`);
+        res.status(400).json({ error: "Account already exists" });
+        return;
+      }
+    } catch (err) {
+      console.log(`Error checking user: ${err}`);
+      res.status(500).json({ error: "Internal server error" });
+      return;
+    }
+  }
+
+  if (!finalPassword || !passwordSchema.validate(finalPassword)) {
+    failureReason = "Invalid or weak password";
+    console.log(`Failed to add account: ${failureReason}`);
+    res.status(400).json({ error: failureReason });
+    return;
+  }
+
+  // Non-migrate account creation route
+  try {
+    const exec = await container.exec({
+      Cmd: ["setup", "email", "add", email, finalPassword],
+      AttachStdout: true,
+      AttachStderr: true,
+    });
+
+    const stream = await new Promise<NodeJS.ReadableStream>((resolve, reject) => {
+      exec.start({}, (err, stream) => {
+        if (err || !stream) {
+          failureReason = "Failed to start Docker exec";
+          console.log(`Failed to add account: ${failureReason}`, err);
+          reject(err || new Error("Exec stream is undefined"));
+        } else {
+          resolve(stream);
+        }
+      });
+    });
+
+    let output = "";
+    stream.on("data", (chunk: Buffer) => {
+      output += chunk.toString();
+    });
+    await new Promise<void>((resolve) => stream.on("end", resolve));
+
+    // poll db
+    let accountFound = false;
+    for (let attempt = 0; attempt < 15; attempt++) {
+      await updateAccountsCache();
+      const addedAccount = await db
+        .select()
+        .from(accounts)
+        .where(eq(accounts.email, email))
+        .limit(1);
+      if (addedAccount.length > 0) {
+        accountFound = true;
+        break;
+      }
+      await new Promise((resolve) => setTimeout(resolve, 1000));
+    }
+
+    if (accountFound) {
+      console.log(`Added account: ${email}`);
+      res.json({ success: true });
+    } else {
+      failureReason = "Timed out waiting for account creation";
+      console.log(`Failed to add account: ${failureReason}`);
+      res.status(500).json({
+        error: "Check timed out waiting for account creation",
+        failureReason,
+      });
+    }
+  } catch (error) {
+    failureReason = "Error executing Docker command";
+    console.log(`Failed to add account: ${failureReason}`, error);
+    res.status(500).json({ error: "Backend error" });
+  }
+});
+
+const PORT = 3000
+app.listen(PORT, () => console.log(`API listening on port ${PORT}`))
+

tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "ESNext",
+    "moduleResolution": "node",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true
+  }
+}