diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ff969d1 --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +MIGRATE_TXT="migrate.txt" +DB_FILE_NAME="db.sqlite3" \ No newline at end of file diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..5ba514e --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,28 @@ +name: Build and Push Nightly CI Image + +on: + schedule: + - cron: '0 0 * * *' + push: + branches: + - main + +jobs: + build_and_push: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Log in to Gitea Package Registry + run: echo "${{ secrets.PACKAGE_TOKEN }}" | docker login $SERVER_URL -u $USERNAME --password-stdin + env: + SERVER_URL: ${{ secrets.SERVER_URL }} + USERNAME: ${{ secrets.USERNAME }} + PACKAGE_TOKEN: ${{ secrets.PACKAGE_TOKEN }} + + - name: Build Docker Image (ci tag) + run: docker build -t git.pontusmail.org/librecloud/mail-connect:ci . + + - name: Push Docker Image (ci tag) + run: docker push git.pontusmail.org/librecloud/mail-connect:ci diff --git a/.gitignore b/.gitignore index 2e917f1..b6d1809 100644 --- a/.gitignore +++ b/.gitignore @@ -132,5 +132,8 @@ dist # extra .idea/ +drizzle/ bun.lockb +migrate.txt +db.sqlite3 diff --git a/Dockerfile b/Dockerfile index b551492..013a24d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ FROM oven/bun:latest WORKDIR /app COPY package*.json /app/ +RUN apt-get update && apt-get install -y python3 build-essential RUN bun install COPY . /app EXPOSE 3000 -CMD [ "bun", "run", "server.js" ] \ No newline at end of file +CMD [ "bun", "run", "src/server.ts" ] \ No newline at end of file diff --git a/README.md b/README.md index 1958de1..737edf8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # mail-connect +[![Build Status](https://git.pontusmail.org/librecloud/web/actions/workflows/docker.yaml/badge.svg)](https://git.pontusmail.org/librecloud/web/actions/workflows/docker.yml) + API bridge for docker-mailserver *mail-connect is still in early beta* @@ -8,17 +10,51 @@ API bridge for docker-mailserver mail-connect aims to connect your `docker-mailserver` to *anything* you can imagine, through the power of an API. Despite being used as a core component of LibreCloud, you can still implement mail-connect any way you wish! -We provide an extendable API which interacts with the `setup` utility via a Docker socket. While this offers advantages, mail-connect is still slow on some functions (such as listing accounts), as it's merely executing pre-made commands and parsing the output. +We provide an extendable API which interacts with the `setup` utility via a Docker socket. We have implemented a SQLite database with Drizzle ORM for faster polling of users, with strategic caching and updating. ## Features All features marked with an **E** are extended features, and are not a part of the original `setup` utility. +## Self-Host with Docker + +1. **Clone the repository** + + ```bash + git clone https://git.pontusmail.org/librecloud/mail-connect.git + ``` + +2. **Initialize database** + + ```bash + bunx drizzle-kit generate + bunx drizzle-kit migrate + ``` + +3. **Copy/modify necessary files** + + ```bash + touch migrate.txt # put emails (one per line) which already exist on the server which users can claim + cp .env.example .env # you don't need to change anything here + vim ratelimit.json # optional, customize to your liking + ``` + +4. **Build and run the container** + + ```bash + docker-compose up -d --build + ``` + + Pre-built images are provided in the [Packages](https://git.pontusmail.org/librecloud/mail-connect/packages) tab + + Your server will now be running on port `6723` by default (change this in `docker-compose.yml`)! + ### Email - [X] Create email - [X] List emails -- [ ] **E** Create email from file +- [X] **E** View individual user details +- [X] **E** Create email from file - [ ] Change password - [ ] Delete email - [ ] Restrict email @@ -54,7 +90,7 @@ All features marked with an **E** are extended features, and are not a part of t ### Fail2Ban - [ ] Ban IP -- [ ] Unban IP +- [ ] Un-ban IP - [ ] Ban log - [ ] Fail2Ban status @@ -71,3 +107,6 @@ I plan to implement a *much* more powerful API, when everything else has settled Since `docker-mailserver` is built on Dovecot and Postfix, I am confident we can improve this API to be speedy and efficient as ever. ## To-Do + +- [ ] Implement aforementioned features +- [ ] Swagger support diff --git a/docker-compose.yml b/docker-compose.yml index ac5dbc1..a58573f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,9 @@ services: mail-connect: build: . container_name: mail-connect + env_file: ".env" ports: - "6723:3000" volumes: - - '/var/run/docker.sock:/var/run/docker.sock' \ No newline at end of file + - "/var/run/docker.sock:/var/run/docker.sock" + - ".:/app" \ No newline at end of file diff --git a/drizzle.config.ts b/drizzle.config.ts new file mode 100644 index 0000000..d280834 --- /dev/null +++ b/drizzle.config.ts @@ -0,0 +1,11 @@ +import 'dotenv/config'; +import { defineConfig } from 'drizzle-kit'; + +export default defineConfig({ + out: './drizzle', + schema: './src/db/schema.ts', + dialect: 'sqlite', + dbCredentials: { + url: process.env.DB_FILE_NAME!, + }, +}); \ No newline at end of file diff --git a/package.json b/package.json index e87b044..70072bf 100644 --- a/package.json +++ b/package.json @@ -1,22 +1,34 @@ { "name": "mail-connect", - "module": "server.js", + "module": "src/server.ts", "type": "module", "devDependencies": { - "@types/bun": "latest" + "@types/bun": "latest", + "@types/dockerode": "^3.3.34", + "@types/express": "^5.0.0", + "@types/node": "^22.13.4", + "@types/validator": "^13.12.2", + "drizzle-kit": "^0.30.4" }, "peerDependencies": { - "typescript": "^5.0.0" + "typescript": "^5.7.3" }, "dependencies": { + "@types/better-sqlite3": "^7.6.12", + "better-sqlite3": "^11.8.1", "child_process": "^1.0.2", "dockerode": "^4.0.4", + "dotenv": "^16.4.7", + "drizzle-orm": "^0.39.3", "express": "^4.21.2", "express-rate-limit": "^7.5.0", "password-validator": "^5.3.0", "validator": "^13.12.0" }, "trustedDependencies": [ - "protobufjs" + "better-sqlite3", + "cpu-features", + "protobufjs", + "ssh2" ] } \ No newline at end of file diff --git a/ratelimit.json b/ratelimit.json index ae8b433..87128d4 100644 --- a/ratelimit.json +++ b/ratelimit.json @@ -1,10 +1,10 @@ { "/list": { "windowMs": 60000, - "max": 20 + "limit": 20 }, "/add": { "windowMs": 60000, - "max": 10 + "limit": 10 } } \ No newline at end of file diff --git a/server.js b/server.js deleted file mode 100644 index 2c436fa..0000000 --- a/server.js +++ /dev/null @@ -1,153 +0,0 @@ -const express = require('express'); -const fs = require('fs'); -const rateLimit = require('express-rate-limit'); -const Docker = require('dockerode'); -const validator = require('validator'); - -const docker = new Docker({ socketPath: '/var/run/docker.sock' }); -const app = express(); -app.use(express.json()); - -// Rate limiting - -let rateLimitConfig = {}; -try { - const data = fs.readFileSync('./ratelimit.json', 'utf8'); - rateLimitConfig = JSON.parse(data); -} catch (err) { - console.error('Error loading rate limit config:', err); -} - -function createLimiter(options) { - return rateLimit({ - windowMs: options.windowMs, - max: options.max, - message: "Too many requests, please try again another time.", - }); -} - -if (rateLimitConfig['/list']) { - app.use('/list', createLimiter(rateLimitConfig['/list'])); -} -if (rateLimitConfig['/add']) { - app.use('/add', createLimiter(rateLimitConfig['/add'])); -} - -// Utility fxns - -function validateEmail(email) { - return typeof email === 'string' && validator.isEmail(email); -} - -const PasswordValidator = require('password-validator'); -const passwordSchema = new PasswordValidator(); -passwordSchema - .is().min(8) - .is().max(64) - .has().letters() - .has().digits() - .has().not().spaces(); - -function listAccounts() { - return new Promise((resolve, reject) => { - const container = docker.getContainer('mailserver'); - container.exec({ - Cmd: ['setup', 'email', 'list'], - AttachStdout: true, - AttachStderr: true, - }, (err, exec) => { - if (err) { - return reject(err); - } - - exec.start((err, stream) => { - if (err) { - return reject(err); - } - - let output = ''; - stream.on('data', (chunk) => { - output += chunk.toString(); - }); - - stream.on('end', () => { - // Remove control characters - const cleanOutput = output.replace(/[\u0000-\u001F]+/g, ''); - const regex = /\*\s*(\S+)\s*\(\s*([^\s\/]+)\s*\/\s*([^)]+)\s*\)\s*\[(\d+)%]/g; - const accounts = []; - - for (const match of cleanOutput.matchAll(regex)) { - accounts.push({ - email: match[1], - used: match[2].trim() === '~' ? 'Unlimited' : match[2].trim(), - capacity: match[3].trim() === '~' ? 'Unlimited' : match[3].trim(), - percentage: match[4] - }); - } - - resolve(accounts); - }); - }); - }); - }); -} - -// Routes - -app.get('/list', (req, res) => { - listAccounts() - .then(accounts => res.json({ accounts })) - .catch(err => res.status(500).json({ error: err.message })); -}); - -app.post('/add', (req, res) => { - const { email, password } = req.body; - if (!email || !validateEmail(email) || !password) { - return res.status(400).json({ error: "A valid email and password is required." }); - } - - const container = docker.getContainer('mailserver'); - container.exec({ - Cmd: ['setup', 'email', 'add', email, password], - AttachStdout: true, - AttachStderr: true, - }, (err, exec) => { - if (err) { - return res.status(500).json({ error: err.message }); - } - - exec.start((err, stream) => { - if (err) { - return res.status(500).json({ error: err.message }); - } - - let output = ''; - stream.on('data', (chunk) => { - output += chunk.toString(); - }); - - stream.on('end', async () => { - const cleanOutput = output.replace(/[\u0000-\u001F]+/g, '').trim(); - if (cleanOutput === '') { - return res.json({ success: true }); - } - - try { - const accounts = await listAccounts(); - const accountFound = accounts.find(acc => acc.email === email); - if (accountFound) { - return res.json({ success: true }); - } else { - return res.json({ success: false, message: "Account creation failed" }); - } - } catch (error) { - return res.status(500).json({ error: error.message }); - } - }); - }); - }); -}); - -app.listen(3000, () => { - console.log(`API listening on port 3000`); -}); \ No newline at end of file diff --git a/src/db/schema.ts b/src/db/schema.ts new file mode 100644 index 0000000..ffa8441 --- /dev/null +++ b/src/db/schema.ts @@ -0,0 +1,15 @@ +import { sqliteTable, text, integer } from "drizzle-orm/sqlite-core" + +export const accounts = sqliteTable("accounts", { + id: integer("id").primaryKey(), + email: text("email").notNull().unique(), + used: text("used").notNull(), + capacity: text("capacity").notNull(), + percentage: text("percentage").notNull(), +}) + +export const cacheInfo = sqliteTable("cache_info", { + id: integer("id").primaryKey(), + lastUpdated: integer("last_updated").notNull(), +}) + diff --git a/src/server.ts b/src/server.ts new file mode 100644 index 0000000..e35a3c9 --- /dev/null +++ b/src/server.ts @@ -0,0 +1,312 @@ +import express from "express" +import fs from "fs/promises" +import rateLimit from "express-rate-limit" +import Docker from "dockerode" +import validator from "validator" +import PasswordValidator from "password-validator" +import { accounts, cacheInfo } from "./db/schema" +import { eq } from "drizzle-orm" +import type { Request, Response } from "express" +import { drizzle } from 'drizzle-orm/bun-sqlite'; + +interface RateLimitOptions { + windowMs: number + limit: number + message?: string +} + +interface RateLimitConfig { + [endpoint: string]: RateLimitOptions +} + +interface Account { + email: string + used: string + capacity: string + percentage: string +} + +const docker = new Docker({ socketPath: "/var/run/docker.sock" }) +const app = express() +const db = drizzle(process.env.DB_FILE_NAME!); +const container = docker.getContainer("mailserver"); +app.use(express.json()) + +let rateLimitConfig: RateLimitConfig = {} +;(async () => { + try { + const data = await fs.readFile("/app/ratelimit.json", "utf8") + rateLimitConfig = JSON.parse(data) + } catch (err) { + console.error("Error loading rate limit config:", err) + } +})() + +const createLimiter = (options: RateLimitOptions) => + rateLimit({ + windowMs: options.windowMs, + limit: options.limit, + message: options.message || "Too many requests, please try again later.", + }) + +Object.entries(rateLimitConfig).forEach(([route, options]) => { + app.use(route, createLimiter(options)) +}) + +const validateEmail = (email: string): boolean => validator.isEmail(email) + +const passwordSchema = new PasswordValidator() +passwordSchema.is().min(8).is().max(64).has().letters().has().digits().has().not().spaces() + +const listAccountsFromDocker = async (): Promise => { + return new Promise((resolve, reject) => { + const container = docker.getContainer("mailserver") + container.exec( + { + Cmd: ["setup", "email", "list"], + AttachStdout: true, + AttachStderr: true, + }, + (err, exec) => { + if (err || !exec) return reject(err || new Error("Exec is undefined")) + exec.start({}, (err, stream) => { + if (err || !stream) return reject(err || new Error("Exec stream is undefined")) + let output = "" + stream.on("data", (chunk: Buffer) => (output += chunk.toString())) + stream.on("end", () => { + const regex = /\*\s*(\S+)\s*\(\s*([^\/]+?)\s*\/\s*([^)]+?)\s*\)\s*\[(\d+)%]/g + const accounts: Account[] = [...output.matchAll(regex)].map((match) => ({ + email: match[1], + used: match[2].trim() === "~" ? "Unlimited" : match[2].trim(), + capacity: match[3].trim() === "~" ? "Unlimited" : match[3].trim(), + percentage: match[4], + })) + resolve(accounts) + }) + }) + }, + ) + }) +} + +const updateAccountsCache = async () => { + const dockerAccounts = await listAccountsFromDocker(); + await db.delete(accounts); + if (dockerAccounts.length > 0) { + await db.insert(accounts).values(dockerAccounts); + } + await db + .insert(cacheInfo) + .values({ lastUpdated: Date.now() }) + .onConflictDoUpdate({ target: cacheInfo.id, set: { lastUpdated: Date.now() } }); +}; + +app.get("/accounts/list", async (_req, res) => { + try { + const cacheData = await db.select().from(cacheInfo).limit(1) + const lastUpdated = cacheData[0]?.lastUpdated || 0 + const currentTime = Date.now() + + if (currentTime - lastUpdated > 30 * 60 * 1000) { + // 30 minutes + await updateAccountsCache() + } + + const accountsList = await db.select().from(accounts) + res.json({ accounts: accountsList }) + } catch (err) { + res.status(500).json({ error: (err as Error).message }) + } +}) + +app.post("/accounts/user", async (req: Request, res: Response): Promise => { + const { email } = req.body; + if (!validateEmail(email)) { + res.status(400).json({ error: "Invalid email format" }); + return; + } + + try { + const account = await db + .select() + .from(accounts) + .where(eq(accounts.email, email)) + .limit(1); + + if (account.length > 0) { + res.json({ account: account[0] }); + } else { + res.status(404).json({ error: "Account not found" }); + } + } catch (err) { + res.status(500).json({ error: (err as Error).message }); + } +}); + +// TODO: The wait upon account creation needs to be more adaptive +app.post("/accounts/add", async (req: Request, res: Response): Promise => { + const { email, password, migrate } = req.body; + let failureReason = ""; + + if (!validateEmail(email)) { + failureReason = "Invalid email format"; + console.log(`Error adding account: ${failureReason}`); + res.status(400).json({ error: failureReason }); + return; + } + + let finalPassword = password; + + if (migrate) { + try { + const data = await fs.readFile("/app/migrate.txt", "utf8"); + const line = data.split("\n").find(l => l.trim() === email); + if (!line) { + failureReason = "Account not eligible"; + console.log(`Error adding account (migrate): ${failureReason}`); + res.status(500).json({ error: "Backend error" }); + return; + } else { + const newData = data.replace(line, ""); + await fs.writeFile("/app/migrate.txt", newData); + const exec = await container.exec({ + Cmd: ["setup", "email", "update", email, finalPassword], + AttachStdout: true, + AttachStderr: true, + }); + const stream = await new Promise((resolve, reject) => { + exec.start({}, (err, stream) => { + if (err || !stream) { + reject(err || new Error("Exec stream is undefined")); + } else { + resolve(stream); + } + }); + }); + let output = ""; + stream.on("data", (chunk: Buffer) => { + output += chunk.toString(); + }); + await new Promise((resolve) => stream.on("end", resolve)); + console.log("Docker output (migrate update):\n", output); + // detect errors + if (/ERROR/i.test(output)) { + failureReason = "Migration failed"; + console.log(`Error during migration: ${failureReason}`); + res.status(500).json({ error: failureReason }); + return; + } else { + // force refresh + await updateAccountsCache(); + const addedAccount = await db + .select() + .from(accounts) + .where(eq(accounts.email, email)) + .limit(1); + if (addedAccount.length > 0) { + console.log(`Added account (via migrate): ${email}`); + res.json({ success: true }); + return; + } else { + failureReason = "Migration failed"; + console.log(`Failed to migrate account: ${failureReason}`); + res.status(500).json({ error: failureReason }); + return; + } + } + } + } catch (error) { + failureReason = "Backend error"; + console.log(`Error adding account (migrate branch): ${failureReason}`, error); + res.status(500).json({ error: failureReason }); + return; + } + } else { + try { + const account = await db + .select() + .from(accounts) + .where(eq(accounts.email, email)) + .limit(1); + if (account.length > 0) { + console.log(`Account already exists: ${email}`); + res.status(400).json({ error: "Account already exists" }); + return; + } + } catch (err) { + console.log(`Error checking user: ${err}`); + res.status(500).json({ error: "Internal server error" }); + return; + } + } + + if (!finalPassword || !passwordSchema.validate(finalPassword)) { + failureReason = "Invalid or weak password"; + console.log(`Failed to add account: ${failureReason}`); + res.status(400).json({ error: failureReason }); + return; + } + + // Non-migrate account creation route + try { + const exec = await container.exec({ + Cmd: ["setup", "email", "add", email, finalPassword], + AttachStdout: true, + AttachStderr: true, + }); + + const stream = await new Promise((resolve, reject) => { + exec.start({}, (err, stream) => { + if (err || !stream) { + failureReason = "Failed to start Docker exec"; + console.log(`Failed to add account: ${failureReason}`, err); + reject(err || new Error("Exec stream is undefined")); + } else { + resolve(stream); + } + }); + }); + + let output = ""; + stream.on("data", (chunk: Buffer) => { + output += chunk.toString(); + }); + await new Promise((resolve) => stream.on("end", resolve)); + + // poll db + let accountFound = false; + for (let attempt = 0; attempt < 15; attempt++) { + await updateAccountsCache(); + const addedAccount = await db + .select() + .from(accounts) + .where(eq(accounts.email, email)) + .limit(1); + if (addedAccount.length > 0) { + accountFound = true; + break; + } + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + if (accountFound) { + console.log(`Added account: ${email}`); + res.json({ success: true }); + } else { + failureReason = "Timed out waiting for account creation"; + console.log(`Failed to add account: ${failureReason}`); + res.status(500).json({ + error: "Check timed out waiting for account creation", + failureReason, + }); + } + } catch (error) { + failureReason = "Error executing Docker command"; + console.log(`Failed to add account: ${failureReason}`, error); + res.status(500).json({ error: "Backend error" }); + } +}); + +const PORT = 3000 +app.listen(PORT, () => console.log(`API listening on port ${PORT}`)) + diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..fbee385 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "ESNext", + "moduleResolution": "node", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true + } +} \ No newline at end of file