fix: fixed SMTP limiter for API (still need to implement better credi…

…t system), added stargazers over time to README, removed old server IPs, fixed concurrency in tests, tuned virus/spam banning technique
forwardemail · Feb 13, 2025 · b238f97 · b238f97
1 parent 6708678
commit b238f97
Show file tree

Hide file tree

Showing 15 changed files with 260 additions and 184 deletions.
diff --git a/README.md b/README.md
@@ -18,6 +18,7 @@
 
 ## Table of Contents
 
+* [Stargazers over time](#stargazers-over-time)
 * [How do I get started](#how-do-i-get-started)
   * [For Consumers](#for-consumers)
   * [For Developers](#for-developers)
@@ -35,6 +36,11 @@
 * [License](#license)
 
 
+## Stargazers over time
+
+[![Stargazers over time](https://starchart.cc/forwardemail/forwardemail.net.svg?variant=adaptive)](https://starchart.cc/forwardemail/forwardemail.net)
+
+
 ## How do I get started
 
 ### For Consumers

diff --git a/app/controllers/api/v1/emails.js b/app/controllers/api/v1/emails.js
@@ -251,9 +251,6 @@ async function create(ctx) {
     message.disableFileAccess = true;
     message.disableUrlAccess = true;
 
-    // TODO: rate limiting emails per day by domain id and alias user id
-    // TODO: implement credit system
-
     // queue the email
     const email = await Emails.queue(
       { message, user: ctx.state.user },

diff --git a/app/models/emails.js b/app/models/emails.js
@@ -3,12 +3,18 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
+const process = require('node:process');
 const punycode = require('node:punycode');
 const { Buffer } = require('node:buffer');
 const { isIP } = require('node:net');
 
 const Axe = require('axe');
 const Boom = require('@hapi/boom');
+const RateLimiter = require('async-ratelimiter');
+const Redis =
+  process.env.NODE_ENV === 'test'
+    ? require('ioredis-mock')
+    : require('@ladjs/redis');
 const SpamScanner = require('spamscanner');
 const _ = require('lodash');
 const bytes = require('@forwardemail/bytes');
@@ -25,6 +31,7 @@ const noReplyList = require('reserved-email-addresses-list/no-reply-list.json');
 const nodemailer = require('nodemailer');
 const pEvent = require('p-event');
 const parseErr = require('parse-err');
+const sharedConfig = require('@ladjs/shared-config');
 const { Headers, Splitter, Joiner } = require('mailsplit');
 const { Iconv } = require('iconv');
 const { boolean } = require('boolean');
@@ -36,6 +43,7 @@ const Users = require('./users');
 const isEmail = require('#helpers/is-email');
 
 const MessageSplitter = require('#helpers/message-splitter');
+const SMTPError = require('#helpers/smtp-error');
 const checkSRS = require('#helpers/check-srs');
 const config = require('#config');
 const emailHelper = require('#helpers/email');
@@ -83,6 +91,22 @@ const scanner = new SpamScanner({
   clamscan: env.NODE_ENV === 'test'
 });
 
+const webSharedConfig = sharedConfig('WEB');
+
+// TODO: we should find a way to share the existing redis connection
+const redis = new Redis(
+  webSharedConfig.redis,
+  logger,
+  webSharedConfig.redisMonitor
+);
+
+const rateLimiter = new RateLimiter({
+  db: redis,
+  max: config.smtpLimitMessages,
+  duration: config.smtpLimitDuration,
+  namespace: config.smtpLimitNamespace
+});
+
 const Emails = new mongoose.Schema(
   {
     is_redacted: {
@@ -810,6 +834,195 @@ Emails.post('save', async function (email) {
   }
 });
 
+Emails.pre('save', async function (next) {
+  this._isNew = this.isNew;
+  next();
+});
+
+async function sendRateLimitEmail(user) {
+  // if the user received rate limit email in past 30d
+  if (
+    _.isDate(user.smtp_rate_limit_sent_at) &&
+    dayjs().isBefore(dayjs(user.smtp_rate_limit_sent_at).add(30, 'days'))
+  ) {
+    logger.info('user was already rate limited');
+    return;
+  }
+
+  await emailHelper({
+    template: 'alert',
+    message: {
+      to: user[config.userFields.fullEmail],
+      bcc: config.email.message.from,
+      locale: user[config.lastLocaleField],
+      subject: i18n.translate(
+        'SMTP_RATE_LIMIT_EXCEEDED',
+        user[config.lastLocaleField]
+      )
+    },
+    locals: {
+      locale: user[config.lastLocaleField],
+      message: i18n.translate(
+        'SMTP_RATE_LIMIT_EXCEEDED',
+        user[config.lastLocaleField]
+      )
+    }
+  });
+
+  // otherwise send the user an email and update the user record
+  await Users.findByIdAndUpdate(user._id, {
+    $set: {
+      smtp_rate_limit_sent_at: new Date()
+    }
+  });
+}
+
+Emails.pre('save', async function (next) {
+  // if it's not a new email then no need to check for credits
+  if (!this._isNew) return next();
+
+  try {
+    const [user, domain] = await Promise.all([
+      // TODO: limit fields returned
+      Users.findById(this.user).lean().exec(),
+      // TODO: limit fields returned
+      Domains.findById(this.domain).lean().exec()
+    ]);
+    if (!user) throw new SMTPError('User does not exist', { ignoreHook: true });
+    if (!domain)
+      throw new SMTPError('Domain does not exist', { ignoreHook: true });
+
+    //
+    // TODO: it's not using the largest SMTP limit from the domain-wide admins here (?)
+    //       (this will change with a new credit system; so we will change this later)
+    //
+    const max = user[config.userFields.smtpLimit] || config.smtpLimitMessages;
+
+    // if any of the domain admins are admins then don't rate limit
+    const adminExists = await Users.exists({
+      _id: {
+        $in: domain.members
+          .filter((m) => m.group === 'admin' && typeof m.user === 'object')
+          .map((m) =>
+            typeof m.user === 'object' && typeof m?.user?._id === 'object'
+              ? m.user._id
+              : m.user
+          )
+      },
+      group: 'admin'
+    });
+
+    if (!adminExists) {
+      // rate limit to X emails per day by domain id then denylist
+      {
+        const count = await redis.zcard(
+          `${config.smtpLimitNamespace}:${domain.id}`
+        );
+        // return 550 error code
+        if (count >= max) {
+          // send one-time email alert to admin + user
+          sendRateLimitEmail(user)
+            .then()
+            .catch((err) => logger.fatal(err));
+          throw new SMTPError('Rate limit exceeded', { ignoreHook: true });
+        }
+      }
+
+      // rate limit to X emails per day by alias user id then denylist
+      {
+        const count = await redis.zcard(
+          `${config.smtpLimitNamespace}:${user.id}`
+        );
+        // return 550 error code
+        if (count >= max) {
+          // send one-time email alert to admin + user
+          sendRateLimitEmail(user)
+            .then()
+            .catch((err) => logger.fatal(err));
+          throw new SMTPError('Rate limit exceeded', { ignoreHook: true });
+        }
+      }
+    }
+
+    next();
+  } catch (err) {
+    next(err);
+  }
+});
+
+Emails.post('save', async function (email, next) {
+  // if it's not a new email then no need to deduct credits
+  if (!email._isNew) return next();
+
+  try {
+    const [user, domain] = await Promise.all([
+      // TODO: limit fields returned
+      Users.findById(this.user).lean().exec(),
+      // TODO: limit fields returned
+      Domains.findById(this.domain).lean().exec()
+    ]);
+    if (!user) throw new SMTPError('User does not exist', { ignoreHook: true });
+    if (!domain)
+      throw new SMTPError('Domain does not exist', { ignoreHook: true });
+
+    //
+    // TODO: it's not using the largest SMTP limit from the domain-wide admins here (?)
+    //       (this will change with a new credit system; so we will change this later)
+    //
+    const max = user[config.userFields.smtpLimit] || config.smtpLimitMessages;
+
+    // if any of the domain admins are admins then don't rate limit
+    const adminExists = await Users.exists({
+      _id: {
+        $in: domain.members
+          .filter((m) => m.group === 'admin' && typeof m.user === 'object')
+          .map((m) =>
+            typeof m.user === 'object' && typeof m?.user?._id === 'object'
+              ? m.user._id
+              : m.user
+          )
+      },
+      group: 'admin'
+    });
+
+    if (!adminExists) {
+      try {
+        // rate limit to X emails per day by domain id then denylist
+        {
+          const limit = await rateLimiter.get({
+            id: domain.id,
+            max
+          });
+
+          // return 550 error code
+          if (!limit.remaining)
+            throw new SMTPError('Rate limit exceeded', { ignoreHook: true });
+        }
+
+        // rate limit to X emails per day by alias user id then denylist
+        const limit = await rateLimiter.get({
+          id: user.id,
+          max
+        });
+
+        // return 550 error code
+        if (!limit.remaining)
+          throw new SMTPError('Rate limit exceeded', { ignoreHook: true });
+      } catch (err) {
+        // remove the job from the queue
+        Emails.findByIdAndRemove(email._id)
+          .then()
+          .catch((err) => logger.fatal(err));
+        throw err;
+      }
+    }
+
+    next();
+  } catch (err) {
+    next(err);
+  }
+});
+
 Emails.statics.getMessage = async function (obj, returnString = false) {
   if (Buffer.isBuffer(obj)) {
     if (returnString) return obj.toString();

diff --git a/app/models/logs.js b/app/models/logs.js
@@ -5,11 +5,15 @@
 
 const dns = require('node:dns');
 const os = require('node:os');
+const process = require('node:process');
 const { Buffer } = require('node:buffer');
 const { isIP } = require('node:net');
 
 const Graceful = require('@ladjs/graceful');
-const Redis = require('@ladjs/redis');
+const Redis =
+  process.env.NODE_ENV === 'test'
+    ? require('ioredis-mock')
+    : require('@ladjs/redis');
 const _ = require('lodash');
 const ansiHTML = require('ansi-html-community');
 const bytes = require('@forwardemail/bytes');

diff --git a/ava.config.js b/ava.config.js
@@ -13,7 +13,7 @@ module.exports = {
   verbose: true,
   failFast: true,
   // serial: true,
-  concurrency: isCI ? 2 : Math.floor(os.cpus().length / 2),
+  concurrency: isCI ? 2 : Math.floor(os.cpus().length / 3),
   files: ['test/*.js', 'test/**/*.js', 'test/**/**/*.js', '!test/utils.js'],
   // <https://github.com/lovell/sharp/issues/3164#issuecomment-1168328811>
   // workerThreads: familySync() !== GLIBC,

diff --git a/ecosystem-api.json b/ecosystem-api.json
@@ -31,7 +31,7 @@
   "deploy": {
     "production": {
       "user": "deploy",
-      "host": ["121.127.44.68","161.35.225.85", "149.28.198.68"],
+      "host": ["121.127.44.68"],
       "ref": "origin/master",
       "repo": "[email protected]:forwardemail/forwardemail.net.git",
       "path": "/var/www/production",

diff --git a/ecosystem-caldav.json b/ecosystem-caldav.json
@@ -31,7 +31,7 @@
   "deploy": {
     "production": {
       "user": "deploy",
-      "host": ["121.127.44.72","64.23.142.64"],
+      "host": ["121.127.44.72"],
       "ref": "origin/master",
       "repo": "[email protected]:forwardemail/forwardemail.net.git",
       "path": "/var/www/production",

diff --git a/ecosystem-imap.json b/ecosystem-imap.json
@@ -35,7 +35,7 @@
   "deploy": {
     "production": {
       "user": "deploy",
-      "host": ["121.127.44.70","64.23.247.113"],
+      "host": ["121.127.44.70"],
       "ref": "origin/master",
       "repo": "[email protected]:forwardemail/forwardemail.net.git",
       "path": "/var/www/production",

diff --git a/ecosystem-pop3.json b/ecosystem-pop3.json
@@ -32,7 +32,7 @@
   "deploy": {
     "production": {
       "user": "deploy",
-      "host": ["121.127.44.71","45.32.138.135"],
+      "host": ["121.127.44.71"],
       "ref": "origin/master",
       "repo": "[email protected]:forwardemail/forwardemail.net.git",
       "path": "/var/www/production",

diff --git a/ecosystem-web.json b/ecosystem-web.json
@@ -31,7 +31,7 @@
   "deploy": {
     "production": {
       "user": "deploy",
-      "host": ["121.127.44.69","144.202.105.188","64.23.134.242"],
+      "host": ["121.127.44.69"],
       "ref": "origin/master",
       "repo": "[email protected]:forwardemail/forwardemail.net.git",
       "path": "/var/www/production",

diff --git a/helpers/get-bounce-info.js b/helpers/get-bounce-info.js
@@ -270,6 +270,12 @@ function getBounceInfo(err) {
       // optimum-specific error message
       response.includes('4.7.1 Resources restricted') ||
       // CenturyLink/Cloudfilter rejection (421 mwd-ibgw-6004a.ext.cloudfilter.net cmsmtp 138.197.213.185 blocked AUP#CNCT:)
+      // cloudfilter.net
+      // tmomail.net
+      // <https://postmaster.t-online.de/index.en.html>
+      // <https://postmaster.t-online.de/kontakt.en.php>
+      // postmaster@t-online.de
+      // postmaster@rx.t-online.de
       response.includes('#CNCT') ||
       response.includes('#CXCNCT') ||
       response.includes('#CXMXRT') ||