API v3 Rate Limiter (#12117)

* simplify ip address management by using the trust proxy express option

* add setupExpress file

* fix redirects middleware tests

* fix lint

* short circuit the ip blocking middleware

* basic implementation with ip based limiting

* improve logging

* upgrade apidoc

* apidoc: add introduction section

* fix lint

* fix tests

* fix lint

* add unit tests for rate limiter

* do not send retry-after header when points are available

* automatically fix lint

* fix more lint issues

* use userId as key for rate limit when available

test/api/unit/middlewares/ipBlocker.test.js

@@ -57,7 +57,7 @@ describe('ipBlocker middleware', () => {
   });
 
   it('does not throw when the ip does not match', () => {
-    req.headers['x-forwarded-for'] = '192.168.1.1';
+    req.ip = '192.168.1.1';
     sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.2');
     const attachIpBlocker = requireAgain(pathToIpBlocker).default;
     attachIpBlocker(req, res, next);
@@ -65,30 +65,12 @@ describe('ipBlocker middleware', () => {
     checkErrorNotThrown(next);
   });
 
-  it('throws when a matching ip exist in x-forwarded-for', () => {
-    req.headers['x-forwarded-for'] = '192.168.1.1';
+  it('throws when the ip is blocked', () => {
+    req.ip = '192.168.1.1';
     sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.1');
     const attachIpBlocker = requireAgain(pathToIpBlocker).default;
     attachIpBlocker(req, res, next);
 
     checkErrorThrown(next);
   });
-
-  it('trims ips in x-forwarded-for', () => {
-    req.headers['x-forwarded-for'] = '192.168.1.1';
-    sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns(',  192.168.1.1 ,   192.168.1.4, ');
-    const attachIpBlocker = requireAgain(pathToIpBlocker).default;
-    attachIpBlocker(req, res, next);
-
-    checkErrorThrown(next);
-  });
-
-  it('works when multiple ips are passed in x-forwarded-for', () => {
-    req.headers['x-forwarded-for'] = '192.168.1.4';
-    sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.1, 192.168.1.4, 192.168.1.3');
-    const attachIpBlocker = requireAgain(pathToIpBlocker).default;
-    attachIpBlocker(req, res, next);
-
-    checkErrorThrown(next);
-  });
 });

test/api/unit/middlewares/rateLimiter.test.js

@@ -0,0 +1,141 @@
+import nconf from 'nconf';
+import { RateLimiterMemory } from 'rate-limiter-flexible';
+import requireAgain from 'require-again';
+import {
+  generateRes,
+  generateReq,
+  generateNext,
+} from '../../../helpers/api-unit.helper';
+import { TooManyRequests } from '../../../../website/server/libs/errors';
+import apiError from '../../../../website/server/libs/apiError';
+import logger from '../../../../website/server/libs/logger';
+
+describe('rateLimiter middleware', () => {
+  const pathToRateLimiter = '../../../../website/server/middlewares/rateLimiter';
+
+  let res; let req; let next; let nconfGetStub;
+
+  beforeEach(() => {
+    nconfGetStub = sandbox.stub(nconf, 'get');
+
+    nconfGetStub.withArgs('NODE_ENV').returns('test');
+    nconfGetStub.withArgs('IS_TEST').returns(true);
+
+    res = generateRes();
+    req = generateReq();
+    next = generateNext();
+  });
+
+  afterEach(() => {
+    sandbox.restore();
+  });
+
+  it('is disabled when the env var is not defined', () => {
+    nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns(undefined);
+    const attachRateLimiter = requireAgain(pathToRateLimiter).default;
+    attachRateLimiter(req, res, next);
+
+    expect(next).to.have.been.calledOnce;
+    const calledWith = next.getCall(0).args;
+    expect(typeof calledWith[0] === 'undefined').to.equal(true);
+    expect(res.set).to.not.have.been.called;
+  });
+
+  it('is disabled when the env var is an not "true"', () => {
+    nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('false');
+    const attachRateLimiter = requireAgain(pathToRateLimiter).default;
+    attachRateLimiter(req, res, next);
+
+    expect(next).to.have.been.calledOnce;
+    const calledWith = next.getCall(0).args;
+    expect(typeof calledWith[0] === 'undefined').to.equal(true);
+    expect(res.set).to.not.have.been.called;
+  });
+
+  it('does not throw when there are available points', async () => {
+    nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
+    const attachRateLimiter = requireAgain(pathToRateLimiter).default;
+    await attachRateLimiter(req, res, next);
+
+    expect(next).to.have.been.calledOnce;
+    const calledWith = next.getCall(0).args;
+    expect(typeof calledWith[0] === 'undefined').to.equal(true);
+
+    expect(res.set).to.have.been.calledOnce;
+    expect(res.set).to.have.been.calledWithMatch({
+      'X-RateLimit-Limit': 30,
+      'X-RateLimit-Remaining': 29,
+      'X-RateLimit-Reset': sinon.match(Date),
+    });
+  });
+
+  it('does not throw when an unknown error is thrown by the rate limiter', async () => {
+    nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
+    sandbox.stub(logger, 'error');
+    sandbox.stub(RateLimiterMemory.prototype, 'consume')
+      .returns(Promise.reject(new Error('Unknown error.')));
+
+    const attachRateLimiter = requireAgain(pathToRateLimiter).default;
+    await attachRateLimiter(req, res, next);
+
+    expect(next).to.have.been.calledOnce;
+    const calledWith = next.getCall(0).args;
+    expect(typeof calledWith[0] === 'undefined').to.equal(true);
+    expect(res.set).to.not.have.been.called;
+
+    expect(logger.error).to.be.calledOnce;
+    expect(logger.error).to.have.been.calledWithMatch(Error, 'Rate Limiter Error');
+  });
+
+  it('throws when there are no available points remaining', async () => {
+    nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
+    const attachRateLimiter = requireAgain(pathToRateLimiter).default;
+
+    // call for 31 times
+    for (let i = 0; i < 31; i += 1) {
+      await attachRateLimiter(req, res, next); // eslint-disable-line no-await-in-loop
+    }
+
+    expect(next).to.have.been.callCount(31);
+    const calledWith = next.getCall(30).args;
+    expect(calledWith[0].message).to.equal(apiError('clientRateLimited'));
+    expect(calledWith[0] instanceof TooManyRequests).to.equal(true);
+
+    expect(res.set).to.have.been.callCount(31);
+    expect(res.set).to.have.been.calledWithMatch({
+      'Retry-After': sinon.match(Number),
+      'X-RateLimit-Limit': 30,
+      'X-RateLimit-Remaining': 0,
+      'X-RateLimit-Reset': sinon.match(Date),
+    });
+  });
+
+  it('uses the user id if supplied or the ip address', async () => {
+    nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
+    const attachRateLimiter = requireAgain(pathToRateLimiter).default;
+
+    req.ip = 1;
+    await attachRateLimiter(req, res, next);
+
+    req.headers['x-api-user'] = 'user-1';
+    await attachRateLimiter(req, res, next);
+    await attachRateLimiter(req, res, next);
+
+    // user id an ip are counted as separate sources
+    expect(res.set).to.have.been.calledWithMatch({
+      'X-RateLimit-Limit': 30,
+      'X-RateLimit-Remaining': 28, // 2 calls with user id
+      'X-RateLimit-Reset': sinon.match(Date),
+    });
+
+    req.headers['x-api-user'] = undefined;
+    await attachRateLimiter(req, res, next);
+    await attachRateLimiter(req, res, next);
+
+    expect(res.set).to.have.been.calledWithMatch({
+      'X-RateLimit-Limit': 30,
+      'X-RateLimit-Remaining': 27, // 3 calls with only ip
+      'X-RateLimit-Reset': sinon.match(Date),
+    });
+  });
+});

test/api/unit/middlewares/redirects.js

@@ -22,7 +22,7 @@ describe('redirects middleware', () => {
       const nconfStub = sandbox.stub(nconf, 'get');
       nconfStub.withArgs('BASE_URL').returns('https://habitica.com');
       nconfStub.withArgs('IS_PROD').returns(true);
-      req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http');
+      req.protocol = 'http';
       req.originalUrl = '/static/front';
 
       const attachRedirects = requireAgain(pathToRedirectsMiddleware);
@@ -37,7 +37,7 @@ describe('redirects middleware', () => {
       const nconfStub = sandbox.stub(nconf, 'get');
       nconfStub.withArgs('BASE_URL').returns('https://habitica.com');
       nconfStub.withArgs('IS_PROD').returns(true);
-      req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('https');
+      req.protocol = 'https';
       req.originalUrl = '/static/front';
 
       const attachRedirects = requireAgain(pathToRedirectsMiddleware);
@@ -51,7 +51,7 @@ describe('redirects middleware', () => {
       const nconfStub = sandbox.stub(nconf, 'get');
       nconfStub.withArgs('BASE_URL').returns('https://habitica.com');
       nconfStub.withArgs('IS_PROD').returns(false);
-      req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http');
+      req.protocol = 'http';
       req.originalUrl = '/static/front';
 
       const attachRedirects = requireAgain(pathToRedirectsMiddleware);
@@ -65,7 +65,7 @@ describe('redirects middleware', () => {
       const nconfStub = sandbox.stub(nconf, 'get');
       nconfStub.withArgs('BASE_URL').returns('http://habitica.com');
       nconfStub.withArgs('IS_PROD').returns(true);
-      req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http');
+      req.protocol = 'http';
       req.originalUrl = '/static/front';
 
       const attachRedirects = requireAgain(pathToRedirectsMiddleware);
@@ -81,7 +81,7 @@ describe('redirects middleware', () => {
       nconfStub.withArgs('IS_PROD').returns(true);
       nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns('test-key');
 
-      req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http');
+      req.protocol = 'http';
       req.originalUrl = '/static/front';
       req.query.skipSSLCheck = 'test-key';
 
@@ -97,7 +97,7 @@ describe('redirects middleware', () => {
       nconfStub.withArgs('IS_PROD').returns(true);
       nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns('test-key');
 
-      req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http');
+      req.protocol = 'http';
       req.originalUrl = '/static/front?skipSSLCheck=INVALID';
       req.query.skipSSLCheck = 'INVALID';
 
@@ -114,7 +114,7 @@ describe('redirects middleware', () => {
       nconfStub.withArgs('IS_PROD').returns(true);
       nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns(null);
 
-      req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http');
+      req.protocol = 'http';
       req.originalUrl = '/static/front';
       req.query.skipSSLCheck = 'INVALID';