API v3 Rate Limiter (#12117)

* simplify ip address management by using the trust proxy express option

* add setupExpress file

* fix redirects middleware tests

* fix lint

* short circuit the ip blocking middleware

* basic implementation with ip based limiting

* improve logging

* upgrade apidoc

* apidoc: add introduction section

* fix lint

* fix tests

* fix lint

* add unit tests for rate limiter

* do not send retry-after header when points are available

* automatically fix lint

* fix more lint issues

* use userId as key for rate limit when available
This commit is contained in:
Matteo Pagliazzi
2020-07-17 16:13:51 +02:00
parent e3bcc48481
commit e7c8833c9a
15 changed files with 332 additions and 61 deletions

View File

@@ -80,5 +80,9 @@
"APPLE_AUTH_CLIENT_ID": "", "APPLE_AUTH_CLIENT_ID": "",
"APPLE_AUTH_KEY_ID": "", "APPLE_AUTH_KEY_ID": "",
"BLOCKED_IPS": "", "BLOCKED_IPS": "",
"LOG_AMPLITUDE_EVENTS": "false" "LOG_AMPLITUDE_EVENTS": "false",
"RATE_LIMITER_ENABLED": "false",
"REDIS_HOST": "aaabbbcccdddeeefff",
"REDIS_PORT": "1234",
"REDIS_PASSWORD": "12345678"
} }

34
package-lock.json generated
View File

@@ -10926,6 +10926,11 @@
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
"integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
}, },
"rate-limiter-flexible": {
"version": "2.1.9",
"resolved": "https://registry.npmjs.org/rate-limiter-flexible/-/rate-limiter-flexible-2.1.9.tgz",
"integrity": "sha512-ueIXEHLZZqDBetuzyMbtSQ1Gh6Y5rw8ULoNuGA7L3xZ6njPIc2oM0ZlmsY9rS8rPU4yvdw7lk6MLbWU7WTNfnQ=="
},
"raw-body": { "raw-body": {
"version": "2.4.0", "version": "2.4.0",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
@@ -11053,6 +11058,35 @@
"strip-indent": "^1.0.1" "strip-indent": "^1.0.1"
} }
}, },
"redis": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz",
"integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==",
"requires": {
"denque": "^1.4.1",
"redis-commands": "^1.5.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0"
}
},
"redis-commands": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz",
"integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg=="
},
"redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60="
},
"redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
"requires": {
"redis-errors": "^1.0.0"
}
},
"referrer-policy": { "referrer-policy": {
"version": "1.2.0", "version": "1.2.0",
"resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.2.0.tgz", "resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.2.0.tgz",

View File

@@ -60,6 +60,8 @@
"paypal-ipn": "3.0.0", "paypal-ipn": "3.0.0",
"paypal-rest-sdk": "^1.8.1", "paypal-rest-sdk": "^1.8.1",
"ps-tree": "^1.0.0", "ps-tree": "^1.0.0",
"rate-limiter-flexible": "^2.1.7",
"redis": "^3.0.2",
"regenerator-runtime": "^0.13.5", "regenerator-runtime": "^0.13.5",
"remove-markdown": "^0.3.0", "remove-markdown": "^0.3.0",
"rimraf": "^3.0.2", "rimraf": "^3.0.2",

View File

@@ -57,7 +57,7 @@ describe('ipBlocker middleware', () => {
}); });
it('does not throw when the ip does not match', () => { it('does not throw when the ip does not match', () => {
req.headers['x-forwarded-for'] = '192.168.1.1'; req.ip = '192.168.1.1';
sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.2'); sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.2');
const attachIpBlocker = requireAgain(pathToIpBlocker).default; const attachIpBlocker = requireAgain(pathToIpBlocker).default;
attachIpBlocker(req, res, next); attachIpBlocker(req, res, next);
@@ -65,30 +65,12 @@ describe('ipBlocker middleware', () => {
checkErrorNotThrown(next); checkErrorNotThrown(next);
}); });
it('throws when a matching ip exist in x-forwarded-for', () => { it('throws when the ip is blocked', () => {
req.headers['x-forwarded-for'] = '192.168.1.1'; req.ip = '192.168.1.1';
sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.1'); sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.1');
const attachIpBlocker = requireAgain(pathToIpBlocker).default; const attachIpBlocker = requireAgain(pathToIpBlocker).default;
attachIpBlocker(req, res, next); attachIpBlocker(req, res, next);
checkErrorThrown(next); checkErrorThrown(next);
}); });
it('trims ips in x-forwarded-for', () => {
req.headers['x-forwarded-for'] = '192.168.1.1';
sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns(', 192.168.1.1 , 192.168.1.4, ');
const attachIpBlocker = requireAgain(pathToIpBlocker).default;
attachIpBlocker(req, res, next);
checkErrorThrown(next);
});
it('works when multiple ips are passed in x-forwarded-for', () => {
req.headers['x-forwarded-for'] = '192.168.1.4';
sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.1, 192.168.1.4, 192.168.1.3');
const attachIpBlocker = requireAgain(pathToIpBlocker).default;
attachIpBlocker(req, res, next);
checkErrorThrown(next);
});
}); });

View File

@@ -0,0 +1,141 @@
import nconf from 'nconf';
import { RateLimiterMemory } from 'rate-limiter-flexible';
import requireAgain from 'require-again';
import {
generateRes,
generateReq,
generateNext,
} from '../../../helpers/api-unit.helper';
import { TooManyRequests } from '../../../../website/server/libs/errors';
import apiError from '../../../../website/server/libs/apiError';
import logger from '../../../../website/server/libs/logger';
describe('rateLimiter middleware', () => {
const pathToRateLimiter = '../../../../website/server/middlewares/rateLimiter';
let res; let req; let next; let nconfGetStub;
beforeEach(() => {
nconfGetStub = sandbox.stub(nconf, 'get');
nconfGetStub.withArgs('NODE_ENV').returns('test');
nconfGetStub.withArgs('IS_TEST').returns(true);
res = generateRes();
req = generateReq();
next = generateNext();
});
afterEach(() => {
sandbox.restore();
});
it('is disabled when the env var is not defined', () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns(undefined);
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
attachRateLimiter(req, res, next);
expect(next).to.have.been.calledOnce;
const calledWith = next.getCall(0).args;
expect(typeof calledWith[0] === 'undefined').to.equal(true);
expect(res.set).to.not.have.been.called;
});
it('is disabled when the env var is an not "true"', () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('false');
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
attachRateLimiter(req, res, next);
expect(next).to.have.been.calledOnce;
const calledWith = next.getCall(0).args;
expect(typeof calledWith[0] === 'undefined').to.equal(true);
expect(res.set).to.not.have.been.called;
});
it('does not throw when there are available points', async () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
await attachRateLimiter(req, res, next);
expect(next).to.have.been.calledOnce;
const calledWith = next.getCall(0).args;
expect(typeof calledWith[0] === 'undefined').to.equal(true);
expect(res.set).to.have.been.calledOnce;
expect(res.set).to.have.been.calledWithMatch({
'X-RateLimit-Limit': 30,
'X-RateLimit-Remaining': 29,
'X-RateLimit-Reset': sinon.match(Date),
});
});
it('does not throw when an unknown error is thrown by the rate limiter', async () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
sandbox.stub(logger, 'error');
sandbox.stub(RateLimiterMemory.prototype, 'consume')
.returns(Promise.reject(new Error('Unknown error.')));
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
await attachRateLimiter(req, res, next);
expect(next).to.have.been.calledOnce;
const calledWith = next.getCall(0).args;
expect(typeof calledWith[0] === 'undefined').to.equal(true);
expect(res.set).to.not.have.been.called;
expect(logger.error).to.be.calledOnce;
expect(logger.error).to.have.been.calledWithMatch(Error, 'Rate Limiter Error');
});
it('throws when there are no available points remaining', async () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
// call for 31 times
for (let i = 0; i < 31; i += 1) {
await attachRateLimiter(req, res, next); // eslint-disable-line no-await-in-loop
}
expect(next).to.have.been.callCount(31);
const calledWith = next.getCall(30).args;
expect(calledWith[0].message).to.equal(apiError('clientRateLimited'));
expect(calledWith[0] instanceof TooManyRequests).to.equal(true);
expect(res.set).to.have.been.callCount(31);
expect(res.set).to.have.been.calledWithMatch({
'Retry-After': sinon.match(Number),
'X-RateLimit-Limit': 30,
'X-RateLimit-Remaining': 0,
'X-RateLimit-Reset': sinon.match(Date),
});
});
it('uses the user id if supplied or the ip address', async () => {
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
req.ip = 1;
await attachRateLimiter(req, res, next);
req.headers['x-api-user'] = 'user-1';
await attachRateLimiter(req, res, next);
await attachRateLimiter(req, res, next);
// user id an ip are counted as separate sources
expect(res.set).to.have.been.calledWithMatch({
'X-RateLimit-Limit': 30,
'X-RateLimit-Remaining': 28, // 2 calls with user id
'X-RateLimit-Reset': sinon.match(Date),
});
req.headers['x-api-user'] = undefined;
await attachRateLimiter(req, res, next);
await attachRateLimiter(req, res, next);
expect(res.set).to.have.been.calledWithMatch({
'X-RateLimit-Limit': 30,
'X-RateLimit-Remaining': 27, // 3 calls with only ip
'X-RateLimit-Reset': sinon.match(Date),
});
});
});

View File

@@ -22,7 +22,7 @@ describe('redirects middleware', () => {
const nconfStub = sandbox.stub(nconf, 'get'); const nconfStub = sandbox.stub(nconf, 'get');
nconfStub.withArgs('BASE_URL').returns('https://habitica.com'); nconfStub.withArgs('BASE_URL').returns('https://habitica.com');
nconfStub.withArgs('IS_PROD').returns(true); nconfStub.withArgs('IS_PROD').returns(true);
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http'); req.protocol = 'http';
req.originalUrl = '/static/front'; req.originalUrl = '/static/front';
const attachRedirects = requireAgain(pathToRedirectsMiddleware); const attachRedirects = requireAgain(pathToRedirectsMiddleware);
@@ -37,7 +37,7 @@ describe('redirects middleware', () => {
const nconfStub = sandbox.stub(nconf, 'get'); const nconfStub = sandbox.stub(nconf, 'get');
nconfStub.withArgs('BASE_URL').returns('https://habitica.com'); nconfStub.withArgs('BASE_URL').returns('https://habitica.com');
nconfStub.withArgs('IS_PROD').returns(true); nconfStub.withArgs('IS_PROD').returns(true);
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('https'); req.protocol = 'https';
req.originalUrl = '/static/front'; req.originalUrl = '/static/front';
const attachRedirects = requireAgain(pathToRedirectsMiddleware); const attachRedirects = requireAgain(pathToRedirectsMiddleware);
@@ -51,7 +51,7 @@ describe('redirects middleware', () => {
const nconfStub = sandbox.stub(nconf, 'get'); const nconfStub = sandbox.stub(nconf, 'get');
nconfStub.withArgs('BASE_URL').returns('https://habitica.com'); nconfStub.withArgs('BASE_URL').returns('https://habitica.com');
nconfStub.withArgs('IS_PROD').returns(false); nconfStub.withArgs('IS_PROD').returns(false);
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http'); req.protocol = 'http';
req.originalUrl = '/static/front'; req.originalUrl = '/static/front';
const attachRedirects = requireAgain(pathToRedirectsMiddleware); const attachRedirects = requireAgain(pathToRedirectsMiddleware);
@@ -65,7 +65,7 @@ describe('redirects middleware', () => {
const nconfStub = sandbox.stub(nconf, 'get'); const nconfStub = sandbox.stub(nconf, 'get');
nconfStub.withArgs('BASE_URL').returns('http://habitica.com'); nconfStub.withArgs('BASE_URL').returns('http://habitica.com');
nconfStub.withArgs('IS_PROD').returns(true); nconfStub.withArgs('IS_PROD').returns(true);
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http'); req.protocol = 'http';
req.originalUrl = '/static/front'; req.originalUrl = '/static/front';
const attachRedirects = requireAgain(pathToRedirectsMiddleware); const attachRedirects = requireAgain(pathToRedirectsMiddleware);
@@ -81,7 +81,7 @@ describe('redirects middleware', () => {
nconfStub.withArgs('IS_PROD').returns(true); nconfStub.withArgs('IS_PROD').returns(true);
nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns('test-key'); nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns('test-key');
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http'); req.protocol = 'http';
req.originalUrl = '/static/front'; req.originalUrl = '/static/front';
req.query.skipSSLCheck = 'test-key'; req.query.skipSSLCheck = 'test-key';
@@ -97,7 +97,7 @@ describe('redirects middleware', () => {
nconfStub.withArgs('IS_PROD').returns(true); nconfStub.withArgs('IS_PROD').returns(true);
nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns('test-key'); nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns('test-key');
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http'); req.protocol = 'http';
req.originalUrl = '/static/front?skipSSLCheck=INVALID'; req.originalUrl = '/static/front?skipSSLCheck=INVALID';
req.query.skipSSLCheck = 'INVALID'; req.query.skipSSLCheck = 'INVALID';
@@ -114,7 +114,7 @@ describe('redirects middleware', () => {
nconfStub.withArgs('IS_PROD').returns(true); nconfStub.withArgs('IS_PROD').returns(true);
nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns(null); nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns(null);
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http'); req.protocol = 'http';
req.originalUrl = '/static/front'; req.originalUrl = '/static/front';
req.query.skipSSLCheck = 'INVALID'; req.query.skipSSLCheck = 'INVALID';

View File

@@ -27,6 +27,7 @@ export default {
missingSubKey: 'Missing "req.query.sub"', missingSubKey: 'Missing "req.query.sub"',
ipAddressBlocked: 'This IP address has been blocked from accessing Habitica. This may be due to a breach of our Terms of Service or technical issue originating at this IP address. For details or to ask to be unblocked, please email admin@habitica.com or ask your parent or guardian to email them. Include your Habitica @ Username or User Id in the email if you have one.', ipAddressBlocked: 'This IP address has been blocked from accessing Habitica. This may be due to a breach of our Terms of Service or technical issue originating at this IP address. For details or to ask to be unblocked, please email admin@habitica.com or ask your parent or guardian to email them. Include your Habitica @ Username or User Id in the email if you have one.',
clientRateLimited: 'This IP address has been rate limited due to an excess amount of API requests. More info can be found in the response headers.',
invalidPlatform: 'Invalid platform specified', invalidPlatform: 'Invalid platform specified',
}; };

View File

@@ -51,6 +51,15 @@ export class Forbidden extends CustomError {
} }
} }
export class TooManyRequests extends CustomError {
constructor (customMessage) {
super();
this.name = this.constructor.name;
this.httpCode = 429;
this.message = customMessage || 'Too many requests.';
}
}
export class NotImplementedError extends CustomError { export class NotImplementedError extends CustomError {
constructor (str) { constructor (str) {
super(); super();

View File

@@ -54,6 +54,19 @@ export const { NotFound } = common.errors;
*/ */
export const { Forbidden } = common.errors; export const { Forbidden } = common.errors;
/**
* @apiDefine TooManyRequests
* @apiError TooManyRequests The client made too many requests to the API and was rate limited.
*
* @apiErrorExample Error-Response:
* HTTP/1.1 429 TooManyRequests
* {
* "error": "TooManyRequests",
* "message": "Access forbidden."
* }
*/
export const { TooManyRequests } = common.errors;
/** /**
* @apiDefine NotificationNotFound * @apiDefine NotificationNotFound
* @apiError NotificationNotFound The notification was not found. * @apiError NotificationNotFound The notification was not found.

View File

@@ -0,0 +1,11 @@
import nconf from 'nconf';
const IS_PROD = nconf.get('IS_PROD');
export default function setupExpress (app) {
app.set('view engine', 'pug');
app.set('views', `${__dirname}/../../views`);
// The production build of Habitica runs behind a proxy
// See https://expressjs.com/it/guide/behind-proxies.html
if (IS_PROD) app.set('trust proxy', true);
}

View File

@@ -3,6 +3,8 @@ import expressValidator from 'express-validator';
import path from 'path'; import path from 'path';
import analytics from './analytics'; import analytics from './analytics';
import setupBody from './setupBody'; import setupBody from './setupBody';
import rateLimiter from './rateLimiter';
import setupExpress from '../libs/setupExpress';
import * as routes from '../libs/routes'; import * as routes from '../libs/routes';
const API_V3_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/api-v3/'); const API_V3_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/api-v3/');
@@ -12,8 +14,7 @@ const TOP_LEVEL_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/top-lev
const app = express(); const app = express();
// re-set the view options because they are not inherited from the top level app // re-set the view options because they are not inherited from the top level app
app.set('view engine', 'pug'); setupExpress(app);
app.set('views', `${__dirname}/../../views`);
app.use(expressValidator()); app.use(expressValidator());
app.use(analytics); app.use(analytics);
@@ -26,7 +27,7 @@ app.use('/', topLevelRouter);
const v3Router = express.Router(); // eslint-disable-line new-cap const v3Router = express.Router(); // eslint-disable-line new-cap
routes.walkControllers(v3Router, API_V3_CONTROLLERS_PATH); routes.walkControllers(v3Router, API_V3_CONTROLLERS_PATH);
app.use('/api/v3', v3Router); app.use('/api/v3', rateLimiter, v3Router);
// API v4 proxies API v3 routes by default. // API v4 proxies API v3 routes by default.
// It can also disable or override v3 routes // It can also disable or override v3 routes

View File

@@ -9,6 +9,7 @@ import methodOverride from 'method-override';
import passport from 'passport'; import passport from 'passport';
import basicAuth from 'express-basic-auth'; import basicAuth from 'express-basic-auth';
import helmet from 'helmet'; import helmet from 'helmet';
import setupExpress from '../libs/setupExpress';
import errorHandler from './errorHandler'; import errorHandler from './errorHandler';
import notFoundHandler from './notFound'; import notFoundHandler from './notFound';
import cors from './cors'; import cors from './cors';
@@ -39,8 +40,7 @@ const SESSION_SECRET = nconf.get('SESSION_SECRET');
const TEN_YEARS = 1000 * 60 * 60 * 24 * 365 * 10; const TEN_YEARS = 1000 * 60 * 60 * 24 * 365 * 10;
export default function attachMiddlewares (app, server) { export default function attachMiddlewares (app, server) {
app.set('view engine', 'pug'); setupExpress(app);
app.set('views', `${__dirname}/../../views`);
app.use(domainMiddleware(server, mongoose)); app.use(domainMiddleware(server, mongoose));

View File

@@ -26,30 +26,8 @@ export default function ipBlocker (req, res, next) {
// If there are no IPs to block, skip the middleware // If there are no IPs to block, skip the middleware
if (blockedIps.length === 0) return next(); if (blockedIps.length === 0) return next();
// If x-forwarded-for is undefined we're not behind the production proxy // Is the client IP, req.ip, blocked?
const originIpsRaw = req.header('x-forwarded-for'); const match = blockedIps.find(blockedIp => blockedIp === req.ip) !== undefined;
if (!originIpsRaw) return next();
// Format xxx.xxx.xxx.xxx, xxx.xxx.xxx.xxx (comma separated list of ip)
const originIps = originIpsRaw
.split(',')
.map(originIp => originIp.trim());
// We try to match any of the origins IPs against the blocked IPs list.
//
// In case we're behind a Google Cloud Load Balancer the last ip
// in the list is added by the load balancer.
// See https://cloud.google.com/load-balancing/docs/https#target-proxies
// In particular:
// << A Google Cloud external HTTP(S) load balancer adds two IP addresses to the header:
// the IP address of the requesting client and the external IP address of the load balancer's
// forwarding rule, in that order.
// Therefore, the IP address that immediately precedes the Google Cloud load balancer's
// IP address is the IP address of the system that contacts the load balancer.
// The system might be a client, or it might be another proxy server, outside Google Cloud,
// that forwards requests on behalf of a client. >>
const match = originIps.find(originIp => blockedIps.includes(originIp)) !== undefined;
if (match === true) { if (match === true) {
// Not translated because no user is loaded at this point // Not translated because no user is loaded at this point

View File

@@ -0,0 +1,94 @@
import nconf from 'nconf';
import redis from 'redis';
import {
RateLimiterRedis,
RateLimiterMemory,
RateLimiterRes,
} from 'rate-limiter-flexible';
import {
TooManyRequests,
} from '../libs/errors';
import logger from '../libs/logger';
import apiError from '../libs/apiError';
// Middleware to rate limit requests to the API
// More info on the API rate limits can be found on the wiki at
// https://habitica.fandom.com/wiki/Guidance_for_Comrades#Rules_for_Third-Party_Tools
const IS_TEST = nconf.get('IS_TEST');
const RATE_LIMITER_ENABLED = nconf.get('RATE_LIMITER_ENABLED') === 'true';
const REDIS_HOST = nconf.get('REDIS_HOST');
const REDIS_PASSWORD = nconf.get('REDIS_PASSWORD');
const REDIS_PORT = nconf.get('REDIS_PORT');
let redisClient;
let rateLimiter;
const rateLimiterOpts = {
keyPrefix: 'api-v3',
points: 30, // 30 requests
duration: 60, // per 1 minute by User ID or IP
};
if (RATE_LIMITER_ENABLED) {
if (IS_TEST) {
rateLimiter = new RateLimiterMemory({
...rateLimiterOpts,
});
} else {
redisClient = redis.createClient({
host: REDIS_HOST,
password: REDIS_PASSWORD,
port: REDIS_PORT,
enable_offline_queue: false,
});
redisClient.on('error', error => {
logger.error(error, 'Redis Error');
});
rateLimiter = new RateLimiterRedis({
...rateLimiterOpts,
storeClient: redisClient,
});
}
}
function setResponseHeaders (res, rateLimiterRes) {
const headers = {
'X-RateLimit-Limit': rateLimiterOpts.points,
'X-RateLimit-Remaining': rateLimiterRes.remainingPoints,
'X-RateLimit-Reset': new Date(Date.now() + rateLimiterRes.msBeforeNext),
};
if (rateLimiterRes.remainingPoints < 1) {
headers['Retry-After'] = rateLimiterRes.msBeforeNext / 1000;
}
res.set(headers);
}
export default function rateLimiterMiddleware (req, res, next) {
if (!RATE_LIMITER_ENABLED) return next();
const userId = req.header('x-api-user');
return rateLimiter.consume(userId || req.ip)
.then(rateLimiterRes => {
setResponseHeaders(res, rateLimiterRes);
return next();
})
.catch(rateLimiterRes => {
if (rateLimiterRes instanceof RateLimiterRes) {
setResponseHeaders(res, rateLimiterRes);
return next(new TooManyRequests(apiError('clientRateLimited')));
}
// In case of an unhandled error we skip the middleware as it could mean
// , for example, that the connection to the redis database is not working.
// We do not want to block all requests in these cases.
logger.error(rateLimiterRes, 'Rate Limiter Error');
return next();
});
}

View File

@@ -4,6 +4,8 @@ import url from 'url';
const IS_PROD = nconf.get('IS_PROD'); const IS_PROD = nconf.get('IS_PROD');
const IGNORE_REDIRECT = nconf.get('IGNORE_REDIRECT') === 'true'; const IGNORE_REDIRECT = nconf.get('IGNORE_REDIRECT') === 'true';
const BASE_URL = nconf.get('BASE_URL'); const BASE_URL = nconf.get('BASE_URL');
const HTTPS_BASE_URL = BASE_URL.indexOf('https') === 0;
// A secret key that if passed as req.query.skipSSLCheck allows to skip // A secret key that if passed as req.query.skipSSLCheck allows to skip
// the redirects to SSL, used for health checks from the load balancer // the redirects to SSL, used for health checks from the load balancer
const SKIP_SSL_CHECK_KEY = nconf.get('SKIP_SSL_CHECK_KEY'); const SKIP_SSL_CHECK_KEY = nconf.get('SKIP_SSL_CHECK_KEY');
@@ -12,10 +14,9 @@ const BASE_URL_HOST = url.parse(BASE_URL).hostname;
function isHTTP (req) { function isHTTP (req) {
return ( // eslint-disable-line no-extra-parens return ( // eslint-disable-line no-extra-parens
req.header('x-forwarded-proto') req.protocol === 'http'
&& req.header('x-forwarded-proto') === 'http'
&& IS_PROD && IS_PROD
&& BASE_URL.indexOf('https') === 0 && HTTPS_BASE_URL === true
); );
} }