mirror of
https://github.com/HabitRPG/habitica.git
synced 2025-12-14 21:27:23 +01:00
API v3 Rate Limiter (#12117)
* simplify ip address management by using the trust proxy express option * add setupExpress file * fix redirects middleware tests * fix lint * short circuit the ip blocking middleware * basic implementation with ip based limiting * improve logging * upgrade apidoc * apidoc: add introduction section * fix lint * fix tests * fix lint * add unit tests for rate limiter * do not send retry-after header when points are available * automatically fix lint * fix more lint issues * use userId as key for rate limit when available
This commit is contained in:
@@ -80,5 +80,9 @@
|
|||||||
"APPLE_AUTH_CLIENT_ID": "",
|
"APPLE_AUTH_CLIENT_ID": "",
|
||||||
"APPLE_AUTH_KEY_ID": "",
|
"APPLE_AUTH_KEY_ID": "",
|
||||||
"BLOCKED_IPS": "",
|
"BLOCKED_IPS": "",
|
||||||
"LOG_AMPLITUDE_EVENTS": "false"
|
"LOG_AMPLITUDE_EVENTS": "false",
|
||||||
|
"RATE_LIMITER_ENABLED": "false",
|
||||||
|
"REDIS_HOST": "aaabbbcccdddeeefff",
|
||||||
|
"REDIS_PORT": "1234",
|
||||||
|
"REDIS_PASSWORD": "12345678"
|
||||||
}
|
}
|
||||||
|
|||||||
34
package-lock.json
generated
34
package-lock.json
generated
@@ -10926,6 +10926,11 @@
|
|||||||
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
|
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
|
||||||
"integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
|
"integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
|
||||||
},
|
},
|
||||||
|
"rate-limiter-flexible": {
|
||||||
|
"version": "2.1.9",
|
||||||
|
"resolved": "https://registry.npmjs.org/rate-limiter-flexible/-/rate-limiter-flexible-2.1.9.tgz",
|
||||||
|
"integrity": "sha512-ueIXEHLZZqDBetuzyMbtSQ1Gh6Y5rw8ULoNuGA7L3xZ6njPIc2oM0ZlmsY9rS8rPU4yvdw7lk6MLbWU7WTNfnQ=="
|
||||||
|
},
|
||||||
"raw-body": {
|
"raw-body": {
|
||||||
"version": "2.4.0",
|
"version": "2.4.0",
|
||||||
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
|
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
|
||||||
@@ -11053,6 +11058,35 @@
|
|||||||
"strip-indent": "^1.0.1"
|
"strip-indent": "^1.0.1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"redis": {
|
||||||
|
"version": "3.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz",
|
||||||
|
"integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==",
|
||||||
|
"requires": {
|
||||||
|
"denque": "^1.4.1",
|
||||||
|
"redis-commands": "^1.5.0",
|
||||||
|
"redis-errors": "^1.2.0",
|
||||||
|
"redis-parser": "^3.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"redis-commands": {
|
||||||
|
"version": "1.5.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz",
|
||||||
|
"integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg=="
|
||||||
|
},
|
||||||
|
"redis-errors": {
|
||||||
|
"version": "1.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
|
||||||
|
"integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60="
|
||||||
|
},
|
||||||
|
"redis-parser": {
|
||||||
|
"version": "3.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
|
||||||
|
"integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
|
||||||
|
"requires": {
|
||||||
|
"redis-errors": "^1.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"referrer-policy": {
|
"referrer-policy": {
|
||||||
"version": "1.2.0",
|
"version": "1.2.0",
|
||||||
"resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.2.0.tgz",
|
"resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.2.0.tgz",
|
||||||
|
|||||||
@@ -60,6 +60,8 @@
|
|||||||
"paypal-ipn": "3.0.0",
|
"paypal-ipn": "3.0.0",
|
||||||
"paypal-rest-sdk": "^1.8.1",
|
"paypal-rest-sdk": "^1.8.1",
|
||||||
"ps-tree": "^1.0.0",
|
"ps-tree": "^1.0.0",
|
||||||
|
"rate-limiter-flexible": "^2.1.7",
|
||||||
|
"redis": "^3.0.2",
|
||||||
"regenerator-runtime": "^0.13.5",
|
"regenerator-runtime": "^0.13.5",
|
||||||
"remove-markdown": "^0.3.0",
|
"remove-markdown": "^0.3.0",
|
||||||
"rimraf": "^3.0.2",
|
"rimraf": "^3.0.2",
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ describe('ipBlocker middleware', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('does not throw when the ip does not match', () => {
|
it('does not throw when the ip does not match', () => {
|
||||||
req.headers['x-forwarded-for'] = '192.168.1.1';
|
req.ip = '192.168.1.1';
|
||||||
sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.2');
|
sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.2');
|
||||||
const attachIpBlocker = requireAgain(pathToIpBlocker).default;
|
const attachIpBlocker = requireAgain(pathToIpBlocker).default;
|
||||||
attachIpBlocker(req, res, next);
|
attachIpBlocker(req, res, next);
|
||||||
@@ -65,30 +65,12 @@ describe('ipBlocker middleware', () => {
|
|||||||
checkErrorNotThrown(next);
|
checkErrorNotThrown(next);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('throws when a matching ip exist in x-forwarded-for', () => {
|
it('throws when the ip is blocked', () => {
|
||||||
req.headers['x-forwarded-for'] = '192.168.1.1';
|
req.ip = '192.168.1.1';
|
||||||
sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.1');
|
sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.1');
|
||||||
const attachIpBlocker = requireAgain(pathToIpBlocker).default;
|
const attachIpBlocker = requireAgain(pathToIpBlocker).default;
|
||||||
attachIpBlocker(req, res, next);
|
attachIpBlocker(req, res, next);
|
||||||
|
|
||||||
checkErrorThrown(next);
|
checkErrorThrown(next);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('trims ips in x-forwarded-for', () => {
|
|
||||||
req.headers['x-forwarded-for'] = '192.168.1.1';
|
|
||||||
sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns(', 192.168.1.1 , 192.168.1.4, ');
|
|
||||||
const attachIpBlocker = requireAgain(pathToIpBlocker).default;
|
|
||||||
attachIpBlocker(req, res, next);
|
|
||||||
|
|
||||||
checkErrorThrown(next);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('works when multiple ips are passed in x-forwarded-for', () => {
|
|
||||||
req.headers['x-forwarded-for'] = '192.168.1.4';
|
|
||||||
sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.1, 192.168.1.4, 192.168.1.3');
|
|
||||||
const attachIpBlocker = requireAgain(pathToIpBlocker).default;
|
|
||||||
attachIpBlocker(req, res, next);
|
|
||||||
|
|
||||||
checkErrorThrown(next);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|||||||
141
test/api/unit/middlewares/rateLimiter.test.js
Normal file
141
test/api/unit/middlewares/rateLimiter.test.js
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
import nconf from 'nconf';
|
||||||
|
import { RateLimiterMemory } from 'rate-limiter-flexible';
|
||||||
|
import requireAgain from 'require-again';
|
||||||
|
import {
|
||||||
|
generateRes,
|
||||||
|
generateReq,
|
||||||
|
generateNext,
|
||||||
|
} from '../../../helpers/api-unit.helper';
|
||||||
|
import { TooManyRequests } from '../../../../website/server/libs/errors';
|
||||||
|
import apiError from '../../../../website/server/libs/apiError';
|
||||||
|
import logger from '../../../../website/server/libs/logger';
|
||||||
|
|
||||||
|
describe('rateLimiter middleware', () => {
|
||||||
|
const pathToRateLimiter = '../../../../website/server/middlewares/rateLimiter';
|
||||||
|
|
||||||
|
let res; let req; let next; let nconfGetStub;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
nconfGetStub = sandbox.stub(nconf, 'get');
|
||||||
|
|
||||||
|
nconfGetStub.withArgs('NODE_ENV').returns('test');
|
||||||
|
nconfGetStub.withArgs('IS_TEST').returns(true);
|
||||||
|
|
||||||
|
res = generateRes();
|
||||||
|
req = generateReq();
|
||||||
|
next = generateNext();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
sandbox.restore();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is disabled when the env var is not defined', () => {
|
||||||
|
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns(undefined);
|
||||||
|
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
|
||||||
|
attachRateLimiter(req, res, next);
|
||||||
|
|
||||||
|
expect(next).to.have.been.calledOnce;
|
||||||
|
const calledWith = next.getCall(0).args;
|
||||||
|
expect(typeof calledWith[0] === 'undefined').to.equal(true);
|
||||||
|
expect(res.set).to.not.have.been.called;
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is disabled when the env var is an not "true"', () => {
|
||||||
|
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('false');
|
||||||
|
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
|
||||||
|
attachRateLimiter(req, res, next);
|
||||||
|
|
||||||
|
expect(next).to.have.been.calledOnce;
|
||||||
|
const calledWith = next.getCall(0).args;
|
||||||
|
expect(typeof calledWith[0] === 'undefined').to.equal(true);
|
||||||
|
expect(res.set).to.not.have.been.called;
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not throw when there are available points', async () => {
|
||||||
|
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
|
||||||
|
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
|
||||||
|
await attachRateLimiter(req, res, next);
|
||||||
|
|
||||||
|
expect(next).to.have.been.calledOnce;
|
||||||
|
const calledWith = next.getCall(0).args;
|
||||||
|
expect(typeof calledWith[0] === 'undefined').to.equal(true);
|
||||||
|
|
||||||
|
expect(res.set).to.have.been.calledOnce;
|
||||||
|
expect(res.set).to.have.been.calledWithMatch({
|
||||||
|
'X-RateLimit-Limit': 30,
|
||||||
|
'X-RateLimit-Remaining': 29,
|
||||||
|
'X-RateLimit-Reset': sinon.match(Date),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not throw when an unknown error is thrown by the rate limiter', async () => {
|
||||||
|
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
|
||||||
|
sandbox.stub(logger, 'error');
|
||||||
|
sandbox.stub(RateLimiterMemory.prototype, 'consume')
|
||||||
|
.returns(Promise.reject(new Error('Unknown error.')));
|
||||||
|
|
||||||
|
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
|
||||||
|
await attachRateLimiter(req, res, next);
|
||||||
|
|
||||||
|
expect(next).to.have.been.calledOnce;
|
||||||
|
const calledWith = next.getCall(0).args;
|
||||||
|
expect(typeof calledWith[0] === 'undefined').to.equal(true);
|
||||||
|
expect(res.set).to.not.have.been.called;
|
||||||
|
|
||||||
|
expect(logger.error).to.be.calledOnce;
|
||||||
|
expect(logger.error).to.have.been.calledWithMatch(Error, 'Rate Limiter Error');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws when there are no available points remaining', async () => {
|
||||||
|
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
|
||||||
|
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
|
||||||
|
|
||||||
|
// call for 31 times
|
||||||
|
for (let i = 0; i < 31; i += 1) {
|
||||||
|
await attachRateLimiter(req, res, next); // eslint-disable-line no-await-in-loop
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(next).to.have.been.callCount(31);
|
||||||
|
const calledWith = next.getCall(30).args;
|
||||||
|
expect(calledWith[0].message).to.equal(apiError('clientRateLimited'));
|
||||||
|
expect(calledWith[0] instanceof TooManyRequests).to.equal(true);
|
||||||
|
|
||||||
|
expect(res.set).to.have.been.callCount(31);
|
||||||
|
expect(res.set).to.have.been.calledWithMatch({
|
||||||
|
'Retry-After': sinon.match(Number),
|
||||||
|
'X-RateLimit-Limit': 30,
|
||||||
|
'X-RateLimit-Remaining': 0,
|
||||||
|
'X-RateLimit-Reset': sinon.match(Date),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses the user id if supplied or the ip address', async () => {
|
||||||
|
nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true');
|
||||||
|
const attachRateLimiter = requireAgain(pathToRateLimiter).default;
|
||||||
|
|
||||||
|
req.ip = 1;
|
||||||
|
await attachRateLimiter(req, res, next);
|
||||||
|
|
||||||
|
req.headers['x-api-user'] = 'user-1';
|
||||||
|
await attachRateLimiter(req, res, next);
|
||||||
|
await attachRateLimiter(req, res, next);
|
||||||
|
|
||||||
|
// user id an ip are counted as separate sources
|
||||||
|
expect(res.set).to.have.been.calledWithMatch({
|
||||||
|
'X-RateLimit-Limit': 30,
|
||||||
|
'X-RateLimit-Remaining': 28, // 2 calls with user id
|
||||||
|
'X-RateLimit-Reset': sinon.match(Date),
|
||||||
|
});
|
||||||
|
|
||||||
|
req.headers['x-api-user'] = undefined;
|
||||||
|
await attachRateLimiter(req, res, next);
|
||||||
|
await attachRateLimiter(req, res, next);
|
||||||
|
|
||||||
|
expect(res.set).to.have.been.calledWithMatch({
|
||||||
|
'X-RateLimit-Limit': 30,
|
||||||
|
'X-RateLimit-Remaining': 27, // 3 calls with only ip
|
||||||
|
'X-RateLimit-Reset': sinon.match(Date),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -22,7 +22,7 @@ describe('redirects middleware', () => {
|
|||||||
const nconfStub = sandbox.stub(nconf, 'get');
|
const nconfStub = sandbox.stub(nconf, 'get');
|
||||||
nconfStub.withArgs('BASE_URL').returns('https://habitica.com');
|
nconfStub.withArgs('BASE_URL').returns('https://habitica.com');
|
||||||
nconfStub.withArgs('IS_PROD').returns(true);
|
nconfStub.withArgs('IS_PROD').returns(true);
|
||||||
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http');
|
req.protocol = 'http';
|
||||||
req.originalUrl = '/static/front';
|
req.originalUrl = '/static/front';
|
||||||
|
|
||||||
const attachRedirects = requireAgain(pathToRedirectsMiddleware);
|
const attachRedirects = requireAgain(pathToRedirectsMiddleware);
|
||||||
@@ -37,7 +37,7 @@ describe('redirects middleware', () => {
|
|||||||
const nconfStub = sandbox.stub(nconf, 'get');
|
const nconfStub = sandbox.stub(nconf, 'get');
|
||||||
nconfStub.withArgs('BASE_URL').returns('https://habitica.com');
|
nconfStub.withArgs('BASE_URL').returns('https://habitica.com');
|
||||||
nconfStub.withArgs('IS_PROD').returns(true);
|
nconfStub.withArgs('IS_PROD').returns(true);
|
||||||
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('https');
|
req.protocol = 'https';
|
||||||
req.originalUrl = '/static/front';
|
req.originalUrl = '/static/front';
|
||||||
|
|
||||||
const attachRedirects = requireAgain(pathToRedirectsMiddleware);
|
const attachRedirects = requireAgain(pathToRedirectsMiddleware);
|
||||||
@@ -51,7 +51,7 @@ describe('redirects middleware', () => {
|
|||||||
const nconfStub = sandbox.stub(nconf, 'get');
|
const nconfStub = sandbox.stub(nconf, 'get');
|
||||||
nconfStub.withArgs('BASE_URL').returns('https://habitica.com');
|
nconfStub.withArgs('BASE_URL').returns('https://habitica.com');
|
||||||
nconfStub.withArgs('IS_PROD').returns(false);
|
nconfStub.withArgs('IS_PROD').returns(false);
|
||||||
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http');
|
req.protocol = 'http';
|
||||||
req.originalUrl = '/static/front';
|
req.originalUrl = '/static/front';
|
||||||
|
|
||||||
const attachRedirects = requireAgain(pathToRedirectsMiddleware);
|
const attachRedirects = requireAgain(pathToRedirectsMiddleware);
|
||||||
@@ -65,7 +65,7 @@ describe('redirects middleware', () => {
|
|||||||
const nconfStub = sandbox.stub(nconf, 'get');
|
const nconfStub = sandbox.stub(nconf, 'get');
|
||||||
nconfStub.withArgs('BASE_URL').returns('http://habitica.com');
|
nconfStub.withArgs('BASE_URL').returns('http://habitica.com');
|
||||||
nconfStub.withArgs('IS_PROD').returns(true);
|
nconfStub.withArgs('IS_PROD').returns(true);
|
||||||
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http');
|
req.protocol = 'http';
|
||||||
req.originalUrl = '/static/front';
|
req.originalUrl = '/static/front';
|
||||||
|
|
||||||
const attachRedirects = requireAgain(pathToRedirectsMiddleware);
|
const attachRedirects = requireAgain(pathToRedirectsMiddleware);
|
||||||
@@ -81,7 +81,7 @@ describe('redirects middleware', () => {
|
|||||||
nconfStub.withArgs('IS_PROD').returns(true);
|
nconfStub.withArgs('IS_PROD').returns(true);
|
||||||
nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns('test-key');
|
nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns('test-key');
|
||||||
|
|
||||||
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http');
|
req.protocol = 'http';
|
||||||
req.originalUrl = '/static/front';
|
req.originalUrl = '/static/front';
|
||||||
req.query.skipSSLCheck = 'test-key';
|
req.query.skipSSLCheck = 'test-key';
|
||||||
|
|
||||||
@@ -97,7 +97,7 @@ describe('redirects middleware', () => {
|
|||||||
nconfStub.withArgs('IS_PROD').returns(true);
|
nconfStub.withArgs('IS_PROD').returns(true);
|
||||||
nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns('test-key');
|
nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns('test-key');
|
||||||
|
|
||||||
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http');
|
req.protocol = 'http';
|
||||||
req.originalUrl = '/static/front?skipSSLCheck=INVALID';
|
req.originalUrl = '/static/front?skipSSLCheck=INVALID';
|
||||||
req.query.skipSSLCheck = 'INVALID';
|
req.query.skipSSLCheck = 'INVALID';
|
||||||
|
|
||||||
@@ -114,7 +114,7 @@ describe('redirects middleware', () => {
|
|||||||
nconfStub.withArgs('IS_PROD').returns(true);
|
nconfStub.withArgs('IS_PROD').returns(true);
|
||||||
nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns(null);
|
nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns(null);
|
||||||
|
|
||||||
req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http');
|
req.protocol = 'http';
|
||||||
req.originalUrl = '/static/front';
|
req.originalUrl = '/static/front';
|
||||||
req.query.skipSSLCheck = 'INVALID';
|
req.query.skipSSLCheck = 'INVALID';
|
||||||
|
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ export default {
|
|||||||
missingSubKey: 'Missing "req.query.sub"',
|
missingSubKey: 'Missing "req.query.sub"',
|
||||||
|
|
||||||
ipAddressBlocked: 'This IP address has been blocked from accessing Habitica. This may be due to a breach of our Terms of Service or technical issue originating at this IP address. For details or to ask to be unblocked, please email admin@habitica.com or ask your parent or guardian to email them. Include your Habitica @ Username or User Id in the email if you have one.',
|
ipAddressBlocked: 'This IP address has been blocked from accessing Habitica. This may be due to a breach of our Terms of Service or technical issue originating at this IP address. For details or to ask to be unblocked, please email admin@habitica.com or ask your parent or guardian to email them. Include your Habitica @ Username or User Id in the email if you have one.',
|
||||||
|
clientRateLimited: 'This IP address has been rate limited due to an excess amount of API requests. More info can be found in the response headers.',
|
||||||
|
|
||||||
invalidPlatform: 'Invalid platform specified',
|
invalidPlatform: 'Invalid platform specified',
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -51,6 +51,15 @@ export class Forbidden extends CustomError {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export class TooManyRequests extends CustomError {
|
||||||
|
constructor (customMessage) {
|
||||||
|
super();
|
||||||
|
this.name = this.constructor.name;
|
||||||
|
this.httpCode = 429;
|
||||||
|
this.message = customMessage || 'Too many requests.';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export class NotImplementedError extends CustomError {
|
export class NotImplementedError extends CustomError {
|
||||||
constructor (str) {
|
constructor (str) {
|
||||||
super();
|
super();
|
||||||
|
|||||||
@@ -54,6 +54,19 @@ export const { NotFound } = common.errors;
|
|||||||
*/
|
*/
|
||||||
export const { Forbidden } = common.errors;
|
export const { Forbidden } = common.errors;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @apiDefine TooManyRequests
|
||||||
|
* @apiError TooManyRequests The client made too many requests to the API and was rate limited.
|
||||||
|
*
|
||||||
|
* @apiErrorExample Error-Response:
|
||||||
|
* HTTP/1.1 429 TooManyRequests
|
||||||
|
* {
|
||||||
|
* "error": "TooManyRequests",
|
||||||
|
* "message": "Access forbidden."
|
||||||
|
* }
|
||||||
|
*/
|
||||||
|
export const { TooManyRequests } = common.errors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @apiDefine NotificationNotFound
|
* @apiDefine NotificationNotFound
|
||||||
* @apiError NotificationNotFound The notification was not found.
|
* @apiError NotificationNotFound The notification was not found.
|
||||||
|
|||||||
11
website/server/libs/setupExpress.js
Normal file
11
website/server/libs/setupExpress.js
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
import nconf from 'nconf';
|
||||||
|
|
||||||
|
const IS_PROD = nconf.get('IS_PROD');
|
||||||
|
|
||||||
|
export default function setupExpress (app) {
|
||||||
|
app.set('view engine', 'pug');
|
||||||
|
app.set('views', `${__dirname}/../../views`);
|
||||||
|
// The production build of Habitica runs behind a proxy
|
||||||
|
// See https://expressjs.com/it/guide/behind-proxies.html
|
||||||
|
if (IS_PROD) app.set('trust proxy', true);
|
||||||
|
}
|
||||||
@@ -3,6 +3,8 @@ import expressValidator from 'express-validator';
|
|||||||
import path from 'path';
|
import path from 'path';
|
||||||
import analytics from './analytics';
|
import analytics from './analytics';
|
||||||
import setupBody from './setupBody';
|
import setupBody from './setupBody';
|
||||||
|
import rateLimiter from './rateLimiter';
|
||||||
|
import setupExpress from '../libs/setupExpress';
|
||||||
import * as routes from '../libs/routes';
|
import * as routes from '../libs/routes';
|
||||||
|
|
||||||
const API_V3_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/api-v3/');
|
const API_V3_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/api-v3/');
|
||||||
@@ -12,8 +14,7 @@ const TOP_LEVEL_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/top-lev
|
|||||||
const app = express();
|
const app = express();
|
||||||
|
|
||||||
// re-set the view options because they are not inherited from the top level app
|
// re-set the view options because they are not inherited from the top level app
|
||||||
app.set('view engine', 'pug');
|
setupExpress(app);
|
||||||
app.set('views', `${__dirname}/../../views`);
|
|
||||||
|
|
||||||
app.use(expressValidator());
|
app.use(expressValidator());
|
||||||
app.use(analytics);
|
app.use(analytics);
|
||||||
@@ -26,7 +27,7 @@ app.use('/', topLevelRouter);
|
|||||||
|
|
||||||
const v3Router = express.Router(); // eslint-disable-line new-cap
|
const v3Router = express.Router(); // eslint-disable-line new-cap
|
||||||
routes.walkControllers(v3Router, API_V3_CONTROLLERS_PATH);
|
routes.walkControllers(v3Router, API_V3_CONTROLLERS_PATH);
|
||||||
app.use('/api/v3', v3Router);
|
app.use('/api/v3', rateLimiter, v3Router);
|
||||||
|
|
||||||
// API v4 proxies API v3 routes by default.
|
// API v4 proxies API v3 routes by default.
|
||||||
// It can also disable or override v3 routes
|
// It can also disable or override v3 routes
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import methodOverride from 'method-override';
|
|||||||
import passport from 'passport';
|
import passport from 'passport';
|
||||||
import basicAuth from 'express-basic-auth';
|
import basicAuth from 'express-basic-auth';
|
||||||
import helmet from 'helmet';
|
import helmet from 'helmet';
|
||||||
|
import setupExpress from '../libs/setupExpress';
|
||||||
import errorHandler from './errorHandler';
|
import errorHandler from './errorHandler';
|
||||||
import notFoundHandler from './notFound';
|
import notFoundHandler from './notFound';
|
||||||
import cors from './cors';
|
import cors from './cors';
|
||||||
@@ -39,8 +40,7 @@ const SESSION_SECRET = nconf.get('SESSION_SECRET');
|
|||||||
const TEN_YEARS = 1000 * 60 * 60 * 24 * 365 * 10;
|
const TEN_YEARS = 1000 * 60 * 60 * 24 * 365 * 10;
|
||||||
|
|
||||||
export default function attachMiddlewares (app, server) {
|
export default function attachMiddlewares (app, server) {
|
||||||
app.set('view engine', 'pug');
|
setupExpress(app);
|
||||||
app.set('views', `${__dirname}/../../views`);
|
|
||||||
|
|
||||||
app.use(domainMiddleware(server, mongoose));
|
app.use(domainMiddleware(server, mongoose));
|
||||||
|
|
||||||
|
|||||||
@@ -26,30 +26,8 @@ export default function ipBlocker (req, res, next) {
|
|||||||
// If there are no IPs to block, skip the middleware
|
// If there are no IPs to block, skip the middleware
|
||||||
if (blockedIps.length === 0) return next();
|
if (blockedIps.length === 0) return next();
|
||||||
|
|
||||||
// If x-forwarded-for is undefined we're not behind the production proxy
|
// Is the client IP, req.ip, blocked?
|
||||||
const originIpsRaw = req.header('x-forwarded-for');
|
const match = blockedIps.find(blockedIp => blockedIp === req.ip) !== undefined;
|
||||||
if (!originIpsRaw) return next();
|
|
||||||
|
|
||||||
// Format xxx.xxx.xxx.xxx, xxx.xxx.xxx.xxx (comma separated list of ip)
|
|
||||||
const originIps = originIpsRaw
|
|
||||||
.split(',')
|
|
||||||
.map(originIp => originIp.trim());
|
|
||||||
|
|
||||||
// We try to match any of the origins IPs against the blocked IPs list.
|
|
||||||
//
|
|
||||||
// In case we're behind a Google Cloud Load Balancer the last ip
|
|
||||||
// in the list is added by the load balancer.
|
|
||||||
// See https://cloud.google.com/load-balancing/docs/https#target-proxies
|
|
||||||
// In particular:
|
|
||||||
// << A Google Cloud external HTTP(S) load balancer adds two IP addresses to the header:
|
|
||||||
// the IP address of the requesting client and the external IP address of the load balancer's
|
|
||||||
// forwarding rule, in that order.
|
|
||||||
// Therefore, the IP address that immediately precedes the Google Cloud load balancer's
|
|
||||||
// IP address is the IP address of the system that contacts the load balancer.
|
|
||||||
// The system might be a client, or it might be another proxy server, outside Google Cloud,
|
|
||||||
// that forwards requests on behalf of a client. >>
|
|
||||||
|
|
||||||
const match = originIps.find(originIp => blockedIps.includes(originIp)) !== undefined;
|
|
||||||
|
|
||||||
if (match === true) {
|
if (match === true) {
|
||||||
// Not translated because no user is loaded at this point
|
// Not translated because no user is loaded at this point
|
||||||
|
|||||||
94
website/server/middlewares/rateLimiter.js
Normal file
94
website/server/middlewares/rateLimiter.js
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
import nconf from 'nconf';
|
||||||
|
import redis from 'redis';
|
||||||
|
import {
|
||||||
|
RateLimiterRedis,
|
||||||
|
RateLimiterMemory,
|
||||||
|
RateLimiterRes,
|
||||||
|
} from 'rate-limiter-flexible';
|
||||||
|
import {
|
||||||
|
TooManyRequests,
|
||||||
|
} from '../libs/errors';
|
||||||
|
import logger from '../libs/logger';
|
||||||
|
import apiError from '../libs/apiError';
|
||||||
|
|
||||||
|
// Middleware to rate limit requests to the API
|
||||||
|
|
||||||
|
// More info on the API rate limits can be found on the wiki at
|
||||||
|
// https://habitica.fandom.com/wiki/Guidance_for_Comrades#Rules_for_Third-Party_Tools
|
||||||
|
|
||||||
|
const IS_TEST = nconf.get('IS_TEST');
|
||||||
|
const RATE_LIMITER_ENABLED = nconf.get('RATE_LIMITER_ENABLED') === 'true';
|
||||||
|
const REDIS_HOST = nconf.get('REDIS_HOST');
|
||||||
|
const REDIS_PASSWORD = nconf.get('REDIS_PASSWORD');
|
||||||
|
const REDIS_PORT = nconf.get('REDIS_PORT');
|
||||||
|
|
||||||
|
let redisClient;
|
||||||
|
let rateLimiter;
|
||||||
|
|
||||||
|
const rateLimiterOpts = {
|
||||||
|
keyPrefix: 'api-v3',
|
||||||
|
points: 30, // 30 requests
|
||||||
|
duration: 60, // per 1 minute by User ID or IP
|
||||||
|
};
|
||||||
|
|
||||||
|
if (RATE_LIMITER_ENABLED) {
|
||||||
|
if (IS_TEST) {
|
||||||
|
rateLimiter = new RateLimiterMemory({
|
||||||
|
...rateLimiterOpts,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
redisClient = redis.createClient({
|
||||||
|
host: REDIS_HOST,
|
||||||
|
password: REDIS_PASSWORD,
|
||||||
|
port: REDIS_PORT,
|
||||||
|
enable_offline_queue: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
redisClient.on('error', error => {
|
||||||
|
logger.error(error, 'Redis Error');
|
||||||
|
});
|
||||||
|
|
||||||
|
rateLimiter = new RateLimiterRedis({
|
||||||
|
...rateLimiterOpts,
|
||||||
|
storeClient: redisClient,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function setResponseHeaders (res, rateLimiterRes) {
|
||||||
|
const headers = {
|
||||||
|
'X-RateLimit-Limit': rateLimiterOpts.points,
|
||||||
|
'X-RateLimit-Remaining': rateLimiterRes.remainingPoints,
|
||||||
|
'X-RateLimit-Reset': new Date(Date.now() + rateLimiterRes.msBeforeNext),
|
||||||
|
};
|
||||||
|
|
||||||
|
if (rateLimiterRes.remainingPoints < 1) {
|
||||||
|
headers['Retry-After'] = rateLimiterRes.msBeforeNext / 1000;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.set(headers);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function rateLimiterMiddleware (req, res, next) {
|
||||||
|
if (!RATE_LIMITER_ENABLED) return next();
|
||||||
|
|
||||||
|
const userId = req.header('x-api-user');
|
||||||
|
|
||||||
|
return rateLimiter.consume(userId || req.ip)
|
||||||
|
.then(rateLimiterRes => {
|
||||||
|
setResponseHeaders(res, rateLimiterRes);
|
||||||
|
return next();
|
||||||
|
})
|
||||||
|
.catch(rateLimiterRes => {
|
||||||
|
if (rateLimiterRes instanceof RateLimiterRes) {
|
||||||
|
setResponseHeaders(res, rateLimiterRes);
|
||||||
|
return next(new TooManyRequests(apiError('clientRateLimited')));
|
||||||
|
}
|
||||||
|
|
||||||
|
// In case of an unhandled error we skip the middleware as it could mean
|
||||||
|
// , for example, that the connection to the redis database is not working.
|
||||||
|
// We do not want to block all requests in these cases.
|
||||||
|
logger.error(rateLimiterRes, 'Rate Limiter Error');
|
||||||
|
return next();
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -4,6 +4,8 @@ import url from 'url';
|
|||||||
const IS_PROD = nconf.get('IS_PROD');
|
const IS_PROD = nconf.get('IS_PROD');
|
||||||
const IGNORE_REDIRECT = nconf.get('IGNORE_REDIRECT') === 'true';
|
const IGNORE_REDIRECT = nconf.get('IGNORE_REDIRECT') === 'true';
|
||||||
const BASE_URL = nconf.get('BASE_URL');
|
const BASE_URL = nconf.get('BASE_URL');
|
||||||
|
const HTTPS_BASE_URL = BASE_URL.indexOf('https') === 0;
|
||||||
|
|
||||||
// A secret key that if passed as req.query.skipSSLCheck allows to skip
|
// A secret key that if passed as req.query.skipSSLCheck allows to skip
|
||||||
// the redirects to SSL, used for health checks from the load balancer
|
// the redirects to SSL, used for health checks from the load balancer
|
||||||
const SKIP_SSL_CHECK_KEY = nconf.get('SKIP_SSL_CHECK_KEY');
|
const SKIP_SSL_CHECK_KEY = nconf.get('SKIP_SSL_CHECK_KEY');
|
||||||
@@ -12,10 +14,9 @@ const BASE_URL_HOST = url.parse(BASE_URL).hostname;
|
|||||||
|
|
||||||
function isHTTP (req) {
|
function isHTTP (req) {
|
||||||
return ( // eslint-disable-line no-extra-parens
|
return ( // eslint-disable-line no-extra-parens
|
||||||
req.header('x-forwarded-proto')
|
req.protocol === 'http'
|
||||||
&& req.header('x-forwarded-proto') === 'http'
|
|
||||||
&& IS_PROD
|
&& IS_PROD
|
||||||
&& BASE_URL.indexOf('https') === 0
|
&& HTTPS_BASE_URL === true
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user