diff --git a/config.json.example b/config.json.example index 3579a673a2..8d8ff52992 100644 --- a/config.json.example +++ b/config.json.example @@ -80,5 +80,9 @@ "APPLE_AUTH_CLIENT_ID": "", "APPLE_AUTH_KEY_ID": "", "BLOCKED_IPS": "", - "LOG_AMPLITUDE_EVENTS": "false" + "LOG_AMPLITUDE_EVENTS": "false", + "RATE_LIMITER_ENABLED": "false", + "REDIS_HOST": "aaabbbcccdddeeefff", + "REDIS_PORT": "1234", + "REDIS_PASSWORD": "12345678" } diff --git a/package-lock.json b/package-lock.json index 7571c57cc8..9925534498 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,36 +23,35 @@ } }, "@babel/core": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.10.4.tgz", - "integrity": "sha512-3A0tS0HWpy4XujGc7QtOIHTeNwUgWaZc/WuS5YQrfhU67jnVmsD6OGPc1AKHH0LJHQICGncy3+YUjIhVlfDdcA==", + "version": "7.10.5", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.10.5.tgz", + "integrity": "sha512-O34LQooYVDXPl7QWCdW9p4NR+QlzOr7xShPPJz8GsuCU3/8ua/wqTr7gmnxXv+WBESiGU/G5s16i6tUvHkNb+w==", "requires": { "@babel/code-frame": "^7.10.4", - "@babel/generator": "^7.10.4", - "@babel/helper-module-transforms": "^7.10.4", + "@babel/generator": "^7.10.5", + "@babel/helper-module-transforms": "^7.10.5", "@babel/helpers": "^7.10.4", - "@babel/parser": "^7.10.4", + "@babel/parser": "^7.10.5", "@babel/template": "^7.10.4", - "@babel/traverse": "^7.10.4", - "@babel/types": "^7.10.4", + "@babel/traverse": "^7.10.5", + "@babel/types": "^7.10.5", "convert-source-map": "^1.7.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.1", "json5": "^2.1.2", - "lodash": "^4.17.13", + "lodash": "^4.17.19", "resolve": "^1.3.2", "semver": "^5.4.1", "source-map": "^0.5.0" } }, "@babel/generator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.10.4.tgz", - "integrity": "sha512-toLIHUIAgcQygFZRAQcsLQV3CBuX6yOIru1kJk/qqqvcRmZrYe6WavZTSG+bB8MxhnL9YPf+pKQfuiP161q7ng==", + "version": "7.10.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.10.5.tgz", + "integrity": "sha512-3vXxr3FEW7E7lJZiWQ3bM4+v/Vyr9C+hpolQ8BGFr9Y8Ri2tFLWTixmwKBafDujO1WVah4fhZBeU1bieKdghig==", "requires": { - "@babel/types": "^7.10.4", + "@babel/types": "^7.10.5", "jsesc": "^2.5.1", - "lodash": "^4.17.13", "source-map": "^0.5.0" } }, @@ -170,17 +169,17 @@ } }, "@babel/helper-module-transforms": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.10.4.tgz", - "integrity": "sha512-Er2FQX0oa3nV7eM1o0tNCTx7izmQtwAQsIiaLRWtavAAEcskb0XJ5OjJbVrYXWOTr8om921Scabn4/tzlx7j1Q==", + "version": "7.10.5", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.10.5.tgz", + "integrity": "sha512-4P+CWMJ6/j1W915ITJaUkadLObmCRRSC234uctJfn/vHrsLNxsR8dwlcXv9ZhJWzl77awf+mWXSZEKt5t0OnlA==", "requires": { "@babel/helper-module-imports": "^7.10.4", "@babel/helper-replace-supers": "^7.10.4", "@babel/helper-simple-access": "^7.10.4", "@babel/helper-split-export-declaration": "^7.10.4", "@babel/template": "^7.10.4", - "@babel/types": "^7.10.4", - "lodash": "^4.17.13" + "@babel/types": "^7.10.5", + "lodash": "^4.17.19" } }, "@babel/helper-optimise-call-expression": { @@ -293,9 +292,9 @@ } }, "@babel/parser": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.10.4.tgz", - "integrity": "sha512-8jHII4hf+YVDsskTF6WuMB3X4Eh+PsUkC2ljq22so5rHvH+T8BzyL94VOdyFLNR8tBSVXOTbNHOKpR4TfRxVtA==" + "version": "7.10.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.10.5.tgz", + "integrity": "sha512-wfryxy4bE1UivvQKSQDU4/X6dr+i8bctjUjj8Zyt3DQy7NtPizJXT8M52nqpNKL+nq2PW8lxk4ZqLj0fD4B4hQ==" }, "@babel/plugin-proposal-async-generator-functions": { "version": "7.10.4", @@ -845,12 +844,12 @@ } }, "@babel/register": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.10.4.tgz", - "integrity": "sha512-whHmgGiWNVyTVnYTSawtDWhaeYsc+noeU8Rmi+MPnbGhDYmr5QpEDMrQcIA07D2RUv0BlThPcN89XcHCqq/O4g==", + "version": "7.10.5", + "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.10.5.tgz", + "integrity": "sha512-eYHdLv43nyvmPn9bfNfrcC4+iYNwdQ8Pxk1MFJuU/U5LpSYl/PH4dFMazCYZDFVi8ueG3shvO+AQfLrxpYulQw==", "requires": { "find-cache-dir": "^2.0.0", - "lodash": "^4.17.13", + "lodash": "^4.17.19", "make-dir": "^2.1.0", "pirates": "^4.0.0", "source-map-support": "^0.5.16" @@ -875,28 +874,28 @@ } }, "@babel/traverse": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.10.4.tgz", - "integrity": "sha512-aSy7p5THgSYm4YyxNGz6jZpXf+Ok40QF3aA2LyIONkDHpAcJzDUqlCKXv6peqYUs2gmic849C/t2HKw2a2K20Q==", + "version": "7.10.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.10.5.tgz", + "integrity": "sha512-yc/fyv2gUjPqzTz0WHeRJH2pv7jA9kA7mBX2tXl/x5iOE81uaVPuGPtaYk7wmkx4b67mQ7NqI8rmT2pF47KYKQ==", "requires": { "@babel/code-frame": "^7.10.4", - "@babel/generator": "^7.10.4", + "@babel/generator": "^7.10.5", "@babel/helper-function-name": "^7.10.4", "@babel/helper-split-export-declaration": "^7.10.4", - "@babel/parser": "^7.10.4", - "@babel/types": "^7.10.4", + "@babel/parser": "^7.10.5", + "@babel/types": "^7.10.5", "debug": "^4.1.0", "globals": "^11.1.0", - "lodash": "^4.17.13" + "lodash": "^4.17.19" } }, "@babel/types": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.10.4.tgz", - "integrity": "sha512-UTCFOxC3FsFHb7lkRMVvgLzaRVamXuAs2Tz4wajva4WxtVY82eZeaUBtC2Zt95FU9TiznuC0Zk35tsim8jeVpg==", + "version": "7.10.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.10.5.tgz", + "integrity": "sha512-ixV66KWfCI6GKoA/2H9v6bQdbfXEwwpOdQ8cRvb4F+eyvhlaHxWFMQB4+3d9QFJXZsiiiqVrewNV0DFEQpyT4Q==", "requires": { "@babel/helper-validator-identifier": "^7.10.4", - "lodash": "^4.17.13", + "lodash": "^4.17.19", "to-fast-properties": "^2.0.0" } }, @@ -1019,9 +1018,9 @@ } }, "@sindresorhus/is": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-3.0.0.tgz", - "integrity": "sha512-kqA5I6Yun7PBHk8WN9BBP1c7FfN2SrD05GuVSEYPqDb4nerv7HqYfgBfMIKmT/EuejURkJKLZuLyGKGs6WEG9w==" + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", + "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==" }, "@sinonjs/commons": { "version": "1.8.0", @@ -3678,7 +3677,6 @@ "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dev": true, "requires": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -3688,14 +3686,12 @@ "path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, "requires": { "shebang-regex": "^3.0.0" } @@ -3703,14 +3699,12 @@ "shebang-regex": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, "requires": { "isexe": "^2.0.0" } @@ -6073,17 +6067,6 @@ "logalot": "^2.0.0" }, "dependencies": { - "cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "optional": true, - "requires": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - } - }, "execa": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/execa/-/execa-4.0.2.tgz", @@ -6124,30 +6107,6 @@ "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "optional": true - }, - "shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "optional": true, - "requires": { - "shebang-regex": "^3.0.0" - } - }, - "shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "optional": true - }, - "which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "optional": true, - "requires": { - "isexe": "^2.0.0" - } } } }, @@ -6524,9 +6483,9 @@ } }, "got": { - "version": "11.5.0", - "resolved": "https://registry.npmjs.org/got/-/got-11.5.0.tgz", - "integrity": "sha512-vOZEcEaK0b6x11uniY0HcblZObKPRO75Jvz53VKuqGSaKCM/zEt0sj2LGYVdqDYJzO3wYdG+FPQQ1hsgoXy7vQ==", + "version": "11.5.1", + "resolved": "https://registry.npmjs.org/got/-/got-11.5.1.tgz", + "integrity": "sha512-reQEZcEBMTGnujmQ+Wm97mJs/OK6INtO6HmLI+xt3+9CvnRwWjXutUvb2mqr+Ao4Lu05Rx6+udx9sOQAmExMxA==", "requires": { "@sindresorhus/is": "^3.0.0", "@szmarczak/http-timer": "^4.0.5", @@ -6535,16 +6494,16 @@ "cacheable-lookup": "^5.0.3", "cacheable-request": "^7.0.1", "decompress-response": "^6.0.0", - "http2-wrapper": "^1.0.0-beta.4.8", + "http2-wrapper": "^1.0.0-beta.5.0", "lowercase-keys": "^2.0.0", "p-cancelable": "^2.0.0", "responselike": "^2.0.0" }, "dependencies": { "@sindresorhus/is": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.1.tgz", - "integrity": "sha512-/aPsuoj/1Dw/kzhkgz+ES6TxG0zfTMGLwuK2ZG00k/iJzYHTLCE8mVU8EPqEOp/lmxPoq1C1C9RYToRKb2KEfg==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-3.0.0.tgz", + "integrity": "sha512-kqA5I6Yun7PBHk8WN9BBP1c7FfN2SrD05GuVSEYPqDb4nerv7HqYfgBfMIKmT/EuejURkJKLZuLyGKGs6WEG9w==" }, "@szmarczak/http-timer": { "version": "4.0.5", @@ -9262,9 +9221,9 @@ } }, "mongoose": { - "version": "5.9.23", - "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-5.9.23.tgz", - "integrity": "sha512-fMYlMRJz0T6Ax2K2P0jt+kxXd4qaRxyfZCha1YBMczmA2EBlT5SnBlcDyJ4YQa4/z+GoDh06uH090w7BfBcdWg==", + "version": "5.9.24", + "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-5.9.24.tgz", + "integrity": "sha512-uxTLy/ExYmOfKvvbjn1PHbjSJg0SQzff+dW6jbnywtbBcfPRC/3etnG9hPv6KJe/5TFZQGxCyiSezkqa0+iJAQ==", "requires": { "bson": "^1.1.4", "kareem": "2.3.1", @@ -9463,11 +9422,6 @@ "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", "dev": true }, - "native-request": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/native-request/-/native-request-1.0.5.tgz", - "integrity": "sha512-7wU3DvBGAJQxWuMR3F62zrhB7hxNj2DdlC/eBVrCgavc6+ZpFZOqS/PsR7QyUPLMkFk0GvvzoeeOAZGLLnObnA==" - }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -10966,6 +10920,11 @@ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, + "rate-limiter-flexible": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/rate-limiter-flexible/-/rate-limiter-flexible-2.1.9.tgz", + "integrity": "sha512-ueIXEHLZZqDBetuzyMbtSQ1Gh6Y5rw8ULoNuGA7L3xZ6njPIc2oM0ZlmsY9rS8rPU4yvdw7lk6MLbWU7WTNfnQ==" + }, "raw-body": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", @@ -11093,6 +11052,35 @@ "strip-indent": "^1.0.1" } }, + "redis": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz", + "integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==", + "requires": { + "denque": "^1.4.1", + "redis-commands": "^1.5.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0" + } + }, + "redis-commands": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.5.0.tgz", + "integrity": "sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg==" + }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "requires": { + "redis-errors": "^1.0.0" + } + }, "referrer-policy": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.2.0.tgz", diff --git a/package.json b/package.json index 0d065423fd..2527283a32 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,8 @@ "paypal-ipn": "3.0.0", "paypal-rest-sdk": "^1.8.1", "ps-tree": "^1.0.0", + "rate-limiter-flexible": "^2.1.7", + "redis": "^3.0.2", "regenerator-runtime": "^0.13.5", "remove-markdown": "^0.3.0", "rimraf": "^3.0.2", diff --git a/test/api/unit/middlewares/ipBlocker.test.js b/test/api/unit/middlewares/ipBlocker.test.js index c55534e4fe..fb834a7e28 100644 --- a/test/api/unit/middlewares/ipBlocker.test.js +++ b/test/api/unit/middlewares/ipBlocker.test.js @@ -57,7 +57,7 @@ describe('ipBlocker middleware', () => { }); it('does not throw when the ip does not match', () => { - req.headers['x-forwarded-for'] = '192.168.1.1'; + req.ip = '192.168.1.1'; sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.2'); const attachIpBlocker = requireAgain(pathToIpBlocker).default; attachIpBlocker(req, res, next); @@ -65,30 +65,12 @@ describe('ipBlocker middleware', () => { checkErrorNotThrown(next); }); - it('throws when a matching ip exist in x-forwarded-for', () => { - req.headers['x-forwarded-for'] = '192.168.1.1'; + it('throws when the ip is blocked', () => { + req.ip = '192.168.1.1'; sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.1'); const attachIpBlocker = requireAgain(pathToIpBlocker).default; attachIpBlocker(req, res, next); checkErrorThrown(next); }); - - it('trims ips in x-forwarded-for', () => { - req.headers['x-forwarded-for'] = '192.168.1.1'; - sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns(', 192.168.1.1 , 192.168.1.4, '); - const attachIpBlocker = requireAgain(pathToIpBlocker).default; - attachIpBlocker(req, res, next); - - checkErrorThrown(next); - }); - - it('works when multiple ips are passed in x-forwarded-for', () => { - req.headers['x-forwarded-for'] = '192.168.1.4'; - sandbox.stub(nconf, 'get').withArgs('BLOCKED_IPS').returns('192.168.1.1, 192.168.1.4, 192.168.1.3'); - const attachIpBlocker = requireAgain(pathToIpBlocker).default; - attachIpBlocker(req, res, next); - - checkErrorThrown(next); - }); }); diff --git a/test/api/unit/middlewares/rateLimiter.test.js b/test/api/unit/middlewares/rateLimiter.test.js new file mode 100644 index 0000000000..6fae11d140 --- /dev/null +++ b/test/api/unit/middlewares/rateLimiter.test.js @@ -0,0 +1,141 @@ +import nconf from 'nconf'; +import { RateLimiterMemory } from 'rate-limiter-flexible'; +import requireAgain from 'require-again'; +import { + generateRes, + generateReq, + generateNext, +} from '../../../helpers/api-unit.helper'; +import { TooManyRequests } from '../../../../website/server/libs/errors'; +import apiError from '../../../../website/server/libs/apiError'; +import logger from '../../../../website/server/libs/logger'; + +describe('rateLimiter middleware', () => { + const pathToRateLimiter = '../../../../website/server/middlewares/rateLimiter'; + + let res; let req; let next; let nconfGetStub; + + beforeEach(() => { + nconfGetStub = sandbox.stub(nconf, 'get'); + + nconfGetStub.withArgs('NODE_ENV').returns('test'); + nconfGetStub.withArgs('IS_TEST').returns(true); + + res = generateRes(); + req = generateReq(); + next = generateNext(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + it('is disabled when the env var is not defined', () => { + nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns(undefined); + const attachRateLimiter = requireAgain(pathToRateLimiter).default; + attachRateLimiter(req, res, next); + + expect(next).to.have.been.calledOnce; + const calledWith = next.getCall(0).args; + expect(typeof calledWith[0] === 'undefined').to.equal(true); + expect(res.set).to.not.have.been.called; + }); + + it('is disabled when the env var is an not "true"', () => { + nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('false'); + const attachRateLimiter = requireAgain(pathToRateLimiter).default; + attachRateLimiter(req, res, next); + + expect(next).to.have.been.calledOnce; + const calledWith = next.getCall(0).args; + expect(typeof calledWith[0] === 'undefined').to.equal(true); + expect(res.set).to.not.have.been.called; + }); + + it('does not throw when there are available points', async () => { + nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true'); + const attachRateLimiter = requireAgain(pathToRateLimiter).default; + await attachRateLimiter(req, res, next); + + expect(next).to.have.been.calledOnce; + const calledWith = next.getCall(0).args; + expect(typeof calledWith[0] === 'undefined').to.equal(true); + + expect(res.set).to.have.been.calledOnce; + expect(res.set).to.have.been.calledWithMatch({ + 'X-RateLimit-Limit': 30, + 'X-RateLimit-Remaining': 29, + 'X-RateLimit-Reset': sinon.match(Date), + }); + }); + + it('does not throw when an unknown error is thrown by the rate limiter', async () => { + nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true'); + sandbox.stub(logger, 'error'); + sandbox.stub(RateLimiterMemory.prototype, 'consume') + .returns(Promise.reject(new Error('Unknown error.'))); + + const attachRateLimiter = requireAgain(pathToRateLimiter).default; + await attachRateLimiter(req, res, next); + + expect(next).to.have.been.calledOnce; + const calledWith = next.getCall(0).args; + expect(typeof calledWith[0] === 'undefined').to.equal(true); + expect(res.set).to.not.have.been.called; + + expect(logger.error).to.be.calledOnce; + expect(logger.error).to.have.been.calledWithMatch(Error, 'Rate Limiter Error'); + }); + + it('throws when there are no available points remaining', async () => { + nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true'); + const attachRateLimiter = requireAgain(pathToRateLimiter).default; + + // call for 31 times + for (let i = 0; i < 31; i += 1) { + await attachRateLimiter(req, res, next); // eslint-disable-line no-await-in-loop + } + + expect(next).to.have.been.callCount(31); + const calledWith = next.getCall(30).args; + expect(calledWith[0].message).to.equal(apiError('clientRateLimited')); + expect(calledWith[0] instanceof TooManyRequests).to.equal(true); + + expect(res.set).to.have.been.callCount(31); + expect(res.set).to.have.been.calledWithMatch({ + 'Retry-After': sinon.match(Number), + 'X-RateLimit-Limit': 30, + 'X-RateLimit-Remaining': 0, + 'X-RateLimit-Reset': sinon.match(Date), + }); + }); + + it('uses the user id if supplied or the ip address', async () => { + nconfGetStub.withArgs('RATE_LIMITER_ENABLED').returns('true'); + const attachRateLimiter = requireAgain(pathToRateLimiter).default; + + req.ip = 1; + await attachRateLimiter(req, res, next); + + req.headers['x-api-user'] = 'user-1'; + await attachRateLimiter(req, res, next); + await attachRateLimiter(req, res, next); + + // user id an ip are counted as separate sources + expect(res.set).to.have.been.calledWithMatch({ + 'X-RateLimit-Limit': 30, + 'X-RateLimit-Remaining': 28, // 2 calls with user id + 'X-RateLimit-Reset': sinon.match(Date), + }); + + req.headers['x-api-user'] = undefined; + await attachRateLimiter(req, res, next); + await attachRateLimiter(req, res, next); + + expect(res.set).to.have.been.calledWithMatch({ + 'X-RateLimit-Limit': 30, + 'X-RateLimit-Remaining': 27, // 3 calls with only ip + 'X-RateLimit-Reset': sinon.match(Date), + }); + }); +}); diff --git a/test/api/unit/middlewares/redirects.js b/test/api/unit/middlewares/redirects.js index 2399717437..11fb763b78 100644 --- a/test/api/unit/middlewares/redirects.js +++ b/test/api/unit/middlewares/redirects.js @@ -22,7 +22,7 @@ describe('redirects middleware', () => { const nconfStub = sandbox.stub(nconf, 'get'); nconfStub.withArgs('BASE_URL').returns('https://habitica.com'); nconfStub.withArgs('IS_PROD').returns(true); - req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http'); + req.protocol = 'http'; req.originalUrl = '/static/front'; const attachRedirects = requireAgain(pathToRedirectsMiddleware); @@ -37,7 +37,7 @@ describe('redirects middleware', () => { const nconfStub = sandbox.stub(nconf, 'get'); nconfStub.withArgs('BASE_URL').returns('https://habitica.com'); nconfStub.withArgs('IS_PROD').returns(true); - req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('https'); + req.protocol = 'https'; req.originalUrl = '/static/front'; const attachRedirects = requireAgain(pathToRedirectsMiddleware); @@ -51,7 +51,7 @@ describe('redirects middleware', () => { const nconfStub = sandbox.stub(nconf, 'get'); nconfStub.withArgs('BASE_URL').returns('https://habitica.com'); nconfStub.withArgs('IS_PROD').returns(false); - req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http'); + req.protocol = 'http'; req.originalUrl = '/static/front'; const attachRedirects = requireAgain(pathToRedirectsMiddleware); @@ -65,7 +65,7 @@ describe('redirects middleware', () => { const nconfStub = sandbox.stub(nconf, 'get'); nconfStub.withArgs('BASE_URL').returns('http://habitica.com'); nconfStub.withArgs('IS_PROD').returns(true); - req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http'); + req.protocol = 'http'; req.originalUrl = '/static/front'; const attachRedirects = requireAgain(pathToRedirectsMiddleware); @@ -81,7 +81,7 @@ describe('redirects middleware', () => { nconfStub.withArgs('IS_PROD').returns(true); nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns('test-key'); - req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http'); + req.protocol = 'http'; req.originalUrl = '/static/front'; req.query.skipSSLCheck = 'test-key'; @@ -97,7 +97,7 @@ describe('redirects middleware', () => { nconfStub.withArgs('IS_PROD').returns(true); nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns('test-key'); - req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http'); + req.protocol = 'http'; req.originalUrl = '/static/front?skipSSLCheck=INVALID'; req.query.skipSSLCheck = 'INVALID'; @@ -114,7 +114,7 @@ describe('redirects middleware', () => { nconfStub.withArgs('IS_PROD').returns(true); nconfStub.withArgs('SKIP_SSL_CHECK_KEY').returns(null); - req.header = sandbox.stub().withArgs('x-forwarded-proto').returns('http'); + req.protocol = 'http'; req.originalUrl = '/static/front'; req.query.skipSSLCheck = 'INVALID'; diff --git a/website/common/script/errors/apiErrorMessages.js b/website/common/script/errors/apiErrorMessages.js index 77991b3f9b..36a304d05d 100644 --- a/website/common/script/errors/apiErrorMessages.js +++ b/website/common/script/errors/apiErrorMessages.js @@ -27,6 +27,7 @@ export default { missingSubKey: 'Missing "req.query.sub"', ipAddressBlocked: 'This IP address has been blocked from accessing Habitica. This may be due to a breach of our Terms of Service or technical issue originating at this IP address. For details or to ask to be unblocked, please email admin@habitica.com or ask your parent or guardian to email them. Include your Habitica @ Username or User Id in the email if you have one.', + clientRateLimited: 'This IP address has been rate limited due to an excess amount of API requests. More info can be found in the response headers.', invalidPlatform: 'Invalid platform specified', }; diff --git a/website/common/script/libs/errors.js b/website/common/script/libs/errors.js index 00d917d16b..c0e1bee9df 100644 --- a/website/common/script/libs/errors.js +++ b/website/common/script/libs/errors.js @@ -51,6 +51,15 @@ export class Forbidden extends CustomError { } } +export class TooManyRequests extends CustomError { + constructor (customMessage) { + super(); + this.name = this.constructor.name; + this.httpCode = 429; + this.message = customMessage || 'Too many requests.'; + } +} + export class NotImplementedError extends CustomError { constructor (str) { super(); diff --git a/website/server/libs/errors.js b/website/server/libs/errors.js index 872a0ec3d2..be85b4f22b 100644 --- a/website/server/libs/errors.js +++ b/website/server/libs/errors.js @@ -54,6 +54,19 @@ export const { NotFound } = common.errors; */ export const { Forbidden } = common.errors; +/** + * @apiDefine TooManyRequests + * @apiError TooManyRequests The client made too many requests to the API and was rate limited. + * + * @apiErrorExample Error-Response: + * HTTP/1.1 429 TooManyRequests + * { + * "error": "TooManyRequests", + * "message": "Access forbidden." + * } + */ +export const { TooManyRequests } = common.errors; + /** * @apiDefine NotificationNotFound * @apiError NotificationNotFound The notification was not found. diff --git a/website/server/libs/setupExpress.js b/website/server/libs/setupExpress.js new file mode 100644 index 0000000000..7715a71025 --- /dev/null +++ b/website/server/libs/setupExpress.js @@ -0,0 +1,11 @@ +import nconf from 'nconf'; + +const IS_PROD = nconf.get('IS_PROD'); + +export default function setupExpress (app) { + app.set('view engine', 'pug'); + app.set('views', `${__dirname}/../../views`); + // The production build of Habitica runs behind a proxy + // See https://expressjs.com/it/guide/behind-proxies.html + if (IS_PROD) app.set('trust proxy', true); +} diff --git a/website/server/middlewares/appRoutes.js b/website/server/middlewares/appRoutes.js index 179ed0082d..646985c3a5 100644 --- a/website/server/middlewares/appRoutes.js +++ b/website/server/middlewares/appRoutes.js @@ -3,6 +3,8 @@ import expressValidator from 'express-validator'; import path from 'path'; import analytics from './analytics'; import setupBody from './setupBody'; +import rateLimiter from './rateLimiter'; +import setupExpress from '../libs/setupExpress'; import * as routes from '../libs/routes'; const API_V3_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/api-v3/'); @@ -12,8 +14,7 @@ const TOP_LEVEL_CONTROLLERS_PATH = path.join(__dirname, '/../controllers/top-lev const app = express(); // re-set the view options because they are not inherited from the top level app -app.set('view engine', 'pug'); -app.set('views', `${__dirname}/../../views`); +setupExpress(app); app.use(expressValidator()); app.use(analytics); @@ -26,7 +27,7 @@ app.use('/', topLevelRouter); const v3Router = express.Router(); // eslint-disable-line new-cap routes.walkControllers(v3Router, API_V3_CONTROLLERS_PATH); -app.use('/api/v3', v3Router); +app.use('/api/v3', rateLimiter, v3Router); // API v4 proxies API v3 routes by default. // It can also disable or override v3 routes diff --git a/website/server/middlewares/index.js b/website/server/middlewares/index.js index e392dda8b5..1e46056a82 100644 --- a/website/server/middlewares/index.js +++ b/website/server/middlewares/index.js @@ -9,6 +9,7 @@ import methodOverride from 'method-override'; import passport from 'passport'; import basicAuth from 'express-basic-auth'; import helmet from 'helmet'; +import setupExpress from '../libs/setupExpress'; import errorHandler from './errorHandler'; import notFoundHandler from './notFound'; import cors from './cors'; @@ -39,8 +40,7 @@ const SESSION_SECRET = nconf.get('SESSION_SECRET'); const TEN_YEARS = 1000 * 60 * 60 * 24 * 365 * 10; export default function attachMiddlewares (app, server) { - app.set('view engine', 'pug'); - app.set('views', `${__dirname}/../../views`); + setupExpress(app); app.use(domainMiddleware(server, mongoose)); diff --git a/website/server/middlewares/ipBlocker.js b/website/server/middlewares/ipBlocker.js index 69c2a48c1a..c0b44aa3b5 100644 --- a/website/server/middlewares/ipBlocker.js +++ b/website/server/middlewares/ipBlocker.js @@ -26,30 +26,8 @@ export default function ipBlocker (req, res, next) { // If there are no IPs to block, skip the middleware if (blockedIps.length === 0) return next(); - // If x-forwarded-for is undefined we're not behind the production proxy - const originIpsRaw = req.header('x-forwarded-for'); - if (!originIpsRaw) return next(); - - // Format xxx.xxx.xxx.xxx, xxx.xxx.xxx.xxx (comma separated list of ip) - const originIps = originIpsRaw - .split(',') - .map(originIp => originIp.trim()); - - // We try to match any of the origins IPs against the blocked IPs list. - // - // In case we're behind a Google Cloud Load Balancer the last ip - // in the list is added by the load balancer. - // See https://cloud.google.com/load-balancing/docs/https#target-proxies - // In particular: - // << A Google Cloud external HTTP(S) load balancer adds two IP addresses to the header: - // the IP address of the requesting client and the external IP address of the load balancer's - // forwarding rule, in that order. - // Therefore, the IP address that immediately precedes the Google Cloud load balancer's - // IP address is the IP address of the system that contacts the load balancer. - // The system might be a client, or it might be another proxy server, outside Google Cloud, - // that forwards requests on behalf of a client. >> - - const match = originIps.find(originIp => blockedIps.includes(originIp)) !== undefined; + // Is the client IP, req.ip, blocked? + const match = blockedIps.find(blockedIp => blockedIp === req.ip) !== undefined; if (match === true) { // Not translated because no user is loaded at this point diff --git a/website/server/middlewares/rateLimiter.js b/website/server/middlewares/rateLimiter.js new file mode 100644 index 0000000000..806cf0325d --- /dev/null +++ b/website/server/middlewares/rateLimiter.js @@ -0,0 +1,94 @@ +import nconf from 'nconf'; +import redis from 'redis'; +import { + RateLimiterRedis, + RateLimiterMemory, + RateLimiterRes, +} from 'rate-limiter-flexible'; +import { + TooManyRequests, +} from '../libs/errors'; +import logger from '../libs/logger'; +import apiError from '../libs/apiError'; + +// Middleware to rate limit requests to the API + +// More info on the API rate limits can be found on the wiki at +// https://habitica.fandom.com/wiki/Guidance_for_Comrades#Rules_for_Third-Party_Tools + +const IS_TEST = nconf.get('IS_TEST'); +const RATE_LIMITER_ENABLED = nconf.get('RATE_LIMITER_ENABLED') === 'true'; +const REDIS_HOST = nconf.get('REDIS_HOST'); +const REDIS_PASSWORD = nconf.get('REDIS_PASSWORD'); +const REDIS_PORT = nconf.get('REDIS_PORT'); + +let redisClient; +let rateLimiter; + +const rateLimiterOpts = { + keyPrefix: 'api-v3', + points: 30, // 30 requests + duration: 60, // per 1 minute by User ID or IP +}; + +if (RATE_LIMITER_ENABLED) { + if (IS_TEST) { + rateLimiter = new RateLimiterMemory({ + ...rateLimiterOpts, + }); + } else { + redisClient = redis.createClient({ + host: REDIS_HOST, + password: REDIS_PASSWORD, + port: REDIS_PORT, + enable_offline_queue: false, + }); + + redisClient.on('error', error => { + logger.error(error, 'Redis Error'); + }); + + rateLimiter = new RateLimiterRedis({ + ...rateLimiterOpts, + storeClient: redisClient, + }); + } +} + +function setResponseHeaders (res, rateLimiterRes) { + const headers = { + 'X-RateLimit-Limit': rateLimiterOpts.points, + 'X-RateLimit-Remaining': rateLimiterRes.remainingPoints, + 'X-RateLimit-Reset': new Date(Date.now() + rateLimiterRes.msBeforeNext), + }; + + if (rateLimiterRes.remainingPoints < 1) { + headers['Retry-After'] = rateLimiterRes.msBeforeNext / 1000; + } + + res.set(headers); +} + +export default function rateLimiterMiddleware (req, res, next) { + if (!RATE_LIMITER_ENABLED) return next(); + + const userId = req.header('x-api-user'); + + return rateLimiter.consume(userId || req.ip) + .then(rateLimiterRes => { + setResponseHeaders(res, rateLimiterRes); + return next(); + }) + .catch(rateLimiterRes => { + if (rateLimiterRes instanceof RateLimiterRes) { + setResponseHeaders(res, rateLimiterRes); + return next(new TooManyRequests(apiError('clientRateLimited'))); + } + + // In case of an unhandled error we skip the middleware as it could mean + // , for example, that the connection to the redis database is not working. + // We do not want to block all requests in these cases. + logger.error(rateLimiterRes, 'Rate Limiter Error'); + return next(); + }); +} diff --git a/website/server/middlewares/redirects.js b/website/server/middlewares/redirects.js index fd10d53d82..42a88a0b3a 100644 --- a/website/server/middlewares/redirects.js +++ b/website/server/middlewares/redirects.js @@ -4,6 +4,8 @@ import url from 'url'; const IS_PROD = nconf.get('IS_PROD'); const IGNORE_REDIRECT = nconf.get('IGNORE_REDIRECT') === 'true'; const BASE_URL = nconf.get('BASE_URL'); +const HTTPS_BASE_URL = BASE_URL.indexOf('https') === 0; + // A secret key that if passed as req.query.skipSSLCheck allows to skip // the redirects to SSL, used for health checks from the load balancer const SKIP_SSL_CHECK_KEY = nconf.get('SKIP_SSL_CHECK_KEY'); @@ -12,10 +14,9 @@ const BASE_URL_HOST = url.parse(BASE_URL).hostname; function isHTTP (req) { return ( // eslint-disable-line no-extra-parens - req.header('x-forwarded-proto') - && req.header('x-forwarded-proto') === 'http' + req.protocol === 'http' && IS_PROD - && BASE_URL.indexOf('https') === 0 + && HTTPS_BASE_URL === true ); }