MongoDB Transactions (#12335)

* add run-rs to dependencies

* wip: add replica set to api unit github action

* wip: add replica set to api unit github action

* wip: fix gh actions mongodb replica set setting

* usa replica set for integration tests

* add correct mongodb version matrix for integration tests

* use different db connection on gh actions

* Revert "use different db connection on gh actions"

This reverts commit aa8db759d3.

* add example transaction

* add mongo script to package.json

* abstract mongodb utils, connect using hostname on windows

* npm scripts: mongo -> mongo:dev

* add setup script for run-rs on windows

* gh actions: run in test environment

* remove test files

* better error handling, use cross-spawn to avoid issues on windows

* fix lint

.github/workflows/test.yml

@@ -22,6 +22,7 @@ jobs:
         npm ci
       env:
         CI: true
+        NODE_ENV: test
     - run: npm run lint-no-fix
   apidoc:
     runs-on: ubuntu-latest
@@ -42,6 +43,7 @@ jobs:
         npm ci
       env:
         CI: true
+        NODE_ENV: test
     - run: npm run apidoc
   sanity:
     runs-on: ubuntu-latest
@@ -62,6 +64,7 @@ jobs:
         npm ci
       env:
         CI: true
+        NODE_ENV: test
     - run: npm run test:sanity
   
   common:
@@ -83,6 +86,7 @@ jobs:
         npm ci
       env:
         CI: true
+        NODE_ENV: test
     - run: npm run test:common
   content:
     runs-on: ubuntu-latest
@@ -103,6 +107,7 @@ jobs:
         npm ci
       env:
         CI: true
+        NODE_ENV: test
     - run: npm run test:content
   
   api-unit:
@@ -110,6 +115,7 @@ jobs:
     strategy:
       matrix:
         node-version: [12.x]
+        mongodb-version: [4.2]
     steps:
     - uses: actions/checkout@v1
       with:
@@ -118,13 +124,18 @@ jobs:
       uses: actions/setup-node@v1
       with:
         node-version: ${{ matrix.node-version }}
-    - run: sudo docker run --name mongo -d -p 27017:27017 mongo:4.2
+    - name: Start MongoDB ${{ matrix.mongodb-version }} Replica Set
+      uses: supercharge/mongodb-github-action@1.3.0
+      with:
+        mongodb-version: ${{ matrix.mongodb-version }}
+        mongodb-replica-set: rs
     - run: cp config.json.example config.json
     - name: npm install
       run: |
         npm ci
       env:
         CI: true
+        NODE_ENV: test
     - run: npm run test:api:unit
       env:
         REQUIRES_SERVER=true: true
@@ -133,6 +144,7 @@ jobs:
     strategy:
       matrix:
         node-version: [12.x]
+        mongodb-version: [4.2]
     steps:
     - uses: actions/checkout@v1
       with:
@@ -141,13 +153,18 @@ jobs:
       uses: actions/setup-node@v1
       with:
         node-version: ${{ matrix.node-version }}
-    - run: sudo docker run --name mongo -d -p 27017:27017 mongo:4.2
+    - name: Start MongoDB ${{ matrix.mongodb-version }} Replica Set
+      uses: supercharge/mongodb-github-action@1.3.0
+      with:
+        mongodb-version: ${{ matrix.mongodb-version }}
+        mongodb-replica-set: rs
     - run: cp config.json.example config.json
     - name: npm install
       run: |
         npm ci
       env:
         CI: true
+        NODE_ENV: test
     - run: npm run test:api-v3:integration
       env:
         REQUIRES_SERVER=true: true
@@ -156,6 +173,7 @@ jobs:
     strategy:
       matrix:
         node-version: [12.x]
+        mongodb-version: [4.2]
     steps:
     - uses: actions/checkout@v1
       with:
@@ -164,13 +182,18 @@ jobs:
       uses: actions/setup-node@v1
       with:
         node-version: ${{ matrix.node-version }}
-    - run: sudo docker run --name mongo -d -p 27017:27017 mongo:4.2
+    - name: Start MongoDB ${{ matrix.mongodb-version }} Replica Set
+      uses: supercharge/mongodb-github-action@1.3.0
+      with:
+        mongodb-version: ${{ matrix.mongodb-version }}
+        mongodb-replica-set: rs
     - run: cp config.json.example config.json
     - name: npm install
       run: |
         npm ci
       env:
         CI: true
+        NODE_ENV: test
     - run: npm run test:api-v4:integration
       env:
         REQUIRES_SERVER=true: true
@@ -194,5 +217,6 @@ jobs:
         npm ci
       env:
         CI: true
+        NODE_ENV: test
     - run: npm run test:unit
       working-directory: ./website/client

.gitignore

@@ -42,3 +42,7 @@ yarn.lock
 
 # webstorm fake webpack for path intellisense
 webpack.webstorm.config
+
+# mongodb replica set for local dev
+mongodb-*.tgz
+/mongodb-data

config.json.example

@@ -32,7 +32,8 @@
   "LOGGLY_SUBDOMAIN": "example-subdomain",
   "LOGGLY_TOKEN": "example-token",
   "MAINTENANCE_MODE": "false",
-  "NODE_DB_URI": "mongodb://localhost:27017/habitrpg",
+  "NODE_DB_URI": "mongodb://localhost:27017/habitica-dev?replicaSet=rs",
+  "TEST_DB_URI": "mongodb://localhost:27017/habitica-test?replicaSet=rs",
   "MONGODB_POOL_SIZE": "10",
   "NODE_ENV": "development",
   "PATH": "bin:node_modules/.bin:/usr/local/bin:/usr/bin:/bin",
@@ -70,7 +71,6 @@
   "SLACK_URL": "https://hooks.slack.com/services/some-url",
   "STRIPE_API_KEY": "aaaabbbbccccddddeeeeffff00001111",
   "STRIPE_PUB_KEY": "22223333444455556666777788889999",
-  "TEST_DB_URI": "mongodb://localhost:27017/habitrpg_test",
   "TRANSIFEX_SLACK_CHANNEL": "transifex",
   "WEB_CONCURRENCY": 1,
   "SKIP_SSL_CHECK_KEY": "key",

gulp/gulp-build.js

@@ -1,5 +1,10 @@
 import gulp from 'gulp';
+import path from 'path';
 import babel from 'gulp-babel';
+import os from 'os';
+import fs from 'fs';
+import spawn from 'cross-spawn'; // eslint-disable-line import/no-extraneous-dependencies
+import clean from 'rimraf';
 
 gulp.task('build:babel:server', () => gulp.src('website/server/**/*.js')
   .pipe(babel())
@@ -24,10 +29,67 @@ gulp.task('build:prod', gulp.series(
   done => done(),
 ));
 
+// Due to this issue https://github.com/vkarpov15/run-rs/issues/45
+// When used on windows `run-rs` must first be run without the `--keep` option
+// in order to be setup correctly, afterwards it can be used.
+
+const MONGO_PATH = path.join(__dirname, '/../mongodb-data/');
+
+gulp.task('build:prepare-mongo', async () => {
+  if (fs.existsSync(MONGO_PATH)) {
+    // console.log('MongoDB data folder exists, skipping setup.');
+    return;
+  }
+
+  if (os.platform() !== 'win32') {
+    // console.log('Not on Windows, skipping MongoDB setup.');
+    return;
+  }
+
+  console.log('MongoDB data folder is missing, setting up.');
+
+  // use run-rs without --keep, kill it as soon as the replica set starts
+  const runRsProcess = spawn('run-rs', ['-v', '4.2.8', '-l', 'ubuntu1804', '--dbpath', 'mongodb-data', '--number', '1', '--quiet']);
+
+  for await (const chunk of runRsProcess.stdout) {
+    const stringChunk = chunk.toString();
+    console.log(stringChunk);
+    // kills the process after the replica set is setup
+    if (stringChunk.includes('Started replica set')) {
+      console.log('MongoDB setup correctly.');
+      runRsProcess.kill();
+    }
+  }
+
+  let error = '';
+  for await (const chunk of runRsProcess.stderr) {
+    const stringChunk = chunk.toString();
+    error += stringChunk;
+  }
+
+  const exitCode = await new Promise(resolve => {
+    runRsProcess.on('close', resolve);
+  });
+
+  if (exitCode || error.length > 0) {
+    // remove any leftover files
+    clean.sync(MONGO_PATH);
+
+    throw new Error(`Error running run-rs: ${error}`);
+  }
+});
+
+gulp.task('build:dev', gulp.series(
+  'build:prepare-mongo',
+  done => done(),
+));
+
 const buildArgs = [];
 
 if (process.env.NODE_ENV === 'production') { // eslint-disable-line no-process-env
   buildArgs.push('build:prod');
+} else if (process.env.NODE_ENV !== 'test') { // eslint-disable-line no-process-env
+  buildArgs.push('build:dev');
 }
 
 gulp.task('build', gulp.series(buildArgs, done => {

gulp/gulp-console.js

@@ -3,6 +3,10 @@ import nconf from 'nconf';
 import repl from 'repl';
 import gulp from 'gulp';
 import logger from '../website/server/libs/logger';
+import {
+  getDevelopmentConnectionUrl,
+  getDefaultConnectionOptions,
+} from '../website/server/libs/mongodb';
 
 // Add additional properties to the repl's context
 const improveRepl = context => {
@@ -26,13 +30,14 @@ const improveRepl = context => {
   context.Group = require('../website/server/models/group').model; // eslint-disable-line global-require
   context.User = require('../website/server/models/user').model; // eslint-disable-line global-require
 
-  const isProd = nconf.get('NODE_ENV') === 'production';
+  const IS_PROD = nconf.get('NODE_ENV') === 'production';
-  const mongooseOptions = !isProd ? {} : {
+  const NODE_DB_URI = nconf.get('NODE_DB_URI');
-    keepAlive: 1,
+
-    connectTimeoutMS: 30000,
+  const mongooseOptions = getDefaultConnectionOptions();
-  };
+  const connectionUrl = IS_PROD ? NODE_DB_URI : getDevelopmentConnectionUrl(NODE_DB_URI);
+
   mongoose.connect(
-    nconf.get('NODE_DB_URI'),
+    connectionUrl,
     mongooseOptions,
     err => {
       if (err) throw err;

gulp/gulp-tests.js

@@ -6,6 +6,10 @@ import nconf from 'nconf';
 import {
   pipe,
 } from './taskHelper';
+import {
+  getDevelopmentConnectionUrl,
+  getDefaultConnectionOptions,
+} from '../website/server/libs/mongodb';
 
 // TODO rewrite
 
@@ -44,7 +48,10 @@ gulp.task('test:nodemon', gulp.series(done => {
 }, 'nodemon'));
 
 gulp.task('test:prepare:mongo', cb => {
-  mongoose.connect(TEST_DB_URI, err => {
+  const mongooseOptions = getDefaultConnectionOptions();
+  const connectionUrl = getDevelopmentConnectionUrl(TEST_DB_URI);
+
+  mongoose.connect(connectionUrl, mongooseOptions, err => {
     if (err) return cb(`Unable to connect to mongo database. Are you sure it's running? \n\n${err}`);
     return mongoose.connection.dropDatabase(err2 => {
       if (err2) return cb(err2);

package.json

@@ -102,6 +102,7 @@
     "client:unit": "cd website/client && npm run test:unit",
     "start": "gulp nodemon",
     "debug": "gulp nodemon --inspect",
+    "mongo:dev": "run-rs -v 4.2.8 -l ubuntu1804 --keep --dbpath mongodb-data --number 1 --quiet",
     "postinstall": "gulp build && cd website/client && npm install",
     "apidoc": "gulp apidoc"
   },
@@ -110,11 +111,13 @@
     "chai": "^4.1.2",
     "chai-as-promised": "^7.1.1",
     "chalk": "^4.1.0",
+    "cross-spawn": "^7.0.3",
     "expect.js": "^0.3.1",
     "istanbul": "^1.1.0-alpha.1",
     "mocha": "^5.1.1",
     "monk": "^7.3.0",
     "require-again": "^2.0.0",
+    "run-rs": "^0.6.2",
     "sinon": "^9.0.2",
     "sinon-chai": "^3.5.0",
     "sinon-stub-promise": "^4.0.0"

test/api/unit/libs/mongodb.js

@@ -0,0 +1,50 @@
+import os from 'os';
+import nconf from 'nconf';
+import requireAgain from 'require-again';
+
+const pathToMongoLib = '../../../../website/server/libs/mongodb';
+
+describe('mongodb', () => {
+  afterEach(() => {
+    sandbox.restore();
+  });
+
+  describe('getDevelopmentConnectionUrl', () => {
+    it('returns the original connection url if not on windows', () => {
+      sandbox.stub(os, 'platform').returns('linux');
+      const mongoLibOverride = requireAgain(pathToMongoLib);
+
+      const originalString = 'mongodb://localhost:3030';
+      const string = mongoLibOverride.getDevelopmentConnectionUrl(originalString);
+      expect(string).to.equal(originalString);
+    });
+
+    it('replaces localhost with hostname on windows', () => {
+      sandbox.stub(os, 'platform').returns('win32');
+      sandbox.stub(os, 'hostname').returns('hostname');
+      const mongoLibOverride = requireAgain(pathToMongoLib);
+
+      const originalString = 'mongodb://localhost:3030';
+      const string = mongoLibOverride.getDevelopmentConnectionUrl(originalString);
+      expect(string).to.equal('mongodb://hostname:3030');
+    });
+  });
+
+  describe('getDefaultConnectionOptions', () => {
+    it('returns development config when IS_PROD is false', () => {
+      sandbox.stub(nconf, 'get').withArgs('IS_PROD').returns(false);
+      const mongoLibOverride = requireAgain(pathToMongoLib);
+
+      const options = mongoLibOverride.getDefaultConnectionOptions();
+      expect(options).to.have.all.keys(['useNewUrlParser', 'useUnifiedTopology']);
+    });
+
+    it('returns production config when IS_PROD is true', () => {
+      sandbox.stub(nconf, 'get').withArgs('IS_PROD').returns(true);
+      const mongoLibOverride = requireAgain(pathToMongoLib);
+
+      const options = mongoLibOverride.getDefaultConnectionOptions();
+      expect(options).to.have.all.keys(['useNewUrlParser', 'useUnifiedTopology', 'keepAlive', 'keepAliveInitialDelay']);
+    });
+  });
+});

website/server/libs/mongodb.js

@@ -0,0 +1,37 @@
+import os from 'os';
+import nconf from 'nconf';
+
+const IS_PROD = nconf.get('IS_PROD');
+
+// Due to some limitation in the `run-rs` module that is used in development
+// In order to connect to the database on Windows the hostname must be used
+// instead of `localhost`.
+// See https://github.com/vkarpov15/run-rs#notes-on-connecting
+// for more info.
+//
+// This function takes in a connection string and in case it's being run on Windows
+// it replaces `localhost` with the hostname.
+export function getDevelopmentConnectionUrl (originalConnectionUrl) {
+  const isWindows = os.platform() === 'win32';
+
+  if (isWindows) {
+    const hostname = os.hostname();
+    return originalConnectionUrl.replace('mongodb://localhost', `mongodb://${hostname}`);
+  }
+
+  return originalConnectionUrl;
+}
+
+export function getDefaultConnectionOptions () {
+  const commonOptions = {
+    useNewUrlParser: true,
+    useUnifiedTopology: true,
+  };
+
+  return !IS_PROD ? commonOptions : {
+    // See https://mongoosejs.com/docs/connections.html#keepAlive
+    keepAlive: true,
+    keepAliveInitialDelay: 300000,
+    ...commonOptions,
+  };
+}

website/server/libs/setupMongoose.js

@@ -1,6 +1,10 @@
 import nconf from 'nconf';
 import mongoose from 'mongoose';
 import logger from './logger';
+import {
+  getDevelopmentConnectionUrl,
+  getDefaultConnectionOptions,
+} from './mongodb';
 
 const IS_PROD = nconf.get('IS_PROD');
 const MAINTENANCE_MODE = nconf.get('MAINTENANCE_MODE');
@@ -8,22 +12,14 @@ const POOL_SIZE = nconf.get('MONGODB_POOL_SIZE');
 
 // Do not connect to MongoDB when in maintenance mode
 if (MAINTENANCE_MODE !== 'true') {
-  const commonOptions = {
+  const mongooseOptions = getDefaultConnectionOptions();
-    useNewUrlParser: true,
-    useUnifiedTopology: true,
-  };
-
-  const mongooseOptions = !IS_PROD ? commonOptions : {
-    keepAlive: 120,
-    connectTimeoutMS: 30000,
-    ...commonOptions,
-  };
 
   if (POOL_SIZE) mongooseOptions.poolSize = Number(POOL_SIZE);
 
-  const NODE_DB_URI = nconf.get('IS_TEST') ? nconf.get('TEST_DB_URI') : nconf.get('NODE_DB_URI');
+  const DB_URI = nconf.get('IS_TEST') ? nconf.get('TEST_DB_URI') : nconf.get('NODE_DB_URI');
+  const connectionUrl = IS_PROD ? DB_URI : getDevelopmentConnectionUrl(DB_URI);
 
-  mongoose.connect(NODE_DB_URI, mongooseOptions, err => {
+  mongoose.connect(connectionUrl, mongooseOptions, err => {
     if (err) throw err;
     logger.info('Connected with Mongoose.');
   });