Upgrade tests tools and lint migrations and scripts (part 2) (#9998)

* upgrade gulp-babel * upgrade babel-eslint * upgrade eslint-friendly-formatter * start upgrading chai * start to upgrade eslint * restore skipped tests * start to upgrqde monk * fix linting and remove unused file * fix mocha notifications, and common tests * fix unit tests * start to fix initrgration tests * more integration tests fixes * upgrade monk to latest version * lint /scripts * migrations: start moving to /archive unused migrations and run eslint with --fix * lint migrations * fix more integration tests * fix test
2025-12-17 06:37:23 +01:00 · 2018-02-17 18:11:24 +01:00
parent b0ae0ef4da
commit 74ba55c20b
362 changed files with 8041 additions and 7813 deletions
--- a/migrations/archive/api_v3/challenges.js
+++ b/migrations/archive/api_v3/challenges.js
@@ -0,0 +1,218 @@
+// Migrate challenges collection to new schema (except for members)
+
+// The console-stamp module must be installed (not included in package.json)
+
+// It requires two environment variables: MONGODB_OLD and MONGODB_NEW
+
+// Due to some big user profiles it needs more RAM than is allowed by default by v8 (arounf 1.7GB).
+// Run the script with --max-old-space-size=4096 to allow up to 4GB of RAM
+
+// IMPORTANT NOTE: this migration was written when we were using version 3 of lodash.
+// We've now upgraded to lodash v4 but the code used in this migration has not been
+// adapted to work with it. Before this migration is used again any lodash method should
+// be checked for compatibility against the v4 changelog and changed if necessary.
+// https://github.com/lodash/lodash/wiki/Changelog#v400
+console.log('Starting migrations/api_v3/challenges.js.');
+
+require('babel-register');
+require('babel-polyfill');
+
+let Bluebird = require('bluebird');
+let MongoDB = require('mongodb');
+let nconf = require('nconf');
+let mongoose = require('mongoose');
+let _ = require('lodash');
+let uuid = require('uuid');
+let consoleStamp = require('console-stamp');
+let fs = require('fs');
+
+// Add timestamps to console messages
+consoleStamp(console);
+
+// Initialize configuration
+require('../../website/server/libs/api-v3/setupNconf')();
+
+let MONGODB_OLD = nconf.get('MONGODB_OLD');
+let MONGODB_NEW = nconf.get('MONGODB_NEW');
+
+let MongoClient = MongoDB.MongoClient;
+
+mongoose.Promise = Bluebird; // otherwise mongoose models won't work
+
+// Load new models
+let NewChallenge = require('../../website/server/models/challenge').model;
+let Tasks = require('../../website/server/models/task');
+
+// To be defined later when MongoClient connects
+let mongoDbOldInstance;
+let oldChallengeCollection;
+
+let mongoDbNewInstance;
+let newChallengeCollection;
+let newTaskCollection;
+
+let BATCH_SIZE = 1000;
+
+let processedChallenges = 0;
+let totoalProcessedTasks = 0;
+
+let newTasksIds = {}; // a map of old id -> [new id, challengeId]
+
+// Only process challenges that fall in a interval ie -> up to 0000-4000-0000-0000
+let AFTER_CHALLENGE_ID = nconf.get('AFTER_CHALLENGE_ID');
+let BEFORE_CHALLENGE_ID = nconf.get('BEFORE_CHALLENGE_ID');
+
+function processChallenges (afterId) {
+  let processedTasks = 0;
+  let lastChallenge = null;
+  let oldChallenges;
+
+  let query = {};
+
+  if (BEFORE_CHALLENGE_ID) {
+    query._id = {$lte: BEFORE_CHALLENGE_ID};
+  }
+
+  if ((afterId || AFTER_CHALLENGE_ID) && !query._id) {
+    query._id = {};
+  }
+
+  if (afterId) {
+    query._id.$gt = afterId;
+  } else if (AFTER_CHALLENGE_ID) {
+    query._id.$gt = AFTER_CHALLENGE_ID;
+  }
+
+  let batchInsertTasks = newTaskCollection.initializeUnorderedBulkOp();
+  let batchInsertChallenges = newChallengeCollection.initializeUnorderedBulkOp();
+
+  console.log(`Executing challenges query.\nMatching challenges after ${afterId ? afterId : AFTER_CHALLENGE_ID} and before ${BEFORE_CHALLENGE_ID} (included).`);
+
+  return oldChallengeCollection
+    .find(query)
+    .sort({_id: 1})
+    .limit(BATCH_SIZE)
+    .toArray()
+    .then(function (oldChallengesR) {
+      oldChallenges = oldChallengesR;
+
+      console.log(`Processing ${oldChallenges.length} challenges. Already processed ${processedChallenges} challenges and ${totoalProcessedTasks} tasks.`);
+
+      if (oldChallenges.length === BATCH_SIZE) {
+        lastChallenge = oldChallenges[oldChallenges.length - 1]._id;
+      }
+
+      oldChallenges.forEach(function (oldChallenge) {
+        let oldTasks = oldChallenge.habits.concat(oldChallenge.dailys).concat(oldChallenge.rewards).concat(oldChallenge.todos);
+        delete oldChallenge.habits;
+        delete oldChallenge.dailys;
+        delete oldChallenge.rewards;
+        delete oldChallenge.todos;
+
+        let createdAt = oldChallenge.timestamp;
+
+        oldChallenge.memberCount = oldChallenge.members.length;
+        if (oldChallenge.prize <= 0) oldChallenge.prize = 0;
+        if (!oldChallenge.name) oldChallenge.name = 'challenge name';
+        if (!oldChallenge.shortName) oldChallenge.name = 'challenge-name';
+
+        if (!oldChallenge.group) throw new Error('challenge.group is required');
+        if (!oldChallenge.leader) throw new Error('challenge.leader is required');
+
+
+        if (oldChallenge.leader === '9') {
+          oldChallenge.leader = '00000000-0000-4000-9000-000000000000';
+        }
+
+        if (oldChallenge.group === 'habitrpg') {
+          oldChallenge.group = '00000000-0000-4000-A000-000000000000';
+        }
+
+        delete oldChallenge.id;
+
+        let newChallenge = new NewChallenge(oldChallenge);
+
+        newChallenge.createdAt = createdAt;
+
+        oldTasks.forEach(function (oldTask) {
+          oldTask._id = uuid.v4();
+          oldTask._legacyId = oldTask.id; // store the old task id
+          delete oldTask.id;
+
+          oldTask.challenge = oldTask.challenge || {};
+          oldTask.challenge.id = newChallenge._id;
+
+          if (newTasksIds[`${oldTask._legacyId  }-${  newChallenge._id}`]) {
+            throw new Error('duplicate :(');
+          } else {
+            newTasksIds[`${oldTask._legacyId  }-${  newChallenge._id}`] = oldTask._id;
+          }
+
+          oldTask.tags = _.map(oldTask.tags || {}, function (tagPresent, tagId) {
+            return tagPresent && tagId;
+          }).filter(function (tag) {
+            return tag !== false;
+          });
+
+          if (!oldTask.text) oldTask.text = 'task text'; // required
+
+          oldTask.createdAt = oldTask.dateCreated;
+
+          newChallenge.tasksOrder[`${oldTask.type}s`].push(oldTask._id);
+          if (oldTask.completed) oldTask.completed = false;
+
+          let newTask = new Tasks[oldTask.type](oldTask);
+
+          batchInsertTasks.insert(newTask.toObject());
+          processedTasks++;
+        });
+
+        batchInsertChallenges.insert(newChallenge.toObject());
+      });
+
+      console.log(`Saving ${oldChallenges.length} challenges and ${processedTasks} tasks.`);
+
+      return Bluebird.all([
+        batchInsertChallenges.execute(),
+        batchInsertTasks.execute(),
+      ]);
+    })
+    .then(function () {
+      totoalProcessedTasks += processedTasks;
+      processedChallenges += oldChallenges.length;
+
+      console.log(`Saved ${oldChallenges.length} challenges and their tasks.`);
+
+      if (lastChallenge) {
+        return processChallenges(lastChallenge);
+      } else {
+        console.log('Writing newTasksIds.json...');
+        fs.writeFileSync('newTasksIds.json', JSON.stringify(newTasksIds, null, 4), 'utf8');
+        return console.log('Done!');
+      }
+    });
+}
+
+// Connect to the databases
+Bluebird.all([
+  MongoClient.connect(MONGODB_OLD),
+  MongoClient.connect(MONGODB_NEW),
+])
+  .then(function (result) {
+    let oldInstance = result[0];
+    let newInstance = result[1];
+
+    mongoDbOldInstance = oldInstance;
+    oldChallengeCollection = mongoDbOldInstance.collection('challenges');
+
+    mongoDbNewInstance = newInstance;
+    newChallengeCollection = mongoDbNewInstance.collection('challenges');
+    newTaskCollection = mongoDbNewInstance.collection('tasks');
+
+    console.log(`Connected with MongoClient to ${MONGODB_OLD} and ${MONGODB_NEW}.`);
+
+    return processChallenges();
+  })
+  .catch(function (err) {
+    console.error(err.stack || err);
+  });