Merge branch 'api-v3' into api-v3-client

2025-12-18 15:17:25 +01:00 · 2016-05-02 00:25:25 +02:00
parent 115c5374cb f0e696b0ec
commit 01c07ae6df
20 changed files with 706 additions and 179 deletions
--- a/common/script/ops/unlock.js
+++ b/common/script/ops/unlock.js
@@ -44,9 +44,11 @@ module.exports = function unlock (user, req = {}, analytics) {

    if (alreadyOwnedItems === setPaths.length) {
      throw new NotAuthorized(i18n.t('alreadyUnlocked', req.language));
-    } else if (alreadyOwnedItems > 0) {
+    // TODO write math formula to check if buying the full set is cheaper than the items individually
+    // (item cost * number of remaining items) < setCost`
+    } /* else if (alreadyOwnedItems > 0) {
      throw new NotAuthorized(i18n.t('alreadyUnlockedPart', req.language));
-    }
+    } */
  } else {
    alreadyOwns = _.get(user, `purchased.${path}`) === true;
  }
--- a/migrations/api_v3/challenges.js
+++ b/migrations/api_v3/challenges.js
@@ -1,81 +1,4 @@
-/*
-  name is required,
-  shortName is required,
-  tasksOrder
-  habits, dailys, todos and rewards must be removed
-  leader is required
-  group is required
-  members must be removed
-  memberCount must be checked
-  prize must be >= 0
-*/
-
-// A map of (original taskId) -> [new taskId in challenge, challendId] of tasks belonging to challenges where the task id had to change
-// This way later we can have use the right task.challenge.taskId in user's tasks
-var duplicateTasks = {};
-
-        // ... convert tasks to individual models
-        async.each(
-          challenge.dailys
-            .concat(challenge.habits)
-            .concat(challenge.rewards)
-            .concat(challenge.todos),
-          function(task, cb1) {
-
-            task = new TaskModel(task); // this should also fix dailies that wen to the habits array or vice-versa
-
-            TaskModel.findOne({_id: task._id}, function(err, taskSameId){
-              if(err) return cb1(err);
-
-              // We already have a task with the same id, change this one
-              // and will require special handling
-              if(taskSameId) {
-                task._id = shared.uuid();
-                task.legacyId = taskSameId._id; // We set this for challenge tasks too
-                // we use an array as the same task may have multiple duplicates
-                duplicateTasks[taskSameId._id] = duplicateTasks[taskSameId._id] || [];
-                duplicateTasks[taskSameId._id].push([task._id, challenge._id]);
-                console.log('Duplicate task ', taskSameId._id, 'challenge ', challenge._id, 'new id ', task._id);
-              }
-
-              task.save(function(err, savedTask){
-                if(err) return cb1(err);
-
-                challenge.tasksOrder[savedTask.type + 's'].push(savedTask._id);
-                cb1();
-              });
-            });
-          }, function(err) {
-            if(err) return cb(err);
-
-            var newChallenge = new NewChallengeModel(challenge); // This will make sure old data is discarded
-            newChallenge.save(function(err, chal){
-              if(err) return cb(err);
-              console.log('Processed: ', chal._id);
-              cb();
-            });
-          });
-      }, function(err) {
-        if(err) throw err;
-
-        processed = processed + challenges.length;
-        console.log('Processed ' + challenges.length + ' challenges.', 'Total: ' + processed);
-
-        if(lastChal && lastChal._id){
-          processChal(lastChal._id);
-        } else {
-          console.log('Done!');
-          // outputting the duplicate tasks
-          console.log(JSON.stringify(duplicateTasks, null, 4));
-        }
-      });
-    });
-};
-
-processChal();
-
-// Migrate users collection to new schema
-// This should run AFTER challenges migration
+// Migrate challenges collection to new schema (except for members)

 // The console-stamp module must be installed (not included in package.json)

@@ -83,7 +6,7 @@ processChal();

 // Due to some big user profiles it needs more RAM than is allowed by default by v8 (arounf 1.7GB).
 // Run the script with --max-old-space-size=4096 to allow up to 4GB of RAM
-console.log('Starting migrations/api_v3/users.js.');
+console.log('Starting migrations/api_v3/challenges.js.');

 require('babel-register');

@@ -153,7 +76,7 @@ function processChallenges (afterId) {
  var batchInsertTasks = newTaskCollection.initializeUnorderedBulkOp();
  var batchInsertChallenges = newChallengeCollection.initializeUnorderedBulkOp();

-  console.log(`Executing challenges query.\nMatching challenges after ${afterId ? afterId : AFTER_USER_ID} and before ${BEFORE_USER_ID} (included).`);
+  console.log(`Executing challenges query.\nMatching challenges after ${afterId ? afterId : AFTER_CHALLENGE_ID} and before ${BEFORE_CHALLENGE_ID} (included).`);

  return oldChallengeCollection
  .find(query)
@@ -176,21 +99,34 @@ function processChallenges (afterId) {
      delete oldChallenge.rewards;
      delete oldChallenge.todos;

+      var createdAt = oldChallenge.timestamp;
+
+      oldChallenge.memberCount = oldChallenge.members.length;
+      if (!oldChallenge.prize <= 0) oldChallenge.prize = 0;
+      if (!oldChallenge.name) oldChallenge.name = 'challenge name';
+      if (!oldChallenge.shortName) oldChallenge.name = 'challenge-name';
+
+      if (!oldChallenge.group) throw new Error('challenge.group is required');
+      if (!oldChallenge.leader) throw new Error('challenge.leader is required');
+
      var newChallenge = new NewChallenge(oldChallenge);

+      newChallenge.createdAt = createdAt;
+
      oldTasks.forEach(function (oldTask) {
-        // TODO
-        oldTask._id = oldTask.id; // keep the old uuid unless duplicated
+        oldTask._id = uuid.v4(); // TODO keep the old uuid unless duplicated
+        oldTask.legacyId = oldTask.id; // store the old task id
        delete oldTask.id;

-        oldTask.challenge = oldTask.challenge || {};
-        oldTask.challenge.id = oldChallenge.id;
-
-        if (!oldTask.text) oldTask.text = 'task text'; // required
-        oldTask.tags = _.map(oldTask.tags, function (tagPresent, tagId) { // TODO used for challenges' tasks?
+        oldTask.tags = _.map(oldTask.tags || {}, function (tagPresent, tagId) {
          return tagPresent && tagId;
        });

+        if (!oldTask.text) oldTask.text = 'task text'; // required
+
+        oldTask.challenge = oldTask.challenge || {};
+        oldTask.challenge.id = oldChallenge._id;
+
        newChallenge.tasksOrder[`${oldTask.type}s`].push(oldTask._id);
        if (oldTask.completed) oldTask.completed = false;

@@ -203,7 +139,7 @@ function processChallenges (afterId) {
      batchInsertChallenges.insert(newChallenge.toObject());
    });

-    console.log(`Saving ${oldChallenges.length} users and ${processedTasks} tasks.`);
+    console.log(`Saving ${oldChallenges.length} challenges and ${processedTasks} tasks.`);

    return Q.all([
      batchInsertChallenges.execute(),
@@ -214,9 +150,9 @@ function processChallenges (afterId) {
    totoalProcessedTasks += processedTasks;
    processedChallenges += oldChallenges.length;

-    console.log(`Saved ${oldChallenges.length} users and their tasks.`);
+    console.log(`Saved ${oldChallenges.length} challenges and their tasks.`);

-    if (lastUser) {
+    if (lastChallenge) {
      return processChallenges(lastChallenge);
    } else {
      return console.log('Done!');
@@ -245,5 +181,5 @@ Q.all([
  return processChallenges();
 })
 .catch(function (err) {
-  console.error(err);
+  console.error(err.stack || err);
 });
--- a/migrations/api_v3/challengesMembers.js
+++ b/migrations/api_v3/challengesMembers.js
@@ -0,0 +1,135 @@
+// Migrate challenges members
+// Run AFTER users migration
+
+// The console-stamp module must be installed (not included in package.json)
+
+// It requires two environment variables: MONGODB_OLD and MONGODB_NEW
+
+// Due to some big user profiles it needs more RAM than is allowed by default by v8 (arounf 1.7GB).
+// Run the script with --max-old-space-size=4096 to allow up to 4GB of RAM
+console.log('Starting migrations/api_v3/challengesMembers.js.');
+
+require('babel-register');
+
+var Q = require('q');
+var MongoDB = require('mongodb');
+var nconf = require('nconf');
+var mongoose = require('mongoose');
+var _ = require('lodash');
+var uuid = require('uuid');
+var consoleStamp = require('console-stamp');
+
+// Add timestamps to console messages
+consoleStamp(console);
+
+// Initialize configuration
+require('../../website/src/libs/api-v3/setupNconf')();
+
+var MONGODB_OLD = nconf.get('MONGODB_OLD');
+var MONGODB_NEW = nconf.get('MONGODB_NEW');
+
+var MongoClient = MongoDB.MongoClient;
+
+mongoose.Promise = Q.Promise; // otherwise mongoose models won't work
+
+// To be defined later when MongoClient connects
+var mongoDbOldInstance;
+var oldChallengeCollection;
+
+var mongoDbNewInstance;
+var newUserCollection;
+
+var BATCH_SIZE = 1000;
+
+var processedChallenges = 0;
+
+// Only process challenges that fall in a interval ie -> up to 0000-4000-0000-0000
+var AFTER_CHALLENGE_ID = nconf.get('AFTER_CHALLENGE_ID');
+var BEFORE_CHALLENGE_ID = nconf.get('BEFORE_CHALLENGE_ID');
+
+function processChallenges (afterId) {
+  var processedTasks = 0;
+  var lastChallenge = null;
+  var oldChallenges;
+
+  var query = {};
+
+  if (BEFORE_CHALLENGE_ID) {
+    query._id = {$lte: BEFORE_CHALLENGE_ID};
+  }
+
+  if ((afterId || AFTER_CHALLENGE_ID) && !query._id) {
+    query._id = {};
+  }
+
+  if (afterId) {
+    query._id.$gt = afterId;
+  } else if (AFTER_CHALLENGE_ID) {
+    query._id.$gt = AFTER_CHALLENGE_ID;
+  }
+
+  console.log(`Executing challenges query.\nMatching challenges after ${afterId ? afterId : AFTER_CHALLENGE_ID} and before ${BEFORE_CHALLENGE_ID} (included).`);
+
+  return oldChallengeCollection
+  .find(query)
+  .sort({_id: 1})
+  .limit(BATCH_SIZE)
+  .toArray()
+  .then(function (oldChallengesR) {
+    oldChallenges = oldChallengesR;
+
+    var promises = [];
+
+    console.log(`Processing ${oldChallenges.length} challenges. Already processed ${processedChallenges} challenges.`);
+
+    if (oldChallenges.length === BATCH_SIZE) {
+      lastChallenge = oldChallenges[oldChallenges.length - 1]._id;
+    }
+
+    oldChallenges.forEach(function (oldChallenge) {
+      promises.push(newUserCollection.updateMany({
+        _id: {$in: oldChallenge.members},
+      }, {
+        $push: {challenges: oldChallenge._id},
+      }, {multi: true}));
+    });
+
+    console.log(`Migrating members of ${oldChallenges.length} challenges.`);
+
+    return Q.all(promises);
+  })
+  .then(function () {
+    processedChallenges += oldChallenges.length;
+
+    console.log(`Migrated members of ${oldChallenges.length} challenges.`);
+
+    if (lastChallenge) {
+      return processChallenges(lastChallenge);
+    } else {
+      return console.log('Done!');
+    }
+  });
+}
+
+// Connect to the databases
+Q.all([
+  MongoClient.connect(MONGODB_OLD),
+  MongoClient.connect(MONGODB_NEW),
+])
+.then(function (result) {
+  var oldInstance = result[0];
+  var newInstance = result[1];
+
+  mongoDbOldInstance = oldInstance;
+  oldChallengeCollection = mongoDbOldInstance.collection('challenges');
+
+  mongoDbNewInstance = newInstance;
+  newUserCollection = mongoDbNewInstance.collection('users');
+
+  console.log(`Connected with MongoClient to ${MONGODB_OLD} and ${MONGODB_NEW}.`);
+
+  return processChallenges();
+})
+.catch(function (err) {
+  console.error(err.stack || err);
+});
--- a/migrations/api_v3/coupons.js
+++ b/migrations/api_v3/coupons.js
@@ -0,0 +1,135 @@
+// Migrate coupons collection to new schema
+
+// The console-stamp module must be installed (not included in package.json)
+
+// It requires two environment variables: MONGODB_OLD and MONGODB_NEW
+
+// Due to some big user profiles it needs more RAM than is allowed by default by v8 (arounf 1.7GB).
+// Run the script with --max-old-space-size=4096 to allow up to 4GB of RAM
+console.log('Starting migrations/api_v3/coupons.js.');
+
+require('babel-register');
+
+var Q = require('q');
+var MongoDB = require('mongodb');
+var nconf = require('nconf');
+var mongoose = require('mongoose');
+var _ = require('lodash');
+var uuid = require('uuid');
+var consoleStamp = require('console-stamp');
+
+// Add timestamps to console messages
+consoleStamp(console);
+
+// Initialize configuration
+require('../../website/src/libs/api-v3/setupNconf')();
+
+var MONGODB_OLD = nconf.get('MONGODB_OLD');
+var MONGODB_NEW = nconf.get('MONGODB_NEW');
+
+var MongoClient = MongoDB.MongoClient;
+
+mongoose.Promise = Q.Promise; // otherwise mongoose models won't work
+
+// Load new models
+var Coupon = require('../../website/src/models/coupon').model;
+
+// To be defined later when MongoClient connects
+var mongoDbOldInstance;
+var oldCouponCollection;
+
+var mongoDbNewInstance;
+var newCouponCollection;
+
+var BATCH_SIZE = 1000;
+
+var processedCoupons = 0;
+
+// Only process coupons that fall in a interval ie -> up to 0000-4000-0000-0000
+var AFTER_COUPON_ID = nconf.get('AFTER_COUPON_ID');
+var BEFORE_COUPON_ID = nconf.get('BEFORE_COUPON_ID');
+
+function processCoupons (afterId) {
+  var processedTasks = 0;
+  var lastCoupon = null;
+  var oldCoupons;
+
+  var query = {};
+
+  if (BEFORE_COUPON_ID) {
+    query._id = {$lte: BEFORE_COUPON_ID};
+  }
+
+  if ((afterId || AFTER_COUPON_ID) && !query._id) {
+    query._id = {};
+  }
+
+  if (afterId) {
+    query._id.$gt = afterId;
+  } else if (AFTER_COUPON_ID) {
+    query._id.$gt = AFTER_COUPON_ID;
+  }
+
+  var batchInsertCoupons = newCouponCollection.initializeUnorderedBulkOp();
+
+  console.log(`Executing coupons query.\nMatching coupons after ${afterId ? afterId : AFTER_COUPON_ID} and before ${BEFORE_COUPON_ID} (included).`);
+
+  return oldCouponCollection
+  .find(query)
+  .sort({_id: 1})
+  .limit(BATCH_SIZE)
+  .toArray()
+  .then(function (oldCouponsR) {
+    oldCoupons = oldCouponsR;
+
+    console.log(`Processing ${oldCoupons.length} coupons. Already processed ${processedCoupons} coupons.`);
+
+    if (oldCoupons.length === BATCH_SIZE) {
+      lastCoupon = oldCoupons[oldCoupons.length - 1]._id;
+    }
+
+    oldCoupons.forEach(function (oldCoupon) {
+      var newCoupon = new Coupon(oldCoupon);
+
+      batchInsertCoupons.insert(newCoupon.toObject());
+    });
+
+    console.log(`Saving ${oldCoupons.length} coupons.`);
+
+    return batchInsertCoupons.execute();
+  })
+  .then(function () {
+    processedCoupons += oldCoupons.length;
+
+    console.log(`Saved ${oldCoupons.length} coupons.`);
+
+    if (lastCoupon) {
+      return processCoupons(lastCoupon);
+    } else {
+      return console.log('Done!');
+    }
+  });
+}
+
+// Connect to the databases
+Q.all([
+  MongoClient.connect(MONGODB_OLD),
+  MongoClient.connect(MONGODB_NEW),
+])
+.then(function (result) {
+  var oldInstance = result[0];
+  var newInstance = result[1];
+
+  mongoDbOldInstance = oldInstance;
+  oldCouponCollection = mongoDbOldInstance.collection('coupons');
+
+  mongoDbNewInstance = newInstance;
+  newCouponCollection = mongoDbNewInstance.collection('coupons');
+
+  console.log(`Connected with MongoClient to ${MONGODB_OLD} and ${MONGODB_NEW}.`);
+
+  return processCoupons();
+})
+.catch(function (err) {
+  console.error(err.stack || err);
+});
--- a/migrations/api_v3/emailUnsubscriptions.js
+++ b/migrations/api_v3/emailUnsubscriptions.js
@@ -1,4 +1,136 @@
-/*
-  email must be lowercase
-  remove unique: true from mongoose schema
-*/
+// Migrate unsubscriptions collection to new schema
+
+// The console-stamp module must be installed (not included in package.json)
+
+// It requires two environment variables: MONGODB_OLD and MONGODB_NEW
+
+// Due to some big user profiles it needs more RAM than is allowed by default by v8 (arounf 1.7GB).
+// Run the script with --max-old-space-size=4096 to allow up to 4GB of RAM
+console.log('Starting migrations/api_v3/unsubscriptions.js.');
+
+require('babel-register');
+
+var Q = require('q');
+var MongoDB = require('mongodb');
+var nconf = require('nconf');
+var mongoose = require('mongoose');
+var _ = require('lodash');
+var uuid = require('uuid');
+var consoleStamp = require('console-stamp');
+
+// Add timestamps to console messages
+consoleStamp(console);
+
+// Initialize configuration
+require('../../website/src/libs/api-v3/setupNconf')();
+
+var MONGODB_OLD = nconf.get('MONGODB_OLD');
+var MONGODB_NEW = nconf.get('MONGODB_NEW');
+
+var MongoClient = MongoDB.MongoClient;
+
+mongoose.Promise = Q.Promise; // otherwise mongoose models won't work
+
+// Load new models
+var EmailUnsubscription = require('../../website/src/models/emailUnsubscription').model;
+
+// To be defined later when MongoClient connects
+var mongoDbOldInstance;
+var oldUnsubscriptionCollection;
+
+var mongoDbNewInstance;
+var newUnsubscriptionCollection;
+
+var BATCH_SIZE = 1000;
+
+var processedUnsubscriptions = 0;
+
+// Only process unsubscriptions that fall in a interval ie -> up to 0000-4000-0000-0000
+var AFTER_UNSUBSCRIPTION_ID = nconf.get('AFTER_UNSUBSCRIPTION_ID');
+var BEFORE_UNSUBSCRIPTION_ID = nconf.get('BEFORE_UNSUBSCRIPTION_ID');
+
+function processUnsubscriptions (afterId) {
+  var processedTasks = 0;
+  var lastUnsubscription = null;
+  var oldUnsubscriptions;
+
+  var query = {};
+
+  if (BEFORE_UNSUBSCRIPTION_ID) {
+    query._id = {$lte: BEFORE_UNSUBSCRIPTION_ID};
+  }
+
+  if ((afterId || AFTER_UNSUBSCRIPTION_ID) && !query._id) {
+    query._id = {};
+  }
+
+  if (afterId) {
+    query._id.$gt = afterId;
+  } else if (AFTER_UNSUBSCRIPTION_ID) {
+    query._id.$gt = AFTER_UNSUBSCRIPTION_ID;
+  }
+
+  var batchInsertUnsubscriptions = newUnsubscriptionCollection.initializeUnorderedBulkOp();
+
+  console.log(`Executing unsubscriptions query.\nMatching unsubscriptions after ${afterId ? afterId : AFTER_UNSUBSCRIPTION_ID} and before ${BEFORE_UNSUBSCRIPTION_ID} (included).`);
+
+  return oldUnsubscriptionCollection
+  .find(query)
+  .sort({_id: 1})
+  .limit(BATCH_SIZE)
+  .toArray()
+  .then(function (oldUnsubscriptionsR) {
+    oldUnsubscriptions = oldUnsubscriptionsR;
+
+    console.log(`Processing ${oldUnsubscriptions.length} unsubscriptions. Already processed ${processedUnsubscriptions} unsubscriptions.`);
+
+    if (oldUnsubscriptions.length === BATCH_SIZE) {
+      lastUnsubscription = oldUnsubscriptions[oldUnsubscriptions.length - 1]._id;
+    }
+
+    oldUnsubscriptions.forEach(function (oldUnsubscription) {
+      oldUnsubscription.email = oldUnsubscription.email.toLowerCase();
+      var newUnsubscription = new EmailUnsubscription(oldUnsubscription);
+
+      batchInsertUnsubscriptions.insert(newUnsubscription.toObject());
+    });
+
+    console.log(`Saving ${oldUnsubscriptions.length} unsubscriptions.`);
+
+    return batchInsertUnsubscriptions.execute();
+  })
+  .then(function () {
+    processedUnsubscriptions += oldUnsubscriptions.length;
+
+    console.log(`Saved ${oldUnsubscriptions.length} unsubscriptions.`);
+
+    if (lastUnsubscription) {
+      return processUnsubscriptions(lastUnsubscription);
+    } else {
+      return console.log('Done!');
+    }
+  });
+}
+
+// Connect to the databases
+Q.all([
+  MongoClient.connect(MONGODB_OLD),
+  MongoClient.connect(MONGODB_NEW),
+])
+.then(function (result) {
+  var oldInstance = result[0];
+  var newInstance = result[1];
+
+  mongoDbOldInstance = oldInstance;
+  oldUnsubscriptionCollection = mongoDbOldInstance.collection('emailunsubscriptions');
+
+  mongoDbNewInstance = newInstance;
+  newUnsubscriptionCollection = mongoDbNewInstance.collection('emailunsubscriptions');
+
+  console.log(`Connected with MongoClient to ${MONGODB_OLD} and ${MONGODB_NEW}.`);
+
+  return processUnsubscriptions();
+})
+.catch(function (err) {
+  console.error(err.stack || err);
+});
--- a/migrations/api_v3/groups.js
+++ b/migrations/api_v3/groups.js
@@ -1,17 +1,201 @@
 /*
-  name is required
-  leader is required
-  type is required
-  privacy is required
-  leaderOnly.challenges is required
  members are not stored anymore
  invites are not stored anymore
-  challenges are not stored anymore
-  balance > 0
-  memberCount must be checked
-  challengeCount must be checked
-  quest.leader must be present (default to party leader)
-  quest.key must be valid (otherwise remove)

  tavern id and leader must be updated
 */
+
+// Migrate groups collection to new schema
+// Run AFTER users migration
+
+// The console-stamp module must be installed (not included in package.json)
+
+// It requires two environment variables: MONGODB_OLD and MONGODB_NEW
+
+// Due to some big user profiles it needs more RAM than is allowed by default by v8 (arounf 1.7GB).
+// Run the script with --max-old-space-size=4096 to allow up to 4GB of RAM
+console.log('Starting migrations/api_v3/groups.js.');
+
+require('babel-register');
+
+var Q = require('q');
+var MongoDB = require('mongodb');
+var nconf = require('nconf');
+var mongoose = require('mongoose');
+var _ = require('lodash');
+var uuid = require('uuid');
+var consoleStamp = require('console-stamp');
+
+// Add timestamps to console messages
+consoleStamp(console);
+
+// Initialize configuration
+require('../../website/src/libs/api-v3/setupNconf')();
+
+var MONGODB_OLD = nconf.get('MONGODB_OLD');
+var MONGODB_NEW = nconf.get('MONGODB_NEW');
+
+var MongoClient = MongoDB.MongoClient;
+
+mongoose.Promise = Q.Promise; // otherwise mongoose models won't work
+
+// Load new models
+var NewGroup = require('../../website/src/models/group').model;
+
+var TAVERN_ID = require('../../website/src/models/group').TAVERN_ID;
+
+// To be defined later when MongoClient connects
+var mongoDbOldInstance;
+var oldGroupCollection;
+
+var mongoDbNewInstance;
+var newGroupCollection;
+var newUserCollection;
+
+var BATCH_SIZE = 1000;
+
+var processedGroups = 0;
+
+// Only process groups that fall in a interval ie -> up to 0000-4000-0000-0000
+var AFTER_GROUP_ID = nconf.get('AFTER_GROUP_ID');
+var BEFORE_GROUP_ID = nconf.get('BEFORE_GROUP_ID');
+
+function processGroups (afterId) {
+  var processedTasks = 0;
+  var lastGroup = null;
+  var oldGroups;
+
+  var query = {};
+
+  if (BEFORE_GROUP_ID) {
+    query._id = {$lte: BEFORE_GROUP_ID};
+  }
+
+  if ((afterId || AFTER_GROUP_ID) && !query._id) {
+    query._id = {};
+  }
+
+  if (afterId) {
+    query._id.$gt = afterId;
+  } else if (AFTER_GROUP_ID) {
+    query._id.$gt = AFTER_GROUP_ID;
+  }
+
+  var batchInsertGroups = newGroupCollection.initializeUnorderedBulkOp();
+
+  console.log(`Executing groups query.\nMatching groups after ${afterId ? afterId : AFTER_GROUP_ID} and before ${BEFORE_GROUP_ID} (included).`);
+
+  return oldGroupCollection
+  .find(query)
+  .sort({_id: 1})
+  .limit(BATCH_SIZE)
+  .toArray()
+  .then(function (oldGroupsR) {
+    oldGroups = oldGroupsR;
+
+    var promises = [];
+
+    console.log(`Processing ${oldGroups.length} groups. Already processed ${processedGroups} groups.`);
+
+    if (oldGroups.length === BATCH_SIZE) {
+      lastGroup = oldGroups[oldGroups.length - 1]._id;
+    }
+
+    oldGroups.forEach(function (oldGroup) {
+      if ((!oldGroup.privacy || oldGroup.privacy === 'private') && (!oldGroup.members || oldGroup.members.length === 0)) return; // delete empty private groups
+      oldGroup.memberCount = oldGroup.members ? oldGroup.members.length : 0;
+      oldGroup.memberCount = oldGroup.challenges ? oldGroup.challenges.length : 0;
+
+      if (!oldGroup.balance <= 0) oldGroup.balance = 0;
+      if (!oldGroup.name) oldGroup.name = 'group name';
+      if (!oldGroup.leaderOnly) oldGroup.leaderOnly = {};
+      if (!oldGroup.leaderOnly.challenges) oldGroup.leaderOnly.challenges = false;
+
+      // Tavern
+      if (oldGroup._id === 'habitrpg') {
+        oldGroup._id = TAVERN_ID;
+        oldGroup.leader = '7bde7864-ebc5-4ee2-a4b7-1070d464cdb0'; // Siena Leslie
+      }
+
+      if (!oldGroup.type) {
+        // throw new Error('group.type is required');
+        oldGroup.type = 'guild';
+      }
+
+      if (!oldGroup.leader) {
+        if (oldGroup.members && oldGroup.members.length > 0) {
+          oldGroup.leader = oldGroup.members[0];
+        } else {
+          throw new Error('group.leader is required and no member available!');
+        }
+      }
+
+      if (!oldGroup.privacy) {
+        // throw new Error('group.privacy is required');
+        group.privacy = 'private';
+      }
+
+      var updateMembers = {};
+
+      if (oldGroup.type === 'guild') {
+        updateMembers.$push = {guilds: oldGroup._id};
+      } else if (oldGroup.type === 'party') {
+        updateMembers.$set = {'party._id': oldGroup._id};
+      }
+
+      if (oldGroup.members) {
+        promises.push(newUserCollection.updateMany({
+          _id: {$in: oldGroup.members},
+        }, updateMembers, {multi: true}));
+      }
+
+      var newGroup = new NewGroup(oldGroup);
+
+      batchInsertGroups.insert(newGroup.toObject());
+    });
+
+    console.log(`Saving ${oldGroups.length} groups and migrating members to users collection.`);
+
+    promises.push(batchInsertGroups.execute());
+    return Q.all(promises);
+  })
+  .then(function () {
+    processedGroups += oldGroups.length;
+
+    console.log(`Saved ${oldGroups.length} groups and migrated their members to the user collection.`);
+
+    if (lastGroup) {
+      return processGroups(lastGroup);
+    } else {
+      return console.log('Done!');
+    }
+  });
+}
+
+// Connect to the databases
+Q.all([
+  MongoClient.connect(MONGODB_OLD),
+  MongoClient.connect(MONGODB_NEW),
+])
+.then(function (result) {
+  var oldInstance = result[0];
+  var newInstance = result[1];
+
+  mongoDbOldInstance = oldInstance;
+  oldGroupCollection = mongoDbOldInstance.collection('groups');
+
+  mongoDbNewInstance = newInstance;
+  newGroupCollection = mongoDbNewInstance.collection('groups');
+  newUserCollection = mongoDbNewInstance.collection('users');
+
+  console.log(`Connected with MongoClient to ${MONGODB_OLD} and ${MONGODB_NEW}.`);
+
+  // First delete the tavern group created by having required the group model
+  return newGroupCollection.deleteOne({_id: TAVERN_ID});
+})
+.then(function () {
+  return processGroups();
+})
+.catch(function (err) {
+  console.error(err.stack || err);
+});
--- a/migrations/api_v3/users.js
+++ b/migrations/api_v3/users.js
@@ -18,6 +18,7 @@ var mongoose = require('mongoose');
 var _ = require('lodash');
 var uuid = require('uuid');
 var consoleStamp = require('console-stamp');
+var common = require('../../common');

 // Add timestamps to console messages
 consoleStamp(console);
@@ -28,6 +29,7 @@ require('../../website/src/libs/api-v3/setupNconf')();
 var MONGODB_OLD = nconf.get('MONGODB_OLD');
 var MONGODB_NEW = nconf.get('MONGODB_NEW');

+var taskDefaults = common.taskDefaults;
 var MongoClient = MongoDB.MongoClient;

 mongoose.Promise = Q.Promise; // otherwise mongoose models won't work
@@ -59,7 +61,6 @@ var BEFORE_USER_ID = nconf.get('BEFORE_USER_ID');
 - groups
 - invitations
 - challenges' tasks
- checklists from .id to ._id (reminders too!)
 */

 function processUsers (afterId) {
@@ -111,12 +112,16 @@ function processUsers (afterId) {

      oldUser.tags = oldUser.tags.map(function (tag) {
        return {
-          _id: tag.id,
-          name: tag.name,
+          id: tag.id,
+          name: tag.name || 'tag name',
          challenge: tag.challenge,
        };
      });

+      if (oldUser._id === '9') { // Tyler Renelle
+        oldUser._id = '00000000-0000-4000-9000-000000000000';
+      }
+
      var newUser = new NewUser(oldUser);

      oldTasks.forEach(function (oldTask) {
@@ -125,7 +130,13 @@ function processUsers (afterId) {
        oldTask.legacyId = oldTask.id; // store the old task id
        delete oldTask.id;

-        oldTask.challenge = {};
+        oldTask.challenge = oldTask.challenge || {};
+        if (oldTask.challenge.id) {
+          oldTask.challenge.taskId = oldTask.legacyId;
+        }
+
+        oldTask.createdAt = old.dateCreated;
+
        if (!oldTask.text) oldTask.text = 'task text'; // required
        oldTask.tags = _.map(oldTask.tags, function (tagPresent, tagId) {
          return tagPresent && tagId;
@@ -135,9 +146,21 @@ function processUsers (afterId) {
          newUser.tasksOrder[`${oldTask.type}s`].push(oldTask._id);
        }

-        var newTask = new NewTasks[oldTask.type](oldTask);
+        var allTasksFields = ['_id', 'type', 'text', 'notes', 'tags', 'value', 'priority', 'attribute', 'challenge', 'reminders'];
+        // using mongoose models is too slow
+        if (oldTask.type === 'habit') {
+          oldTask = _.pick(oldTask, allTasksFields.concat(['history', 'up', 'down']));
+        } else if (oldTask.type === 'daily') {
+          oldTask = _.pick(oldTask, allTasksFields.concat(['completed', 'collapseChecklist', 'checklist', 'history', 'frequency', 'everyX', 'startDate', 'repeat', 'streak']));
+        } else if (oldTask.type === 'todo') {
+          oldTask = _.pick(oldTask, allTasksFields.concat(['completed', 'collapseChecklist', 'checklist', 'date', 'dateCompleted']));
+        } else if (oldTask.type === 'reward') {
+          oldTask = _.pick(oldTask, allTasksFields);
+        } else {
+          throw new Error('Task with no or invalid type!');
+        }

-        batchInsertTasks.insert(newTask.toObject());
+        batchInsertTasks.insert(taskDefaults(oldTask));
        processedTasks++;
      });

@@ -165,30 +188,6 @@ function processUsers (afterId) {
  });
 }

-/*
-
-TODO var challengeTasksChangedId = {};
-
-tasksArr.forEach(function(task){
-  task.challenge = task.challenge || {};
-  if(task.challenge.id) {
-    // If challengeTasksChangedId[task._id] then we got on of the duplicates from the challenges migration
-    if (challengeTasksChangedId[task.legacyId]) {
-      var res = _.find(challengeTasksChangedId[task.legacyId], function(arr){
-        return arr[1] === task.challenge.id;
-      });
-
-      // If res, id changed, otherwise matches the original one
-      task.challenge.taskId = res ? res[0] : task.legacyId;
-    } else {
-      task.challenge.taskId = task.legacyId;
-    }
-  }
-
-  if(!task.type) console.log('Task without type ', task._id, ' user ', user._id);
-});
-*/
-
 // Connect to the databases
 Q.all([
  MongoClient.connect(MONGODB_OLD),
@@ -210,5 +209,5 @@ Q.all([
  return processUsers();
 })
 .catch(function (err) {
-  console.error(err);
+  console.error(err.stack || err);
 });
--- a/package.json
+++ b/package.json
@@ -67,6 +67,7 @@
    "morgan": "^1.7.0",
    "nconf": "~0.8.2",
    "newrelic": "~1.26.1",
+    "uuid": "^2.0.1",
    "nib": "~1.0.1",
    "nodemailer": "^1.9.0",
    "object-path": "^0.9.2",
--- a/test/api/v3/integration/models/GET-model_paths.test.js
+++ b/test/api/v3/integration/models/GET-model_paths.test.js
@@ -23,7 +23,9 @@ describe('GET /models/:model/paths', () => {
    it(`returns the model paths for ${model}`, async () => {
      let res = await user.get(`/models/${model}/paths`);

-      expect(res._id).to.equal('String');
+      if (model !== 'tag') expect(res._id).to.equal('String');
+      if (model === 'tag') expect(res.id).to.equal('String');
+
      expect(res).to.not.have.keys('__v');
    });
  });
--- a/test/api/v3/unit/models/challenge.test.js
+++ b/test/api/v3/unit/models/challenge.test.js
@@ -96,7 +96,7 @@ describe('Challenge Model', () => {
        });

        expect(updatedNewMember.challenges).to.contain(challenge._id);
-        expect(updatedNewMember.tags[3]._id).to.equal(challenge._id);
+        expect(updatedNewMember.tags[3].id).to.equal(challenge._id);
        expect(updatedNewMember.tags[3].name).to.equal(challenge.shortName);
        expect(syncedTask).to.exist;
      });
--- a/test/common/ops/unlock.js
+++ b/test/common/ops/unlock.js
@@ -54,7 +54,8 @@ describe('shared.ops.unlock', () => {
    }
  });

-  it('returns an error when user already owns items in a full set', (done) => {
+  // disabled untill fully implemente
+  xit('returns an error when user already owns items in a full set', (done) => {
    try {
      unlock(user, {query: {path: unlockPath}});
      unlock(user, {query: {path: unlockPath}});
--- a/website/src/controllers/api-v2/user.js
+++ b/website/src/controllers/api-v2/user.js
@@ -493,21 +493,21 @@ api.getTags = function (req, res, next) {
  res.json(res.locals.user.tags.toObject().map(tag => {
    return {
      name: tag.name,
-      id: tag._id,
+      id: tag.id,
      challenge: tag.challenge,
    }
  }));
 };

 api.getTag = function (req, res, next) {
-  let tag = res.locals.user.tags.id(req.params.id);
+  let tag = _.find(res.locals.user.tags, {id: req.params.id});
  if (!tag) {
    return res.status(404).json({err: i18n.t('messageTagNotFound', req.language)});
  }

  res.json({
    name: tag.name,
-    id: tag._id,
+    id: tag.id,
    challenge: tag.challenge,
  });
 };
@@ -522,7 +522,7 @@ api.addTag = function (req, res, next) {
    res.json(user.tags.toObject().map(tag => {
      return {
        name: tag.name,
-        id: tag._id,
+        id: tag.id,
        challenge: tag.challenge,
      }
    }));
@@ -532,7 +532,7 @@ api.addTag = function (req, res, next) {
 api.updateTag = function (req, res, next) {
  let user = res.locals.user;

-  let tag = user.tags.id(req.params.id);
+  let tag = _.find(res.locals.user.tags, {id: req.params.id});
  if (!tag) {
    return res.status(404).json({err: i18n.t('messageTagNotFound', req.language)});
  }
@@ -543,7 +543,7 @@ api.updateTag = function (req, res, next) {

    res.json({
      name: tag.name,
-      id: tag._id,
+      id: tag.id,
      challenge: tag.challenge,
    });
  });
@@ -566,7 +566,7 @@ api.sortTag = function (req, res, next) {
    res.json(user.tags.toObject().map(tag => {
      return {
        name: tag.name,
-        id: tag._id,
+        id: tag.id,
        challenge: tag.challenge,
      }
    }));
@@ -576,18 +576,17 @@ api.sortTag = function (req, res, next) {
 api.deleteTag = function (req, res, next) {
  let user = res.locals.user;

-  let tag = user.tags.id(req.params.id);
+  let tag = removeFromArray(user.tags, { id: req.params.id });
  if (!tag) {
    return res.status(404).json({err: i18n.t('messageTagNotFound', req.language)});
  }

-  tag.remove();

  Tasks.Task.update({
    userId: user._id,
  }, {
    $pull: {
-      tags: tag._id,
+      tags: tag.id,
    },
  }, {multi: true}).exec();

@@ -597,7 +596,7 @@ api.deleteTag = function (req, res, next) {
    res.json(user.tags.toObject().map(tag => {
      return {
        name: tag.name,
-        id: tag._id,
+        id: tag.id,
        challenge: tag.challenge,
      }
    }));
@@ -988,6 +987,7 @@ api.batchUpdate = function(req, res, next) {
        response = transformedData;

        response.todos = shared.preenTodos(response.todos);
+        response.wasModified = true;
        res.status(200).json(response);
      });
    // return only the version number
--- a/website/src/controllers/api-v3/tags.js
+++ b/website/src/controllers/api-v3/tags.js
@@ -5,6 +5,7 @@ import {
  NotFound,
 } from '../../libs/api-v3/errors';
 import _ from 'lodash';
+import { removeFromArray } from '../../libs/api-v3/collectionManipulators';

 let api = {};

@@ -134,9 +135,8 @@ api.deleteTag = {
    let validationErrors = req.validationErrors();
    if (validationErrors) throw validationErrors;

-    let tag = _.find(user.tags, {id: req.params.tagId});
+    let tag = removeFromArray(user.tags, { id: req.params.tagId });
    if (!tag) throw new NotFound(res.t('tagNotFound'));
-    tag.remove();

    // Remove from all the tasks TODO test
    await Tasks.Task.update({
--- a/website/src/libs/api-v3/baseModel.js
+++ b/website/src/libs/api-v3/baseModel.js
@@ -4,13 +4,15 @@ import objectPath from 'object-path'; // TODO use lodash's unset once v4 is out
 import _ from 'lodash';

 module.exports = function baseModel (schema, options = {}) {
-  schema.add({
-    _id: {
-      type: String,
-      default: uuid,
-      validate: [validator.isUUID, 'Invalid uuid.'],
-    },
-  });
+  if (options._id !== false) {
+    schema.add({
+      _id: {
+        type: String,
+        default: uuid,
+        validate: [validator.isUUID, 'Invalid uuid.'],
+      },
+    });
+  }

  if (options.timestamps) {
    schema.add({
--- a/website/src/models/challenge.js
+++ b/website/src/models/challenge.js
@@ -83,7 +83,7 @@ schema.methods.syncToUser = async function syncChallengeToUser (user) {

  // Sync tags
  let userTags = user.tags;
-  let i = _.findIndex(userTags, {_id: challenge._id});
+  let i = _.findIndex(userTags, {id: challenge._id});

  if (i !== -1) {
    if (userTags[i].name !== challenge.shortName) {
@@ -92,7 +92,7 @@ schema.methods.syncToUser = async function syncChallengeToUser (user) {
    }
  } else {
    userTags.push({
-      _id: challenge._id,
+      id: challenge._id,
      name: challenge.shortName,
      challenge: true,
    });
--- a/website/src/models/coupon.js
+++ b/website/src/models/coupon.js
@@ -11,6 +11,7 @@ import {
 } from '../libs/api-v3/errors';

 export let schema = new mongoose.Schema({
+  _id: {type: String, default: couponCode.generate},
  event: {type: String, enum: ['wondercon', 'google_6mo']},
  user: {type: String, ref: 'User'},
 }, {
@@ -20,14 +21,9 @@ export let schema = new mongoose.Schema({

 schema.plugin(baseModel, {
  timestamps: true,
+  _id: false,
 });

-// Add _id field after plugin to override default _id format
-schema.add({
-  _id: {type: String, default: couponCode.generate},
-});
-
-
 schema.statics.generate = async function generateCoupons (event, count = 1) {
  let coupons = _.times(count, () => {
    return {event};
--- a/website/src/models/emailUnsubscription.js
+++ b/website/src/models/emailUnsubscription.js
@@ -1,18 +1,14 @@
 import mongoose from 'mongoose';
-import common from '../../../common';
 import validator from 'validator';
+import baseModel from '../libs/api-v3/baseModel';

 // A collection used to store mailing list unsubscription for non registered email addresses
 export let schema = new mongoose.Schema({
-  _id: {
-    type: String,
-    default: common.uuid,
-  },
  email: {
    type: String,
    required: true,
    trim: true,
-    lowercase: true, // TODO migrate existing to lowerCase
+    lowercase: true,
    validator: [validator.isEmail, 'Invalid email.'],
  },
 }, {
@@ -20,4 +16,9 @@ export let schema = new mongoose.Schema({
  minimize: false, // So empty objects are returned
 });

+schema.plugin(baseModel, {
+  noSet: ['_id'],
+  timestamps: true,
+});
+
 export let model = mongoose.model('EmailUnsubscription', schema);
--- a/website/src/models/group.js
+++ b/website/src/models/group.js
@@ -44,7 +44,7 @@ export let schema = new Schema({
  */
  leaderOnly: { // restrict group actions to leader (members can't do them)
    challenges: {type: Boolean, default: false, required: true},
-    // invites: {type:Boolean, 'default':false}
+    // invites: {type: Boolean, default: false, required: true},
  },
  memberCount: {type: Number, default: 1},
  challengeCount: {type: Number, default: 0},
--- a/website/src/models/tag.js
+++ b/website/src/models/tag.js
@@ -6,7 +6,6 @@ import validator from 'validator';
 let Schema = mongoose.Schema;

 export let schema = new Schema({
-  _id: false, // use id not _id
  id: {
    type: String,
    default: uuid,
@@ -17,10 +16,12 @@ export let schema = new Schema({
 }, {
  strict: true,
  minimize: false, // So empty objects are returned
+  _id: false, // use id instead of _id
 });

 schema.plugin(baseModel, {
  noSet: ['_id', 'id', 'challenge'],
+  _id: false, // use id instead of _id
 });

 export let model = mongoose.model('Tag', schema);
--- a/website/src/models/user.js
+++ b/website/src/models/user.js
@@ -760,7 +760,7 @@ schema.methods.addTasksToUser = function addTasksToUser (tasks) {

  obj.tags = obj.tags.map(tag => {
    return {
-      id: tag._id,
+      id: tag.id,
      name: tag.name,
      challenge: tag.challenge,
    };