Api v3 Migration (WIP) (#7131)

* v3 migration: remove old code and polish user migration

* v3 migration: start to work on challenges

* wip v3 migration

* wip v3 migration: fix _id -> id for reminders, tags and checklists

common/script/libs/taskDefaults.js

@@ -23,7 +23,7 @@ module.exports = function taskDefaults (task = {}) {
     value: task.type === 'reward' ? 10 : 0,
     priority: 1,
     challenge: {},
-    reminders: {},
+    reminders: [],
     attribute: 'str',
     createdAt: new Date(), // TODO these are going to be overwritten by the server...
     updatedAt: new Date(),

migrations/api_v3/challenges.js

@@ -9,3 +9,241 @@
   memberCount must be checked
   prize must be >= 0
 */
+
+// A map of (original taskId) -> [new taskId in challenge, challendId] of tasks belonging to challenges where the task id had to change
+// This way later we can have use the right task.challenge.taskId in user's tasks
+var duplicateTasks = {};
+
+        // ... convert tasks to individual models
+        async.each(
+          challenge.dailys
+            .concat(challenge.habits)
+            .concat(challenge.rewards)
+            .concat(challenge.todos),
+          function(task, cb1) {
+
+            task = new TaskModel(task); // this should also fix dailies that wen to the habits array or vice-versa
+
+            TaskModel.findOne({_id: task._id}, function(err, taskSameId){
+              if(err) return cb1(err);
+
+              // We already have a task with the same id, change this one
+              // and will require special handling
+              if(taskSameId) {
+                task._id = shared.uuid();
+                task.legacyId = taskSameId._id; // We set this for challenge tasks too
+                // we use an array as the same task may have multiple duplicates
+                duplicateTasks[taskSameId._id] = duplicateTasks[taskSameId._id] || [];
+                duplicateTasks[taskSameId._id].push([task._id, challenge._id]);
+                console.log('Duplicate task ', taskSameId._id, 'challenge ', challenge._id, 'new id ', task._id);
+              }
+
+              task.save(function(err, savedTask){
+                if(err) return cb1(err);
+
+                challenge.tasksOrder[savedTask.type + 's'].push(savedTask._id);
+                cb1();
+              });
+            });
+          }, function(err) {
+            if(err) return cb(err);
+
+            var newChallenge = new NewChallengeModel(challenge); // This will make sure old data is discarded
+            newChallenge.save(function(err, chal){
+              if(err) return cb(err);
+              console.log('Processed: ', chal._id);
+              cb();
+            });
+          });
+      }, function(err) {
+        if(err) throw err;
+
+        processed = processed + challenges.length;
+        console.log('Processed ' + challenges.length + ' challenges.', 'Total: ' + processed);
+
+        if(lastChal && lastChal._id){
+          processChal(lastChal._id);
+        } else {
+          console.log('Done!');
+          // outputting the duplicate tasks
+          console.log(JSON.stringify(duplicateTasks, null, 4));
+        }
+      });
+    });
+};
+
+processChal();
+
+// Migrate users collection to new schema
+// This should run AFTER challenges migration
+
+// The console-stamp module must be installed (not included in package.json)
+
+// It requires two environment variables: MONGODB_OLD and MONGODB_NEW
+
+// Due to some big user profiles it needs more RAM than is allowed by default by v8 (arounf 1.7GB).
+// Run the script with --max-old-space-size=4096 to allow up to 4GB of RAM
+console.log('Starting migrations/api_v3/users.js.');
+
+require('babel-register');
+
+var Q = require('q');
+var MongoDB = require('mongodb');
+var nconf = require('nconf');
+var mongoose = require('mongoose');
+var _ = require('lodash');
+var uuid = require('uuid');
+var consoleStamp = require('console-stamp');
+
+// Add timestamps to console messages
+consoleStamp(console);
+
+// Initialize configuration
+require('../../website/src/libs/api-v3/setupNconf')();
+
+var MONGODB_OLD = nconf.get('MONGODB_OLD');
+var MONGODB_NEW = nconf.get('MONGODB_NEW');
+
+var MongoClient = MongoDB.MongoClient;
+
+mongoose.Promise = Q.Promise; // otherwise mongoose models won't work
+
+// Load new models
+var NewChallenge = require('../../website/src/models/challenge').model;
+var Tasks = require('../../website/src/models/task');
+
+// To be defined later when MongoClient connects
+var mongoDbOldInstance;
+var oldChallengeCollection;
+
+var mongoDbNewInstance;
+var newChallengeCollection;
+var newTaskCollection;
+
+var BATCH_SIZE = 1000;
+
+var processedChallenges = 0;
+var totoalProcessedTasks = 0;
+
+// Only process challenges that fall in a interval ie -> up to 0000-4000-0000-0000
+var AFTER_CHALLENGE_ID = nconf.get('AFTER_CHALLENGE_ID');
+var BEFORE_CHALLENGE_ID = nconf.get('BEFORE_CHALLENGE_ID');
+
+function processChallenges (afterId) {
+  var processedTasks = 0;
+  var lastChallenge = null;
+  var oldChallenges;
+
+  var query = {};
+
+  if (BEFORE_CHALLENGE_ID) {
+    query._id = {$lte: BEFORE_CHALLENGE_ID};
+  }
+
+  if ((afterId || AFTER_CHALLENGE_ID) && !query._id) {
+    query._id = {};
+  }
+
+  if (afterId) {
+    query._id.$gt = afterId;
+  } else if (AFTER_CHALLENGE_ID) {
+    query._id.$gt = AFTER_CHALLENGE_ID;
+  }
+
+  var batchInsertTasks = newTaskCollection.initializeUnorderedBulkOp();
+  var batchInsertChallenges = newChallengeCollection.initializeUnorderedBulkOp();
+
+  console.log(`Executing challenges query.\nMatching challenges after ${afterId ? afterId : AFTER_USER_ID} and before ${BEFORE_USER_ID} (included).`);
+
+  return oldChallengeCollection
+  .find(query)
+  .sort({_id: 1})
+  .limit(BATCH_SIZE)
+  .toArray()
+  .then(function (oldChallengesR) {
+    oldChallenges = oldChallengesR;
+
+    console.log(`Processing ${oldChallenges.length} challenges. Already processed ${processedChallenges} challenges and ${totoalProcessedTasks} tasks.`);
+
+    if (oldChallenges.length === BATCH_SIZE) {
+      lastChallenge = oldChallenges[oldChallenges.length - 1]._id;
+    }
+
+    oldChallenges.forEach(function (oldChallenge) {
+      var oldTasks = oldChallenge.habits.concat(oldChallenge.dailys).concat(oldChallenge.rewards).concat(oldChallenge.todos);
+      delete oldChallenge.habits;
+      delete oldChallenge.dailys;
+      delete oldChallenge.rewards;
+      delete oldChallenge.todos;
+
+      var newChallenge = new NewChallenge(oldChallenge);
+
+      oldTasks.forEach(function (oldTask) {
+        // TODO
+        oldTask._id = oldTask.id; // keep the old uuid unless duplicated
+        delete oldTask.id;
+
+        oldTask.challenge = oldTask.challenge || {};
+        oldTask.challenge.id = oldChallenge.id;
+
+        if (!oldTask.text) oldTask.text = 'task text'; // required
+        oldTask.tags = _.map(oldTask.tags, function (tagPresent, tagId) { // TODO used for challenges' tasks?
+          return tagPresent && tagId;
+        });
+
+        newChallenge.tasksOrder[`${oldTask.type}s`].push(oldTask._id);
+        if (oldTask.completed) oldTask.completed = false;
+
+        var newTask = new Tasks[oldTask.type](oldTask);
+
+        batchInsertTasks.insert(newTask.toObject());
+        processedTasks++;
+      });
+
+      batchInsertChallenges.insert(newChallenge.toObject());
+    });
+
+    console.log(`Saving ${oldChallenges.length} users and ${processedTasks} tasks.`);
+
+    return Q.all([
+      batchInsertChallenges.execute(),
+      batchInsertTasks.execute(),
+    ]);
+  })
+  .then(function () {
+    totoalProcessedTasks += processedTasks;
+    processedChallenges += oldChallenges.length;
+
+    console.log(`Saved ${oldChallenges.length} users and their tasks.`);
+
+    if (lastUser) {
+      return processChallenges(lastChallenge);
+    } else {
+      return console.log('Done!');
+    }
+  });
+}
+
+// Connect to the databases
+Q.all([
+  MongoClient.connect(MONGODB_OLD),
+  MongoClient.connect(MONGODB_NEW),
+])
+.then(function (result) {
+  var oldInstance = result[0];
+  var newInstance = result[1];
+
+  mongoDbOldInstance = oldInstance;
+  oldChallengeCollection = mongoDbOldInstance.collection('challenges');
+
+  mongoDbNewInstance = newInstance;
+  newChallengeCollection = mongoDbNewInstance.collection('challenges');
+  newTaskCollection = mongoDbNewInstance.collection('tasks');
+
+  console.log(`Connected with MongoClient to ${MONGODB_OLD} and ${MONGODB_NEW}.`);
+
+  return processChallenges();
+})
+.catch(function (err) {
+  console.error(err);
+});

migrations/api_v3/users.js

@@ -28,11 +28,13 @@ require('../../website/src/libs/api-v3/setupNconf')();
 var MONGODB_OLD = nconf.get('MONGODB_OLD');
 var MONGODB_NEW = nconf.get('MONGODB_NEW');
 
+var MongoClient = MongoDB.MongoClient;
+
 mongoose.Promise = Q.Promise; // otherwise mongoose models won't work
 
-// Load old and new models
+// Load new models
-//import { model as NewUser } from '../../website/src/models/user';
+var NewUser = require('../../website/src/models/user').model;
-//import * as Tasks from '../../website/src/models/task';
+var NewTasks = require('../../website/src/models/task');
 
 // To be defined later when MongoClient connects
 var mongoDbOldInstance;
@@ -47,16 +49,17 @@ var BATCH_SIZE = 1000;
 var processedUsers = 0;
 var totoalProcessedTasks = 0;
 
-// Only process users that fall in a interval ie -> 0000-4000-0000-0000
+// Only process users that fall in a interval ie up to -> 0000-4000-0000-0000
 var AFTER_USER_ID = nconf.get('AFTER_USER_ID');
 var BEFORE_USER_ID = nconf.get('BEFORE_USER_ID');
 
-/* TODO
+/* TODO compare old and new model
 - _id 9
 - challenges
 - groups
 - invitations
 - challenges' tasks
+- checklists from .id to ._id (reminders too!)
 */
 
 function processUsers (afterId) {
@@ -70,10 +73,14 @@ function processUsers (afterId) {
     query._id = {$lte: BEFORE_USER_ID};
   }
 
+  if ((afterId || AFTER_USER_ID) && !query._id) {
+    query._id = {};
+  }
+
   if (afterId) {
-    query._id = {$gt: afterId};
+    query._id.$gt = afterId;
   } else if (AFTER_USER_ID) {
-    query._id = {$gt: AFTER_USER_ID};
+    query._id.$gt = AFTER_USER_ID;
   }
 
   var batchInsertTasks = newTaskCollection.initializeUnorderedBulkOp();
@@ -95,17 +102,13 @@ function processUsers (afterId) {
       lastUser = oldUsers[oldUsers.length - 1]._id;
     }
 
-
     oldUsers.forEach(function (oldUser) {
       var oldTasks = oldUser.habits.concat(oldUser.dailys).concat(oldUser.rewards).concat(oldUser.todos);
-      oldUser.habits = oldUser.dailys = oldUser.rewards = oldUser.todos = undefined;
+      delete oldUser.habits;
+      delete oldUser.dailys;
+      delete oldUser.rewards;
+      delete oldUser.todos;
 
-      oldUser.challenges = [];
-      if (oldUser.invitations) {
-        oldUser.invitations.guilds = [];
-        oldUser.invitations.party = {};
-      }
-      oldUser.party = {};
       oldUser.tags = oldUser.tags.map(function (tag) {
         return {
           _id: tag.id,
@@ -114,37 +117,31 @@ function processUsers (afterId) {
         };
       });
 
-      oldUser.tasksOrder = {
+      var newUser = new NewUser(oldUser);
-        habits: [],
-        dailys: [],
-        rewards: [],
-        todos: [],
-      };
-
-      //let newUser = new NewUser(oldUser);
 
       oldTasks.forEach(function (oldTask) {
         oldTask._id = uuid.v4(); // create a new unique uuid
-        oldTask.userId = oldUser._id;
+        oldTask.userId = newUser._id;
         oldTask.legacyId = oldTask.id; // store the old task id
+        delete oldTask.id;
 
         oldTask.challenge = {};
-        if (!oldTask.text) oldTask.text = 'text';
+        if (!oldTask.text) oldTask.text = 'task text'; // required
         oldTask.tags = _.map(oldTask.tags, function (tagPresent, tagId) {
           return tagPresent && tagId;
         });
 
         if (oldTask.type !== 'todo' || (oldTask.type === 'todo' && !oldTask.completed)) {
-          oldUser.tasksOrder[`${oldTask.type}s`].push(oldTask._id);
+          newUser.tasksOrder[`${oldTask.type}s`].push(oldTask._id);
         }
 
-        //let newTask = new Tasks[oldTask.type](oldTask);
+        var newTask = new NewTasks[oldTask.type](oldTask);
 
-        batchInsertTasks.insert(oldTask);
+        batchInsertTasks.insert(newTask.toObject());
         processedTasks++;
       });
 
-      batchInsertUsers.insert(oldUser);
+      batchInsertUsers.insert(newUser.toObject());
     });
 
     console.log(`Saving ${oldUsers.length} users and ${processedTasks} tasks.`);
@@ -171,126 +168,28 @@ function processUsers (afterId) {
 /*
 
 TODO var challengeTasksChangedId = {};
-... given a user
 
-let processed = 0;
+tasksArr.forEach(function(task){
-let batchSize = 1000;
+  task.challenge = task.challenge || {};
-
+  if(task.challenge.id) {
-var db; // defined later by MongoClient
+    // If challengeTasksChangedId[task._id] then we got on of the duplicates from the challenges migration
-var dbNewUsers;
+    if (challengeTasksChangedId[task.legacyId]) {
-var dbTasks;
+      var res = _.find(challengeTasksChangedId[task.legacyId], function(arr){
-
+        return arr[1] === task.challenge.id;
-var processUser = function(gt) {
-  var query = {
-    _id: {}
-  };
-  if(gt) query._id.$gt = gt;
-
-  console.log('Launching query', query);
-
-  // take batchsize docs from users and process them
-  OldUserModel
-    .find(query)
-    .lean() // Use plain JS objects as old user data won't match the new model
-    .limit(batchSize)
-    .sort({_id: 1})
-    .exec(function(err, users) {
-      if(err) throw err;
-
-      console.log('Processing ' + users.length + ' users.', 'Already processed: ' + processed);
-
-      var lastUser = null;
-      if(users.length === batchSize){
-        lastUser = users[users.length - 1];
-      }
-
-      var tasksToSave = 0;
-
-      // Initialize batch operation for later
-      var batchInsertUsers = dbNewUsers.initializeUnorderedBulkOp();
-      var batchInsertTasks = dbTasks.initializeUnorderedBulkOp();
-
-      users.forEach(function(user){
-        // user obj is a plain js object because we used .lean()
-
-        // add tasks order arrays
-        user.tasksOrder = {
-          habits: [],
-          rewards: [],
-          todos: [],
-          dailys: []
-        };
-
-        // ... convert tasks to individual models
-
-        var tasksArr = user.dailys
-                          .concat(user.habits)
-                          .concat(user.todos)
-                          .concat(user.rewards);
-
-        // free memory?
-        user.dailys = user.habits = user.todos = user.rewards = undefined;
-
-        tasksArr.forEach(function(task){
-          task.userId = user._id;
-
-          task._id = shared.uuid(); // we rely on these to be unique... hopefully!
-          task.legacyId = task.id;
-          task.id = undefined;
-
-          task.challenge = task.challenge || {};
-          if(task.challenge.id) {
-            // If challengeTasksChangedId[task._id] then we got on of the duplicates from the challenges migration
-            if (challengeTasksChangedId[task.legacyId]) {
-              var res = _.find(challengeTasksChangedId[task.legacyId], function(arr){
-                return arr[1] === task.challenge.id;
-              });
-
-              // If res, id changed, otherwise matches the original one
-              task.challenge.taskId = res ? res[0] : task.legacyId;
-            } else {
-              task.challenge.taskId = task.legacyId;
-            }
-          }
-
-          if(!task.type) console.log('Task without type ', task._id, ' user ', user._id);
-
-          task = new TaskModel(task); // this should also fix dailies that wen to the habits array or vice-versa
-          user.tasksOrder[task.type + 's'].push(task._id);
-          tasksToSave++;
-          batchInsertTasks.insert(task.toObject());
-        });
-
-        batchInsertUsers.insert((new NewUserModel(user)).toObject());
       });
 
-      console.log('Saving', users.length, 'users and', tasksToSave, 'tasks');
+      // If res, id changed, otherwise matches the original one
+      task.challenge.taskId = res ? res[0] : task.legacyId;
+    } else {
+      task.challenge.taskId = task.legacyId;
+    }
+  }
 
-      // Save in the background and dispatch another processUser();
+  if(!task.type) console.log('Task without type ', task._id, ' user ', user._id);
-
+});
-      batchInsertUsers.execute(function(err, result){
-        if(err) throw err // we can't simply accept errors
-        console.log('Saved', result.nInserted, 'users')
-      });
-
-      batchInsertTasks.execute(function(err, result){
-        if(err) throw err // we can't simply accept errors
-        console.log('Saved', result.nInserted, 'tasks')
-      });
-
-      processed = processed + users.length;
-      if(lastUser && lastUser._id){
-        processUser(lastUser._id);
-      } else {
-        console.log('Done!');
-      }
-    });
-};
 */
 
 // Connect to the databases
-var MongoClient = MongoDB.MongoClient;
-
 Q.all([
   MongoClient.connect(MONGODB_OLD),
   MongoClient.connect(MONGODB_NEW),

test/api/v3/integration/tags/DELETE-tags_id.test.js

@@ -14,7 +14,7 @@ describe('DELETE /tags/:tagId', () => {
     let tag = await user.post('/tags', {name: tagName});
     let numberOfTags = (await user.get('/tags')).length;
 
-    await user.del(`/tags/${tag._id}`);
+    await user.del(`/tags/${tag.id}`);
 
     let tags = await user.get('/tags');
     let tagNames = tags.map((t) => {

test/api/v3/integration/tags/GET-tags_id.test.js

@@ -11,7 +11,7 @@ describe('GET /tags/:tagId', () => {
 
   it('returns a tag given it\'s id', async () => {
     let createdTag = await user.post('/tags', {name: 'Tag 1'});
-    let tag = await user.get(`/tags/${createdTag._id}`);
+    let tag = await user.get(`/tags/${createdTag.id}`);
 
     expect(tag).to.deep.equal(createdTag);
   });

test/api/v3/integration/tags/POST-tags.test.js

@@ -16,7 +16,7 @@ describe('POST /tags', () => {
       ignored: false,
     });
 
-    let tag = await user.get(`/tags/${createdTag._id}`);
+    let tag = await user.get(`/tags/${createdTag.id}`);
 
     expect(tag.name).to.equal(tagName);
     expect(tag.ignored).to.not.exist;

test/api/v3/integration/tags/PUT-tags_id.test.js

@@ -12,12 +12,12 @@ describe('PUT /tags/:tagId', () => {
   it('updates a tag given it\'s id', async () => {
     let updatedTagName = 'Tag updated';
     let createdTag = await user.post('/tags', {name: 'Tag 1'});
-    let updatedTag = await user.put(`/tags/${createdTag._id}`, {
+    let updatedTag = await user.put(`/tags/${createdTag.id}`, {
       name: updatedTagName,
       ignored: true,
     });
 
-    createdTag = await user.get(`/tags/${updatedTag._id}`);
+    createdTag = await user.get(`/tags/${updatedTag.id}`);
 
     expect(updatedTag.name).to.equal(updatedTagName);
     expect(updatedTag.ignored).to.not.exist;

test/api/v3/integration/tasks/POST-tasks_user.test.js

@@ -154,14 +154,14 @@ describe('POST /tasks/user', () => {
         text: 'test habit',
         type: 'habit',
         reminders: [
-          {_id: id1, startDate: new Date(), time: new Date()},
+          {id: id1, startDate: new Date(), time: new Date()},
         ],
       });
 
       expect(task.reminders).to.be.an('array');
       expect(task.reminders.length).to.eql(1);
       expect(task.reminders[0]).to.be.an('object');
-      expect(task.reminders[0]._id).to.eql(id1);
+      expect(task.reminders[0].id).to.eql(id1);
       expect(task.reminders[0].startDate).to.be.a('string'); // json doesn't have dates
       expect(task.reminders[0].time).to.be.a('string');
     });
@@ -345,7 +345,7 @@ describe('POST /tasks/user', () => {
       expect(task.checklist[0]).to.be.an('object');
       expect(task.checklist[0].text).to.eql('checklist');
       expect(task.checklist[0].completed).to.eql(false);
-      expect(task.checklist[0]._id).to.be.a('string');
+      expect(task.checklist[0].id).to.be.a('string');
     });
   });
 
@@ -487,7 +487,7 @@ describe('POST /tasks/user', () => {
       expect(task.checklist[0]).to.be.an('object');
       expect(task.checklist[0].text).to.eql('checklist');
       expect(task.checklist[0].completed).to.eql(false);
-      expect(task.checklist[0]._id).to.be.a('string');
+      expect(task.checklist[0].id).to.be.a('string');
     });
   });
 

test/api/v3/integration/tasks/PUT-tasks_id.test.js

@@ -80,14 +80,14 @@ describe('PUT /tasks/:id', () => {
 
       let savedDaily = await user.put(`/tasks/${daily._id}`, {
         reminders: [
-          {_id: id1, time: new Date(), startDate: new Date()},
+          {id: id1, time: new Date(), startDate: new Date()},
-          {_id: id2, time: new Date(), startDate: new Date()},
+          {id: id2, time: new Date(), startDate: new Date()},
         ],
       });
 
       expect(savedDaily.reminders.length).to.equal(2);
-      expect(savedDaily.reminders[0]._id).to.equal(id1);
+      expect(savedDaily.reminders[0].id).to.equal(id1);
-      expect(savedDaily.reminders[1]._id).to.equal(id2);
+      expect(savedDaily.reminders[1].id).to.equal(id2);
     });
   });
 

test/api/v3/integration/tasks/challenges/DELETE-tasks_challenge_challengeId_checklist_itemId.test.js

@@ -51,7 +51,7 @@ describe('DELETE /tasks/:taskId/checklist/:itemId', () => {
 
     let anotherUser = await generateUser();
 
-    await expect(anotherUser.del(`/tasks/${task._id}/checklist/${savedTask.checklist[0]._id}`))
+    await expect(anotherUser.del(`/tasks/${task._id}/checklist/${savedTask.checklist[0].id}`))
       .to.eventually.be.rejected.and.eql({
         code: 401,
         error: 'NotAuthorized',
@@ -67,7 +67,7 @@ describe('DELETE /tasks/:taskId/checklist/:itemId', () => {
 
     let savedTask = await user.post(`/tasks/${task._id}/checklist`, {text: 'Checklist Item 1', completed: false});
 
-    await user.del(`/tasks/${task._id}/checklist/${savedTask.checklist[0]._id}`);
+    await user.del(`/tasks/${task._id}/checklist/${savedTask.checklist[0].id}`);
     savedTask = await user.get(`/tasks/${task._id}`);
 
     expect(savedTask.checklist.length).to.equal(0);
@@ -81,7 +81,7 @@ describe('DELETE /tasks/:taskId/checklist/:itemId', () => {
 
     let savedTask = await user.post(`/tasks/${task._id}/checklist`, {text: 'Checklist Item 1', completed: false});
 
-    await user.del(`/tasks/${task._id}/checklist/${savedTask.checklist[0]._id}`);
+    await user.del(`/tasks/${task._id}/checklist/${savedTask.checklist[0].id}`);
     savedTask = await user.get(`/tasks/${task._id}`);
 
     expect(savedTask.checklist.length).to.equal(0);

test/api/v3/integration/tasks/challenges/POST-tasks_challenge_challengeId_taskId_checklist.test.js

@@ -62,8 +62,8 @@ describe('POST /tasks/:taskId/checklist/', () => {
     expect(savedTask.checklist.length).to.equal(1);
     expect(savedTask.checklist[0].text).to.equal('Checklist Item 1');
     expect(savedTask.checklist[0].completed).to.equal(false);
-    expect(savedTask.checklist[0]._id).to.be.a('string');
+    expect(savedTask.checklist[0].id).to.be.a('string');
-    expect(savedTask.checklist[0]._id).to.not.equal('123');
+    expect(savedTask.checklist[0].id).to.not.equal('123');
     expect(savedTask.checklist[0].ignored).to.be.an('undefined');
   });
 
@@ -82,8 +82,8 @@ describe('POST /tasks/:taskId/checklist/', () => {
     expect(savedTask.checklist.length).to.equal(1);
     expect(savedTask.checklist[0].text).to.equal('Checklist Item 1');
     expect(savedTask.checklist[0].completed).to.equal(false);
-    expect(savedTask.checklist[0]._id).to.be.a('string');
+    expect(savedTask.checklist[0].id).to.be.a('string');
-    expect(savedTask.checklist[0]._id).to.not.equal('123');
+    expect(savedTask.checklist[0].id).to.not.equal('123');
     expect(savedTask.checklist[0].ignored).to.be.an('undefined');
   });
 

test/api/v3/integration/tasks/challenges/PUT-tasks_challenge_challengeId_tasksId_checklist_itemId.test.js

@@ -48,7 +48,7 @@ describe('PUT /tasks/:taskId/checklist/:itemId', () => {
 
     let anotherUser = await generateUser();
 
-    await expect(anotherUser.put(`/tasks/${task._id}/checklist/${savedTask.checklist[0]._id}`, {
+    await expect(anotherUser.put(`/tasks/${task._id}/checklist/${savedTask.checklist[0].id}`, {
       text: 'updated',
       completed: true,
       _id: 123, // ignored
@@ -71,7 +71,7 @@ describe('PUT /tasks/:taskId/checklist/:itemId', () => {
       completed: false,
     });
 
-    savedTask = await user.put(`/tasks/${task._id}/checklist/${savedTask.checklist[0]._id}`, {
+    savedTask = await user.put(`/tasks/${task._id}/checklist/${savedTask.checklist[0].id}`, {
       text: 'updated',
       completed: true,
       _id: 123, // ignored
@@ -80,7 +80,7 @@ describe('PUT /tasks/:taskId/checklist/:itemId', () => {
     expect(savedTask.checklist.length).to.equal(1);
     expect(savedTask.checklist[0].text).to.equal('updated');
     expect(savedTask.checklist[0].completed).to.equal(true);
-    expect(savedTask.checklist[0]._id).to.not.equal('123');
+    expect(savedTask.checklist[0].id).to.not.equal('123');
   });
 
   it('updates a checklist item on todos', async () => {
@@ -94,7 +94,7 @@ describe('PUT /tasks/:taskId/checklist/:itemId', () => {
       completed: false,
     });
 
-    savedTask = await user.put(`/tasks/${task._id}/checklist/${savedTask.checklist[0]._id}`, {
+    savedTask = await user.put(`/tasks/${task._id}/checklist/${savedTask.checklist[0].id}`, {
       text: 'updated',
       completed: true,
       _id: 123, // ignored
@@ -103,7 +103,7 @@ describe('PUT /tasks/:taskId/checklist/:itemId', () => {
     expect(savedTask.checklist.length).to.equal(1);
     expect(savedTask.checklist[0].text).to.equal('updated');
     expect(savedTask.checklist[0].completed).to.equal(true);
-    expect(savedTask.checklist[0]._id).to.not.equal('123');
+    expect(savedTask.checklist[0].id).to.not.equal('123');
   });
 
   it('fails on habits', async () => {

test/api/v3/integration/tasks/checklists/DELETE-tasks_taskId_checklist_itemId.test.js

@@ -19,7 +19,7 @@ describe('DELETE /tasks/:taskId/checklist/:itemId', () => {
 
     let savedTask = await user.post(`/tasks/${task._id}/checklist`, {text: 'Checklist Item 1', completed: false});
 
-    await user.del(`/tasks/${task._id}/checklist/${savedTask.checklist[0]._id}`);
+    await user.del(`/tasks/${task._id}/checklist/${savedTask.checklist[0].id}`);
     savedTask = await user.get(`/tasks/${task._id}`);
 
     expect(savedTask.checklist.length).to.equal(0);

test/api/v3/integration/tasks/checklists/POST-tasks_taskId_checklist.test.js

@@ -26,8 +26,8 @@ describe('POST /tasks/:taskId/checklist/', () => {
     expect(savedTask.checklist.length).to.equal(1);
     expect(savedTask.checklist[0].text).to.equal('Checklist Item 1');
     expect(savedTask.checklist[0].completed).to.equal(false);
-    expect(savedTask.checklist[0]._id).to.be.a('string');
+    expect(savedTask.checklist[0].id).to.be.a('string');
-    expect(savedTask.checklist[0]._id).to.not.equal('123');
+    expect(savedTask.checklist[0].id).to.not.equal('123');
     expect(savedTask.checklist[0].ignored).to.be.an('undefined');
   });
 

test/api/v3/integration/tasks/checklists/POST-tasks_taskId_checklist_itemId_score.test.js

@@ -22,7 +22,7 @@ describe('POST /tasks/:taskId/checklist/:itemId/score', () => {
       completed: false,
     });
 
-    savedTask = await user.post(`/tasks/${task._id}/checklist/${savedTask.checklist[0]._id}/score`);
+    savedTask = await user.post(`/tasks/${task._id}/checklist/${savedTask.checklist[0].id}/score`);
 
     expect(savedTask.checklist.length).to.equal(1);
     expect(savedTask.checklist[0].completed).to.equal(true);

test/api/v3/integration/tasks/checklists/PUT-tasks_taskId_checklist_itemId.test.js

@@ -22,7 +22,7 @@ describe('PUT /tasks/:taskId/checklist/:itemId', () => {
       completed: false,
     });
 
-    savedTask = await user.put(`/tasks/${task._id}/checklist/${savedTask.checklist[0]._id}`, {
+    savedTask = await user.put(`/tasks/${task._id}/checklist/${savedTask.checklist[0].id}`, {
       text: 'updated',
       completed: true,
       _id: 123, // ignored
@@ -31,7 +31,7 @@ describe('PUT /tasks/:taskId/checklist/:itemId', () => {
     expect(savedTask.checklist.length).to.equal(1);
     expect(savedTask.checklist[0].text).to.equal('updated');
     expect(savedTask.checklist[0].completed).to.equal(true);
-    expect(savedTask.checklist[0]._id).to.not.equal('123');
+    expect(savedTask.checklist[0].id).to.not.equal('123');
   });
 
   it('fails on habits', async () => {

test/api/v3/integration/tasks/tags/DELETE-tasks_taskId_tags_tagId.test.js

@@ -19,8 +19,8 @@ describe('DELETE /tasks/:taskId/tags/:tagId', () => {
 
     let tag = await user.post('/tags', {name: 'Tag 1'});
 
-    await user.post(`/tasks/${task._id}/tags/${tag._id}`);
+    await user.post(`/tasks/${task._id}/tags/${tag.id}`);
-    await user.del(`/tasks/${task._id}/tags/${tag._id}`);
+    await user.del(`/tasks/${task._id}/tags/${tag.id}`);
 
     let updatedTask = await user.get(`/tasks/${task._id}`);
 

test/api/v3/integration/tasks/tags/POST-tasks_taskId_tags_tagId.test.js

@@ -18,9 +18,9 @@ describe('POST /tasks/:taskId/tags/:tagId', () => {
     });
 
     let tag = await user.post('/tags', {name: 'Tag 1'});
-    let savedTask = await user.post(`/tasks/${task._id}/tags/${tag._id}`);
+    let savedTask = await user.post(`/tasks/${task._id}/tags/${tag.id}`);
 
-    expect(savedTask.tags[0]).to.equal(tag._id);
+    expect(savedTask.tags[0]).to.equal(tag.id);
   });
 
   it('does not add a tag to a task twice', async () => {
@@ -31,9 +31,9 @@ describe('POST /tasks/:taskId/tags/:tagId', () => {
 
     let tag = await user.post('/tags', {name: 'Tag 1'});
 
-    await user.post(`/tasks/${task._id}/tags/${tag._id}`);
+    await user.post(`/tasks/${task._id}/tags/${tag.id}`);
 
-    await expect(user.post(`/tasks/${task._id}/tags/${tag._id}`)).to.eventually.be.rejected.and.eql({
+    await expect(user.post(`/tasks/${task._id}/tags/${tag.id}`)).to.eventually.be.rejected.and.eql({
       code: 400,
       error: 'BadRequest',
       message: t('alreadyTagged'),

test/common/ops/updateTask.js

@@ -13,7 +13,7 @@ describe('shared.ops.updateTask', () => {
       ],
 
       reminders: [{
-        _id: '123',
+        id: '123',
         startDate: now,
         time: now,
       }],
@@ -29,7 +29,7 @@ describe('shared.ops.updateTask', () => {
         checklist: [{
           completed: false,
           text: 'item',
-          _id: '123',
+          id: '123',
         }],
       },
     });
@@ -41,10 +41,10 @@ describe('shared.ops.updateTask', () => {
     expect(res.checklist).to.eql([{
       completed: false,
       text: 'item',
-      _id: '123',
+      id: '123',
     }]);
     expect(res.reminders).to.eql([{
-      _id: '123',
+      id: '123',
       startDate: now,
       time: now,
     }]);

website/src/controllers/api-v3/tags.js

@@ -72,7 +72,7 @@ api.getTag = {
     let validationErrors = req.validationErrors();
     if (validationErrors) throw validationErrors;
 
-    let tag = user.tags.id(req.params.tagId);
+    let tag = _.find(user.tags, {id: req.params.tagId});
     if (!tag) throw new NotFound(res.t('tagNotFound'));
     res.respond(200, tag);
   },
@@ -102,13 +102,13 @@ api.updateTag = {
     let validationErrors = req.validationErrors();
     if (validationErrors) throw validationErrors;
 
-    let tag = user.tags.id(tagId);
+    let tag = _.find(user.tags, {id: tagId});
     if (!tag) throw new NotFound(res.t('tagNotFound'));
 
     _.merge(tag, Tag.sanitize(req.body));
 
     let savedUser = await user.save();
-    res.respond(200, savedUser.tags.id(tagId));
+    res.respond(200, _.find(savedUser.tags, {id: tagId}));
   },
 };
 
@@ -134,7 +134,7 @@ api.deleteTag = {
     let validationErrors = req.validationErrors();
     if (validationErrors) throw validationErrors;
 
-    let tag = user.tags.id(req.params.tagId);
+    let tag = _.find(user.tags, {id: req.params.tagId});
     if (!tag) throw new NotFound(res.t('tagNotFound'));
     tag.remove();
 
@@ -143,7 +143,7 @@ api.deleteTag = {
       userId: user._id,
     }, {
       $pull: {
-        tags: tag._id,
+        tags: tag.id,
       },
     }, {multi: true}).exec();
 

website/src/controllers/api-v3/tasks.js

@@ -518,7 +518,7 @@ api.addChecklistItem = {
 
     if (task.type !== 'daily' && task.type !== 'todo') throw new BadRequest(res.t('checklistOnlyDailyTodo'));
 
-    task.checklist.push(Tasks.Task.sanitizeChecklist(req.body)); // TODO why not allow to supply _id on creation?
+    task.checklist.push(Tasks.Task.sanitizeChecklist(req.body));
     let savedTask = await task.save();
 
     res.respond(200, savedTask);
@@ -558,7 +558,7 @@ api.scoreCheckListItem = {
     if (!task) throw new NotFound(res.t('taskNotFound'));
     if (task.type !== 'daily' && task.type !== 'todo') throw new BadRequest(res.t('checklistOnlyDailyTodo'));
 
-    let item = _.find(task.checklist, {_id: req.params.itemId});
+    let item = _.find(task.checklist, {id: req.params.itemId});
 
     if (!item) throw new NotFound(res.t('checklistItemNotFound'));
     item.completed = !item.completed;
@@ -608,7 +608,7 @@ api.updateChecklistItem = {
     }
     if (task.type !== 'daily' && task.type !== 'todo') throw new BadRequest(res.t('checklistOnlyDailyTodo'));
 
-    let item = _.find(task.checklist, {_id: req.params.itemId});
+    let item = _.find(task.checklist, {id: req.params.itemId});
     if (!item) throw new NotFound(res.t('checklistItemNotFound'));
 
     _.merge(item, Tasks.Task.sanitizeChecklist(req.body));
@@ -659,7 +659,7 @@ api.removeChecklistItem = {
     }
     if (task.type !== 'daily' && task.type !== 'todo') throw new BadRequest(res.t('checklistOnlyDailyTodo'));
 
-    let hasItem = removeFromArray(task.checklist, { _id: req.params.itemId });
+    let hasItem = removeFromArray(task.checklist, { id: req.params.itemId });
     if (!hasItem) throw new NotFound(res.t('checklistItemNotFound'));
 
     let savedTask = await task.save();
@@ -687,7 +687,7 @@ api.addTagToTask = {
     let user = res.locals.user;
 
     req.checkParams('taskId', res.t('taskIdRequired')).notEmpty().isUUID();
-    let userTags = user.tags.map(tag => tag._id);
+    let userTags = user.tags.map(tag => tag.id);
     req.checkParams('tagId', res.t('tagIdRequired')).notEmpty().isUUID().isIn(userTags);
 
     let validationErrors = req.validationErrors();

website/src/controllers/top-level/pages.js

@@ -1,8 +1,8 @@
 import locals from '../../middlewares/api-v3/locals';
 import _ from 'lodash';
-import Remarkable from 'remarkable';
+import markdownIt from 'markdown-it';
 
-const md = new Remarkable({
+const md = markdownIt({
   html: true,
 });
 

website/src/models/tag.js

@@ -1,9 +1,17 @@
 import mongoose from 'mongoose';
 import baseModel from '../libs/api-v3/baseModel';
+import { v4 as uuid } from 'uuid';
+import validator from 'validator';
 
 let Schema = mongoose.Schema;
 
 export let schema = new Schema({
+  _id: false, // use id not _id
+  id: {
+    type: String,
+    default: uuid,
+    validate: [validator.isUUID, 'Invalid uuid.'],
+  },
   name: {type: String, required: true},
   challenge: {type: String},
 }, {
@@ -12,7 +20,7 @@ export let schema = new Schema({
 });
 
 schema.plugin(baseModel, {
-  noSet: ['_id', 'challenge'],
+  noSet: ['_id', 'id', 'challenge'],
 });
 
 export let model = mongoose.model('Tag', schema);

website/src/models/task.js

@@ -45,7 +45,8 @@ export let TaskSchema = new Schema({
   },
 
   reminders: [{
-    _id: {type: String, validate: [validator.isUUID, 'Invalid uuid.'], default: shared.uuid, required: true},
+    _id: false,
+    id: {type: String, validate: [validator.isUUID, 'Invalid uuid.'], default: shared.uuid, required: true},
     startDate: {type: Date, required: true},
     time: {type: Date, required: true},
   }],
@@ -67,15 +68,15 @@ TaskSchema.plugin(baseModel, {
   timestamps: true,
 });
 
-// Sanitize checklist objects (disallowing _id)
+// Sanitize checklist objects (disallowing id)
 TaskSchema.statics.sanitizeChecklist = function sanitizeChecklist (checklistObj) {
-  delete checklistObj._id;
+  delete checklistObj.id;
   return checklistObj;
 };
 
 // Sanitize reminder objects (disallowing id)
 TaskSchema.statics.sanitizeReminder = function sanitizeReminder (reminderObj) {
-  delete reminderObj.id; // TODO convert to _id?
+  delete reminderObj.id;
   return reminderObj;
 };
 
@@ -159,8 +160,9 @@ let dailyTodoSchema = () => {
     collapseChecklist: {type: Boolean, default: false},
     checklist: [{
       completed: {type: Boolean, default: false},
-      text: {type: String, required: true},
+      text: {type: String, required: false, default: ''}, // required:false because it can be empty on creation
-      _id: {type: String, default: shared.uuid, validate: [validator.isUUID, 'Invalid uuid.']},
+      _id: false,
+      id: {type: String, default: shared.uuid, validate: [validator.isUUID, 'Invalid uuid.']},
     }],
   };
 };