diff --git a/migrations/20160529_fix_challenges.js b/migrations/20160529_fix_challenges.js new file mode 100644 index 0000000000..979a314e25 --- /dev/null +++ b/migrations/20160529_fix_challenges.js @@ -0,0 +1,276 @@ +'use strict'; + +/**************************************** + * Reason: After the api v3 maintenance migration, some challenge tasks + * became unlinked from their challenges. We're still not sure why, + * but this re-links them + * + * Note: We ran this on a local backup of the DB, and from that, grabbed + * the ids of the tasks that could be fixed and the updates that would + * be applied to them. We only ran the `updateTasks` promise task. + ***************************************/ + +const authorName = 'Blade'; +const authorUuid = '75f270e8-c5db-4722-a5e6-a83f1b23f76b'; + +global.Promise = require('bluebird'); +const MongoClient = require('mongodb').MongoClient; +const chalk = require('chalk'); +const TaskQueue = require('cwait').TaskQueue; + +const logger = { + info: _logger('info', 'cyan'), + success: _logger('info', 'green'), + error: _logger('error', 'red'), + log: _logger('log', 'white'), + warn: _logger('warn', 'yellow'), +} + +function _logger (type, color) { + return function () { + let args = Array.from(arguments).map(arg => chalk[color](arg)); + console[type].apply(null, args); + } +} + +// PROD: Enable prod db +// const NODE_DB_URI = 'mongodb://username:password@dsXXXXXX-a0.mlab.com:XXXXX,dsXXXXXX-a1.mlab.com:XXXXX/habitica?replicaSet=rs-dsXXXXXX'; +const NODE_DB_URI = 'mongodb://localhost/new-prod-copy'; + +// Cached ids from running the findBrokenChallengeTasks query on a local copy of the db +// These are all the ids that _are_ fixable +const TASK_IDS = require('../fixable_task_ids.json'); +const TASK_UPDATE_DATA = require('../challenge_fixes.json'); + +let db; +let count = 0; + +var timer = setInterval(function(){ + count++; + if (count % 30 === 0) { + logger.warn('Process has been running for', count / 60, 'minutes'); + } +}, 1000); + +connectToDb() + // .then(findBrokenChallengeTasks) + // .then(getDataFromTasks) + // .then(getUserChallenges) + // .then(getChallengeTasks) + // .then(correctUserTasks) + .then(updateTasks) + .then(closeDb) + .catch(reportError) + +function connectToDb () { + return new Promise((resolve, reject) => { + MongoClient.connect(NODE_DB_URI, (err, database) => { + if (err) { + logger.error('Uh oh... Problem connecting to the new database'); + return reject(err); + } + + logger.success('Connected to the database'); + + db = database; + + resolve(db); + }); + }); +} + +function reportError (err) { + logger.error('Uh oh, an error occurred'); + closeDb(); + throw err; +} + +function unique (array) { + return Array.from(new Set(array)); +} + +function findBrokenChallengeTasks () { + logger.info('Looking for broken tasks...'); + + // return db.collection('tasks').find({'challenge.broken': 'CHALLENGE_TASK_NOT_FOUND'}).toArray() + return db.collection('tasks').find({'_id': { '$in': TASK_IDS }}).toArray() + .then((tasks) => { + logger.success('Found', tasks.length, 'broken tasks.'); + return Promise.resolve(tasks); + }); +} + +function getDataFromTasks (tasks) { + logger.info('Collecting data about the tasks...'); + + let userTasks = {}; + + tasks.forEach((task) => { + let userId = task.userId; + + if (!userTasks[userId]) { + userTasks[userId] = []; + } + userTasks[userId].push(task); + }); + + let users = unique(tasks.map(task => task.userId)); + + return Promise.resolve({ + users, + userTasks, + tasks, + }); +} + +function getUserChallenges (data) { + logger.info('Collecting user challenges...'); + + return db.collection('users').find({_id: { '$in': data.users }}, {challenges: 1}).toArray().then((docs) => { + logger.success('Found', docs.length, 'users from broken challenge tasks.'); + + let challenges = []; + docs.forEach((user) => { + challenges.push.apply(challenges, user.challenges); + }); + + challenges = unique(challenges); + + let userChallenges = {}; + + docs.forEach((user) => { + let userId = user._id; + if (!userChallenges[userId]) { + userChallenges[userId] = []; + } + userChallenges[userId].push.apply(userChallenges[userId], user.challenges); + }); + + data.userChallenges = userChallenges; + data.challenges = challenges; + + logger.success('Found', challenges.length, 'unique challenges.'); + + return Promise.resolve(data); + }); +} + +function getChallengeTasks (data) { + logger.info('Looking up original challenge tasks...'); + + return db.collection('tasks').find({'userId': null, 'challenge.id': { '$in': data.challenges }}, [ 'text', 'type', 'challenge', '_legacyId' ]).toArray().then((docs) => { + logger.success('Found', docs.length, 'challenge tasks.'); + + let challengeTasks = {}; + + docs.forEach((task) => { + let chalId = task.challenge.id; + if (!challengeTasks[chalId]) { + challengeTasks[chalId] = []; + } + challengeTasks[chalId].push(task); + }); + data.challengeTasks = challengeTasks; + + return Promise.resolve(data); + }); +} + +function correctUserTasks (data) { + logger.info('Correcting user tasks...'); + + let tasksToUpdate = {}; + let duplicateTasks = {}; + + for (let user in data.userChallenges) { + if (user === authorUuid) { + logger.success('Processing data for', authorName); + } + if (data.userChallenges.hasOwnProperty(user)) { + let challenges = data.userChallenges[user]; + + challenges.forEach((chal) => { + let challengeTasks = data.challengeTasks[chal]; + let userTasks = data.userTasks[user]; + + if (challengeTasks) { + challengeTasks.forEach((challengeTask) => { + let text = challengeTask.text; + let type = challengeTask.type; + let taskId = challengeTask._id; + let legacyId = challengeTask._legacyId; + + let foundTask = userTasks.find((task) => { + return TASK_IDS.indexOf(task._id) > -1 && task._legacyId === legacyId && task.type === type && task.text === text; + }) + + if (foundTask && !tasksToUpdate[foundTask._id]) { + tasksToUpdate[foundTask._id] = { + id: chal, + broken: null, + taskId, + } + } else if (foundTask && taskId !== tasksToUpdate[foundTask._id].taskId) { + logger.error('Duplicate task found, id:', foundTask._id); + duplicateTasks[foundTask._id] = duplicateTasks[foundTask._id] || [tasksToUpdate[foundTask._id].taskId]; + duplicateTasks[foundTask._id].push(taskId); + } + }); + } + }); + } + } + + let numberOfDuplicateTasksFound = Object.keys(duplicateTasks).length; + + if (numberOfDuplicateTasksFound > 0) { + logger.error('Found', numberOfDuplicateTasksFound, 'duplicate taks'); + } + + + data.tasksToUpdate = tasksToUpdate; + + return Promise.resolve(data); +} + +function updateTasks (data) { + let tasksToUpdate = TASK_UPDATE_DATA; + let taskIdsToUpdate = Object.keys(tasksToUpdate); + let queue = new TaskQueue(Promise, 300); + let promiseCount = 0; + + logger.info('About to update', taskIdsToUpdate.length, 'user tasks'); + + function updateTaskById (taskId) { + promiseCount++; + + if (promiseCount % 500 === 0) { + logger.info(promiseCount, 'updates started'); + } + + return db.collection('tasks').findOneAndUpdate({_id: taskId, 'challenge.broken': 'CHALLENGE_TASK_NOT_FOUND'}, {$set: {challenge: tasksToUpdate[taskId]}}, {returnOriginal: false}) + } + + return Promise.map(taskIdsToUpdate, queue.wrap(updateTaskById)).then((result) => { + let updates = result.filter(res => res.lastErrorObject.updatedExisting) + let failures = result.filter(res => !res.lastErrorObject.updatedExisting); + + logger.success(updates.length, 'tasks have been fixed'); + + if (failures.length > 0) { + logger.error(failures.length, 'tasks could not be updated'); + logger.error('Manually check these results'); + logger.error(failures); + } + + return Promise.resolve(data); + }); +} + +function closeDb (data) { + logger.success('The process took ' + count + ' seconds'); + + clearInterval(timer) + + db.close(); +} diff --git a/package.json b/package.json index 0f619109bc..53b6c5c2c5 100644 --- a/package.json +++ b/package.json @@ -126,6 +126,7 @@ "babel-eslint": "^6.0.0", "chai": "^3.4.0", "chai-as-promised": "^5.1.0", + "chalk": "^1.1.3", "coveralls": "^2.11.2", "csv": "~0.3.6", "deep-diff": "~0.1.4",