krikkert/habitica
1
0
Fork 0
mirror of https://github.com/HabitRPG/habitica.git synced 2025-12-17 06:37:23 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor cron to avoid double cronning - work in progress

Browse Source
This commit is contained in:
Matteo Pagliazzi
2016-05-25 16:55:57 +02:00
parent 1a26965542
commit 31850830a0
3 changed files with 138 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
website/server/controllers/api-v3/groups.js
View File

@@ -119,7 +119,7 @@ api.getGroup = {
url: '/groups/:groupId', url: '/groups/:groupId',
// Disable cron when getting groups to avoid race conditions when the site is loaded // Disable cron when getting groups to avoid race conditions when the site is loaded
// and requests for party and user data are concurrent // and requests for party and user data are concurrent
runCron: false, runCron: true,
middlewares: [authWithHeaders()], middlewares: [authWithHeaders()],
async handler (req, res) { async handler (req, res) {
let user = res.locals.user; let user = res.locals.user;

280
website/server/middlewares/api-v3/cron.js
View File

@@ -10,132 +10,127 @@ import { v4 as uuid } from 'uuid';
const daysSince = common.daysSince; const daysSince = common.daysSince;
module.exports = function cronMiddleware (req, res, next) { module.exports = async function cronMiddleware (req, res, next) {
let user = res.locals.user; let user = res.locals.user;
if (!user) return next(); // User might not be available when authentication is not mandatory if (!user) return next(); // User might not be available when authentication is not mandatory
let analytics = res.analytics; let analytics = res.analytics;
let now = new Date(); let now = new Date();
let _cronSignature = uuid();
// If the user's timezone has changed (due to travel or daylight savings), try {
// cron can be triggered twice in one day, so we check for that and use console.log('CHECKING RUN CRON', req.originalUrl, req.method, (new Date()).toISOString());
// both timezones to work out if cron should run.
// CDS = Custom Day Start time.
let timezoneOffsetFromUserPrefs = user.preferences.timezoneOffset || 0;
let timezoneOffsetAtLastCron = _.isFinite(user.preferences.timezoneOffsetAtLastCron) ? user.preferences.timezoneOffsetAtLastCron : timezoneOffsetFromUserPrefs;
let timezoneOffsetFromBrowser = Number(req.header('x-user-timezoneoffset'));
timezoneOffsetFromBrowser = _.isFinite(timezoneOffsetFromBrowser) ? timezoneOffsetFromBrowser : timezoneOffsetFromUserPrefs;
// NB: All timezone offsets can be 0, so can't use `... || ...` to apply non-zero defaults
if (timezoneOffsetFromBrowser !== timezoneOffsetFromUserPrefs) { // If the user's timezone has changed (due to travel or daylight savings),
// The user's browser has just told Habitica that the user's timezone has // cron can be triggered twice in one day, so we check for that and use
// changed so store and use the new zone. // both timezones to work out if cron should run.
user.preferences.timezoneOffset = timezoneOffsetFromBrowser; // CDS = Custom Day Start time.
timezoneOffsetFromUserPrefs = timezoneOffsetFromBrowser; let timezoneOffsetFromUserPrefs = user.preferences.timezoneOffset || 0;
} let timezoneOffsetAtLastCron = _.isFinite(user.preferences.timezoneOffsetAtLastCron) ? user.preferences.timezoneOffsetAtLastCron : timezoneOffsetFromUserPrefs;
let timezoneOffsetFromBrowser = Number(req.header('x-user-timezoneoffset'));
timezoneOffsetFromBrowser = _.isFinite(timezoneOffsetFromBrowser) ? timezoneOffsetFromBrowser : timezoneOffsetFromUserPrefs;
// NB: All timezone offsets can be 0, so can't use `... || ...` to apply non-zero defaults
// How many days have we missed using the user's current timezone: if (timezoneOffsetFromBrowser !== timezoneOffsetFromUserPrefs) {
let daysMissed = daysSince(user.lastCron, _.defaults({now}, user.preferences)); // The user's browser has just told Habitica that the user's timezone has
// changed so store and use the new zone.
if (timezoneOffsetAtLastCron !== timezoneOffsetFromUserPrefs) { user.preferences.timezoneOffset = timezoneOffsetFromBrowser;
// Since cron last ran, the user's timezone has changed. timezoneOffsetFromUserPrefs = timezoneOffsetFromBrowser;
// How many days have we missed using the old timezone:
let daysMissedNewZone = daysMissed;
let daysMissedOldZone = daysSince(user.lastCron, _.defaults({
now,
timezoneOffsetOverride: timezoneOffsetAtLastCron,
}, user.preferences));
if (timezoneOffsetAtLastCron < timezoneOffsetFromUserPrefs) {
// The timezone change was in the unsafe direction.
// E.g., timezone changes from UTC+1 (offset -60) to UTC+0 (offset 0).
// or timezone changes from UTC-4 (offset 240) to UTC-5 (offset 300).
// Local time changed from, for example, 03:00 to 02:00.
if (daysMissedOldZone > 0 && daysMissedNewZone > 0) {
// Both old and new timezones indicate that we SHOULD run cron, so
// it is safe to do so immediately.
daysMissed = Math.min(daysMissedOldZone, daysMissedNewZone);
// use minimum value to be nice to user
} else if (daysMissedOldZone > 0) {
// The old timezone says that cron should run; the new timezone does not.
// This should be impossible for this direction of timezone change, but
// just in case I'm wrong...
// TODO
// console.log("zone has changed - old zone says run cron, NEW zone says no - stop cron now only -- SHOULD NOT HAVE GOT TO HERE", timezoneOffsetAtLastCron, timezoneOffsetFromUserPrefs, now); // used in production for confirming this never happens
} else if (daysMissedNewZone > 0) {
// The old timezone says that cron should NOT run -- i.e., cron has
// already run today, from the old timezone's point of view.
// The new timezone says that cron SHOULD run, but this is almost
// certainly incorrect.
// This happens when cron occurred at a time soon after the CDS. When
// you reinterpret that time in the new timezone, it looks like it
// was before the CDS, because local time has stepped backwards.
// To fix this, rewrite the cron time to a time that the new
// timezone interprets as being in today.
daysMissed = 0; // prevent cron running now
let timezoneOffsetDiff = timezoneOffsetAtLastCron - timezoneOffsetFromUserPrefs;
// e.g., for dangerous zone change: 240 - 300 = -60 or -660 - -600 = -60
user.lastCron = moment(user.lastCron).subtract(timezoneOffsetDiff, 'minutes');
// NB: We don't change user.auth.timestamps.loggedin so that will still record the time that the previous cron actually ran.
// From now on we can ignore the old timezone:
user.preferences.timezoneOffsetAtLastCron = timezoneOffsetFromUserPrefs;
} else {
// Both old and new timezones indicate that cron should
// NOT run.
daysMissed = 0; // prevent cron running now
}
} else if (timezoneOffsetAtLastCron > timezoneOffsetFromUserPrefs) {
daysMissed = daysMissedNewZone;
// TODO: Either confirm that there is nothing that could possibly go wrong here and remove the need for this else branch, or fix stuff.
// There are probably situations where the Dailies do not reset early enough for a user who was expecting the zone change and wants to use all their Dailies immediately in the new zone;
// if so, we should provide an option for easy reset of Dailies (can't be automatic because there will be other situations where the user was not prepared).
} }
}
if (daysMissed <= 0) return next(); // How many days have we missed using the user's current timezone:
let daysMissed = daysSince(user.lastCron, _.defaults({now}, user.preferences));
let quest; if (timezoneOffsetAtLastCron !== timezoneOffsetFromUserPrefs) {
let progress; // Since cron last ran, the user's timezone has changed.
let tasks; // How many days have we missed using the old timezone:
let daysMissedNewZone = daysMissed;
let daysMissedOldZone = daysSince(user.lastCron, _.defaults({
now,
timezoneOffsetOverride: timezoneOffsetAtLastCron,
}, user.preferences));
// To avoid double cron we set _cronSignature on the user to a random string if (timezoneOffsetAtLastCron < timezoneOffsetFromUserPrefs) {
// and check that it has remained the same before saving // The timezone change was in the unsafe direction.
user._cronSignature = uuid(); // E.g., timezone changes from UTC+1 (offset -60) to UTC+0 (offset 0).
// or timezone changes from UTC-4 (offset 240) to UTC-5 (offset 300).
// Local time changed from, for example, 03:00 to 02:00.
if (daysMissedOldZone > 0 && daysMissedNewZone > 0) {
// Both old and new timezones indicate that we SHOULD run cron, so
// it is safe to do so immediately.
daysMissed = Math.min(daysMissedOldZone, daysMissedNewZone);
// use minimum value to be nice to user
} else if (daysMissedOldZone > 0) {
// The old timezone says that cron should run; the new timezone does not.
// This should be impossible for this direction of timezone change, but
// just in case I'm wrong...
// TODO
// console.log("zone has changed - old zone says run cron, NEW zone says no - stop cron now only -- SHOULD NOT HAVE GOT TO HERE", timezoneOffsetAtLastCron, timezoneOffsetFromUserPrefs, now); // used in production for confirming this never happens
} else if (daysMissedNewZone > 0) {
// The old timezone says that cron should NOT run -- i.e., cron has
// already run today, from the old timezone's point of view.
// The new timezone says that cron SHOULD run, but this is almost
// certainly incorrect.
// This happens when cron occurred at a time soon after the CDS. When
// you reinterpret that time in the new timezone, it looks like it
// was before the CDS, because local time has stepped backwards.
// To fix this, rewrite the cron time to a time that the new
// timezone interprets as being in today.
daysMissed = 0; // prevent cron running now
let timezoneOffsetDiff = timezoneOffsetAtLastCron - timezoneOffsetFromUserPrefs;
// e.g., for dangerous zone change: 240 - 300 = -60 or -660 - -600 = -60
user.lastCron = moment(user.lastCron).subtract(timezoneOffsetDiff, 'minutes');
// NB: We don't change user.auth.timestamps.loggedin so that will still record the time that the previous cron actually ran.
// From now on we can ignore the old timezone:
user.preferences.timezoneOffsetAtLastCron = timezoneOffsetFromUserPrefs;
} else {
// Both old and new timezones indicate that cron should
// NOT run.
daysMissed = 0; // prevent cron running now
}
} else if (timezoneOffsetAtLastCron > timezoneOffsetFromUserPrefs) {
daysMissed = daysMissedNewZone;
// TODO: Either confirm that there is nothing that could possibly go wrong here and remove the need for this else branch, or fix stuff.
// There are probably situations where the Dailies do not reset early enough for a user who was expecting the zone change and wants to use all their Dailies immediately in the new zone;
// if so, we should provide an option for easy reset of Dailies (can't be automatic because there will be other situations where the user was not prepared).
}
}
if (daysMissed <= 0) return next();
console.log('RUNNING CRON FOR REAL', req.originalUrl, req.method, (new Date()).toISOString());
// To avoid double cron we first set _cronSignature to now and then check that it's not changed while processing
let userUpdateResult = await User.update({
_id: user._id,
_cronSignature: 'not-running', // Check that in the meantime another cron has not started
}, {
$set: {
_cronSignature,
},
}).exec();
console.log('FIRST USER UPDATE?', userUpdateResult, req.originalUrl, req.method, (new Date()).toISOString());
User.update({
_id: user._id,
_cronSignature: 'not-running',
}, {
$set: {
_cronSignature: user._cronSignature,
},
}).exec()
.then((updateResult) => { // Fetch active tasks (no completed todos)
// if the cron signature is set, throw an error and recover later // if the cron signature is set, throw an error and recover later
if (updateResult.nMatched === 0 || updateResult.nUpdated === 0) { if (userUpdateResult.nMatched === 0 || userUpdateResult.nModified === 0) {
throw new Error('cron-already-running'); throw new Error('cron-already-running');
} }
return Tasks.Task.find({ let tasks = await Tasks.Task.find({
userId: user._id, userId: user._id,
$or: [ // Exclude completed todos $or: [ // Exclude completed todos
{type: 'todo', completed: false}, {type: 'todo', completed: false},
{type: {$in: ['habit', 'daily', 'reward']}}, {type: {$in: ['habit', 'daily', 'reward']}},
], ],
}).exec(); }).exec();
})
.then(tasksFetched => {
tasks = tasksFetched;
let tasksByType = {habits: [], dailys: [], todos: [], rewards: []}; let tasksByType = {habits: [], dailys: [], todos: [], rewards: []};
tasks.forEach(task => tasksByType[`${task.type}s`].push(task)); tasks.forEach(task => tasksByType[`${task.type}s`].push(task));
// Run cron // Run cron
progress = cron({user, tasksByType, now, daysMissed, analytics, timezoneOffsetFromUserPrefs}); let progress = cron({user, tasksByType, now, daysMissed, analytics, timezoneOffsetFromUserPrefs});
// Clear old completed todos - 30 days for free users, 90 for subscribers // Clear old completed todos - 30 days for free users, 90 for subscribers
// Do not delete challenges completed todos TODO unless the task is broken? // Do not delete challenges completed todos TODO unless the task is broken?
@@ -149,74 +144,65 @@ module.exports = function cronMiddleware (req, res, next) {
'challenge.id': {$exists: false}, 'challenge.id': {$exists: false},
}).exec(); }).exec();
let ranCron = user.isModified(); let quest = common.content.quests[user.party.quest.key];
quest = common.content.quests[user.party.quest.key];
if (ranCron) res.locals.wasModified = true; // TODO remove after v2 is retired res.locals.wasModified = true; // TODO remove after v2 is retired
if (!ranCron) return next();
// Group.tavernBoss(user, progress); // Group.tavernBoss(user, progress);
let reallyModifiedPaths = {}; // Mongoose stores both path and path.nested in user.modifiedPaths()
// Save user and tasks user.modifiedPaths().forEach(path => {
// Uses mongoose's internals to get update command let rootPath = path.split('.')[0];
let mongooseDelta = user.$__delta(); if (Object.keys(reallyModifiedPaths).indexOf(rootPath) === -1) {
if (mongooseDelta instanceof Error) { let dataForPath = user[rootPath].toObject ? user[rootPath].toObject() : user[rootPath];
throw mongooseDelta; reallyModifiedPaths[rootPath] = dataForPath;
} }
});
reallyModifiedPaths._cronSignature = 'not-running'; // Finish running cron
let mongooseWhere = user.$__where(mongooseDelta[0]); // Save user (only if another cron has done it already!)
if (mongooseWhere instanceof Error) { let secondUserUpdateResult = await User.update({
throw mongooseWhere; _id: user._id,
} _cronSignature,
mongooseWhere._cronSignature = user._cronSignature; // Only update the user if cron signature matches }, {
$set: reallyModifiedPaths,
}).exec();
console.log('SECOND USER UPDATE?', secondUserUpdateResult, req.originalUrl, req.method, (new Date()).toISOString());
return User.update(mongooseWhere, mongooseDelta[1]); // if cron already run, throw and recover later
}) if (secondUserUpdateResult.nMatched === 0 || secondUserUpdateResult.nModified === 0) {
.then(updateResult => {
// if the cron signature is set, throw an error and recover later
if (updateResult.nMatched === 0 || updateResult.nUpdated === 0) {
throw new Error('cron-already-running'); throw new Error('cron-already-running');
} }
let toSave = []; let tasksToSave = [];
tasks.forEach(task => { tasks.forEach(task => {
if (task.isModified()) toSave.push(task.save()); if (task.isModified()) tasksToSave.push(task.save());
}); });
await Bluebird.all(tasksToSave);
return Bluebird.all(toSave);
})
.then(() => {
if (!quest) return; if (!quest) return;
// If user is on a quest, roll for boss & player, or handle collections // If user is on a quest, roll for boss & player, or handle collections
let questType = quest.boss ? 'boss' : 'collect'; let questType = quest.boss ? 'boss' : 'collect';
// TODO this saves user, runs db updates, loads user. Is there a better way to handle this? // TODO this saves user, runs db updates, loads user. Is there a better way to handle this?
return Group[`${questType}Quest`](user, progress); await Group[`${questType}Quest`](user, progress);
})
.then(() => {
User.findByIdAndUpdate(user._id, {
$set: {_cronSignature: 'not-running'},
}, {
new: true, // return the updated document
}).exec();
}) // fetch the updated user...
.then(updatedUser => {
user = res.locals.user = updatedUser;
return null; res.locals.user = await User.findById(user._id).exec();
})
.then(() => next()) return next();
.catch((err) => { } catch (err) {
if (err.message === 'cron-already-running') { if (err.message === 'cron-already-running') {
console.log('RECOVERING FROM CRON', req.originalUrl, req.method, (new Date()).toISOString());
// recovering after abort, wait 200ms and reload user // recovering after abort, wait 200ms and reload user
setTimeout(() => { Bluebird.delay(200).then(() => {
User.findById(user._id, (reloadErr, reloadedUser) => { return User.findById(user._id).exec();
if (reloadErr) return next(reloadErr); }).then((reloadedUser) => {
user = res.locals.user = reloadedUser; res.locals.user = reloadedUser;
return next(); console.log('RECOVERED FROM CRON', req.originalUrl, req.method, (new Date()).toISOString());
}); return next();
}, 200); }).catch(secondError => next(secondError));
} else { } else {
return next(err); return next(err);
} }
}); }
}; };

5
website/server/models/group.js
View File

@@ -25,6 +25,9 @@ const Schema = mongoose.Schema;
export const INVITES_LIMIT = 100; export const INVITES_LIMIT = 100;
export const TAVERN_ID = shared.TAVERN_ID; export const TAVERN_ID = shared.TAVERN_ID;
const CRON_SAFE_MODE = nconf.get('CRON_SAFE_MODE') === 'true';
const CRON_SEMI_SAFE_MODE = nconf.get('CRON_SEMI_SAFE_MODE') === 'true';
// NOTE once Firebase is enabled any change to groups' members in MongoDB will have to be run through the API // NOTE once Firebase is enabled any change to groups' members in MongoDB will have to be run through the API
// changes made directly to the db will cause Firebase to get out of sync // changes made directly to the db will cause Firebase to get out of sync
export let schema = new Schema({ export let schema = new Schema({
@@ -516,7 +519,7 @@ schema.statics.bossQuest = async function bossQuest (user, progress) {
group.quest.progress.hp -= progress.up; group.quest.progress.hp -= progress.up;
// TODO Create a party preferred language option so emits like this can be localized. Suggestion: Always display the English version too. Or, if English is not displayed to the players, at least include it in a new field in the chat object that's visible in the database - essential for admins when troubleshooting quests! // TODO Create a party preferred language option so emits like this can be localized. Suggestion: Always display the English version too. Or, if English is not displayed to the players, at least include it in a new field in the chat object that's visible in the database - essential for admins when troubleshooting quests!
let playerAttack = `${user.profile.name} attacks ${quest.boss.name('en')} for ${progress.up.toFixed(1)} damage.`; let playerAttack = `${user.profile.name} attacks ${quest.boss.name('en')} for ${progress.up.toFixed(1)} damage.`;
let bossAttack = nconf.get('CRON_SAFE_MODE') === 'true' || nconf.get('CRON_SEMI_SAFE_MODE') === 'true' ? `${quest.boss.name('en')} does not attack, because it respects the fact that there are some bugs\` \`post-maintenance and it doesn't want to hurt anyone unfairly. It will continue its rampage soon!` : `${quest.boss.name('en')} attacks party for ${Math.abs(down).toFixed(1)} damage.`; let bossAttack = CRON_SAFE_MODE || CRON_SEMI_SAFE_MODE ? `${quest.boss.name('en')} does not attack, because it respects the fact that there are some bugs\` \`post-maintenance and it doesn't want to hurt anyone unfairly. It will continue its rampage soon!` : `${quest.boss.name('en')} attacks party for ${Math.abs(down).toFixed(1)} damage.`;
// TODO Consider putting the safe mode boss attack message in an ENV var // TODO Consider putting the safe mode boss attack message in an ENV var
group.sendChat(`\`${playerAttack}\` \`${bossAttack}\``); group.sendChat(`\`${playerAttack}\` \`${bossAttack}\``);