krikkert/habitica
1
0
Fork 0
mirror of https://github.com/HabitRPG/habitica.git synced 2025-12-18 23:27:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Content API Cache improvements (#12020)

Browse Source 
* content api improvements

* add content cache to build step

* add tests
This commit is contained in:
Matteo Pagliazzi
2020-03-29 16:15:23 +02:00
committed by GitHub
parent 9ab9b0f553
commit 3fffe7aa5c
7 changed files with 101 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.nodemonignore
View File

@@ -1,4 +1,6 @@
node_modules/**
content_cache
content_cache/**
website/client/**
test/**
.git/**

8
gulp/gulp-build.js
View File

@@ -1,7 +1,7 @@
import gulp from 'gulp';
import babel from 'gulp-babel';
gulp.task('build:src', () => gulp.src('website/server/**/*.js')
gulp.task('build:server', () => gulp.src('website/server/**/*.js')
.pipe(babel())
.pipe(gulp.dest('website/transpiled-babel/')));
@@ -9,11 +9,11 @@ gulp.task('build:common', () => gulp.src('website/common/script/**/*.js')
.pipe(babel())
.pipe(gulp.dest('website/common/transpiled-babel/')));
gulp.task('build:server', gulp.series('build:src', 'build:common', done => done()));
gulp.task('build:prod', gulp.series(
gulp.task('build:prod', gulp.parallel(
'build:server',
'build:common',
'apidoc',
'content:cache',
done => done(),
));

35
gulp/gulp-content.js Normal file
View File

@@ -0,0 +1,35 @@
import gulp from 'gulp';
import fs from 'fs';
import clean from 'rimraf';
import { CONTENT_CACHE_PATH, getLocalizedContent } from '../website/server/libs/content';
import { langCodes } from '../website/server/libs/i18n';
gulp.task('content:cache:clean', done => {
clean(CONTENT_CACHE_PATH, done);
});
// TODO parallelize, use gulp file helpers
gulp.task('content:cache', gulp.series('content:cache:clean', done => {
try {
// create the cache folder (if it doesn't exist)
try {
fs.mkdirSync(CONTENT_CACHE_PATH);
} catch (err) {
if (err.code !== 'EEXIST') throw err;
}
// clone the content for each language and save
// localize it
// save the result
langCodes.forEach(langCode => {
fs.writeFileSync(
`${CONTENT_CACHE_PATH}${langCode}.json`,
getLocalizedContent(langCode),
'utf8',
);
});
done();
} catch (err) {
done(err);
}
}));

10
gulpfile.js
View File

@@ -13,8 +13,16 @@ const gulp = require('gulp');
if (process.env.NODE_ENV === 'production') { // eslint-disable-line no-process-env
require('./gulp/gulp-apidoc'); // eslint-disable-line global-require
require('./gulp/gulp-content'); // eslint-disable-line global-require
require('./gulp/gulp-build'); // eslint-disable-line global-require
} else {
require('glob').sync('./gulp/gulp-*').forEach(require); // eslint-disable-line global-require
require('./gulp/gulp-apidoc'); // eslint-disable-line global-require
require('./gulp/gulp-content'); // eslint-disable-line global-require
require('./gulp/gulp-build'); // eslint-disable-line global-require
require('./gulp/gulp-console'); // eslint-disable-line global-require
require('./gulp/gulp-sprites'); // eslint-disable-line global-require
require('./gulp/gulp-start'); // eslint-disable-line global-require
require('./gulp/gulp-tests'); // eslint-disable-line global-require
require('./gulp/gulp-transifex-test'); // eslint-disable-line global-require
require('gulp').task('default', gulp.series('test')); // eslint-disable-line global-require
}

17
test/api/unit/libs/content.test.js Normal file
View File

@@ -0,0 +1,17 @@
import * as contentLib from '../../../../website/server/libs/content';
import content from '../../../../website/common/script/content';
describe('contentLib', () => {
describe('CONTENT_CACHE_PATH', () => {
it('exports CONTENT_CACHE_PATH', () => {
expect(contentLib.CONTENT_CACHE_PATH).to.be.a.string;
});
});
describe('getLocalizedContent', () => {
it('clones, not modify, the original content data', () => {
contentLib.getLocalizedContent();
expect(typeof content.backgrounds.backgrounds062014.beach.text).to.equal('function');
});
});
});

89
website/server/controllers/api-v3/content.js
View File

@@ -1,70 +1,11 @@
import _ from 'lodash';
import fsCallback from 'fs';
import path from 'path';
import util from 'util';
import logger from '../../libs/logger';
import nconf from 'nconf';
import { langCodes } from '../../libs/i18n';
import common from '../../../common';
import { CONTENT_CACHE_PATH, getLocalizedContent } from '../../libs/content';
// Transform fs methods that accept callbacks in ones that return promises
const fs = {
readFile: util.promisify(fsCallback.readFile).bind(fsCallback),
writeFile: util.promisify(fsCallback.writeFile).bind(fsCallback),
stat: util.promisify(fsCallback.stat).bind(fsCallback),
mkdir: util.promisify(fsCallback.mkdir).bind(fsCallback),
};
const IS_PROD = nconf.get('IS_PROD');
const api = {};
function walkContent (obj, lang) {
_.each(obj, (item, key, source) => {
if (_.isPlainObject(item) || _.isArray(item)) {
walkContent(item, lang);
} else if (_.isFunction(item) && item.i18nLangFunc) {
source[key] = item(lang);
}
});
}
// After the getContent route is called the first time for a certain language
// the response is saved on disk and subsequentially served
// directly from there to reduce computation.
// Example: if `cachedContentResponses.en` is true it means that the response is cached
const cachedContentResponses = {};
// Language key set to true while the cache file is being written
const cacheBeingWritten = {};
_.each(langCodes, code => {
cachedContentResponses[code] = false;
cacheBeingWritten[code] = false;
});
const CONTENT_CACHE_PATH = path.join(__dirname, '/../../../../content_cache/');
async function saveContentToDisk (language, content) {
try {
cacheBeingWritten[language] = true;
// check if the directory exists, if it doesn't an error is thrown
await fs.stat(CONTENT_CACHE_PATH);
await fs.writeFile(`${CONTENT_CACHE_PATH}${language}.json`, content, 'utf8');
cacheBeingWritten[language] = false;
cachedContentResponses[language] = true;
} catch (err) {
// the directory doesn't exists, create it and retry
if (err.code === 'ENOENT' && err.syscall === 'stat') {
await fs.mkdir(CONTENT_CACHE_PATH);
saveContentToDisk(language, content);
} else {
cacheBeingWritten[language] = false;
logger.error(err);
}
}
}
/**
* @api {get} /api/v3/content Get all available content objects
* @apiDescription Does not require authentication.
@@ -118,33 +59,21 @@ api.getContent = {
noLanguage: true,
async handler (req, res) {
let language = 'en';
const proposedLang = req.query.language && req.query.language.toString();
const proposedLang = req.query.language;
if (proposedLang in cachedContentResponses) {
if (proposedLang && langCodes.includes(proposedLang)) {
language = proposedLang;
}
let content;
// is the content response for this language cached?
if (cachedContentResponses[language] === true) {
content = await fs.readFile(`${CONTENT_CACHE_PATH}${language}.json`, 'utf8');
} else { // generate the response
content = _.cloneDeep(common.content);
walkContent(content, language);
content = JSON.stringify(content);
}
if (IS_PROD) {
res.sendFile(`${CONTENT_CACHE_PATH}${language}.json`);
} else {
res.set({
'Content-Type': 'application/json',
});
const jsonResString = `{"success": true, "data": ${content}}`;
const jsonResString = getLocalizedContent(language);
res.status(200).send(jsonResString);
// save the file in background unless it's already cached or being written right now
if (cachedContentResponses[language] !== true && cacheBeingWritten[language] !== true) {
saveContentToDisk(language, content);
}
},
};

21
website/server/libs/content.js Normal file
View File

@@ -0,0 +1,21 @@
import _ from 'lodash';
import path from 'path';
import common from '../../common';
export const CONTENT_CACHE_PATH = path.join(__dirname, '/../../../content_cache/');
function walkContent (obj, lang) {
_.each(obj, (item, key, source) => {
if (_.isPlainObject(item) || _.isArray(item)) {
walkContent(item, lang);
} else if (_.isFunction(item) && item.i18nLangFunc) {
source[key] = item(lang);
}
});
}
export function getLocalizedContent (langCode) {
const contentClone = _.cloneDeep(common.content);
walkContent(contentClone, langCode);
return `{"success": true, "data": ${JSON.stringify(contentClone)}}`;
}