diff --git a/.nodemonignore b/.nodemonignore index 4275bc9184..dc58adb455 100644 --- a/.nodemonignore +++ b/.nodemonignore @@ -1,4 +1,6 @@ node_modules/** +content_cache +content_cache/** website/client/** test/** .git/** diff --git a/gulp/gulp-build.js b/gulp/gulp-build.js index 535b0178b5..8af26328c7 100644 --- a/gulp/gulp-build.js +++ b/gulp/gulp-build.js @@ -1,7 +1,7 @@ import gulp from 'gulp'; import babel from 'gulp-babel'; -gulp.task('build:src', () => gulp.src('website/server/**/*.js') +gulp.task('build:server', () => gulp.src('website/server/**/*.js') .pipe(babel()) .pipe(gulp.dest('website/transpiled-babel/'))); @@ -9,11 +9,11 @@ gulp.task('build:common', () => gulp.src('website/common/script/**/*.js') .pipe(babel()) .pipe(gulp.dest('website/common/transpiled-babel/'))); -gulp.task('build:server', gulp.series('build:src', 'build:common', done => done())); - -gulp.task('build:prod', gulp.series( +gulp.task('build:prod', gulp.parallel( 'build:server', + 'build:common', 'apidoc', + 'content:cache', done => done(), )); diff --git a/gulp/gulp-content.js b/gulp/gulp-content.js new file mode 100644 index 0000000000..a52162d414 --- /dev/null +++ b/gulp/gulp-content.js @@ -0,0 +1,35 @@ +import gulp from 'gulp'; +import fs from 'fs'; +import clean from 'rimraf'; +import { CONTENT_CACHE_PATH, getLocalizedContent } from '../website/server/libs/content'; +import { langCodes } from '../website/server/libs/i18n'; + +gulp.task('content:cache:clean', done => { + clean(CONTENT_CACHE_PATH, done); +}); + +// TODO parallelize, use gulp file helpers +gulp.task('content:cache', gulp.series('content:cache:clean', done => { + try { + // create the cache folder (if it doesn't exist) + try { + fs.mkdirSync(CONTENT_CACHE_PATH); + } catch (err) { + if (err.code !== 'EEXIST') throw err; + } + + // clone the content for each language and save + // localize it + // save the result + langCodes.forEach(langCode => { + fs.writeFileSync( + `${CONTENT_CACHE_PATH}${langCode}.json`, + getLocalizedContent(langCode), + 'utf8', + ); + }); + done(); + } catch (err) { + done(err); + } +})); diff --git a/gulpfile.js b/gulpfile.js index 9283119db9..354011b2fe 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -13,8 +13,16 @@ const gulp = require('gulp'); if (process.env.NODE_ENV === 'production') { // eslint-disable-line no-process-env require('./gulp/gulp-apidoc'); // eslint-disable-line global-require + require('./gulp/gulp-content'); // eslint-disable-line global-require require('./gulp/gulp-build'); // eslint-disable-line global-require } else { - require('glob').sync('./gulp/gulp-*').forEach(require); // eslint-disable-line global-require + require('./gulp/gulp-apidoc'); // eslint-disable-line global-require + require('./gulp/gulp-content'); // eslint-disable-line global-require + require('./gulp/gulp-build'); // eslint-disable-line global-require + require('./gulp/gulp-console'); // eslint-disable-line global-require + require('./gulp/gulp-sprites'); // eslint-disable-line global-require + require('./gulp/gulp-start'); // eslint-disable-line global-require + require('./gulp/gulp-tests'); // eslint-disable-line global-require + require('./gulp/gulp-transifex-test'); // eslint-disable-line global-require require('gulp').task('default', gulp.series('test')); // eslint-disable-line global-require } diff --git a/test/api/unit/libs/content.test.js b/test/api/unit/libs/content.test.js new file mode 100644 index 0000000000..bd037ef4fb --- /dev/null +++ b/test/api/unit/libs/content.test.js @@ -0,0 +1,17 @@ +import * as contentLib from '../../../../website/server/libs/content'; +import content from '../../../../website/common/script/content'; + +describe('contentLib', () => { + describe('CONTENT_CACHE_PATH', () => { + it('exports CONTENT_CACHE_PATH', () => { + expect(contentLib.CONTENT_CACHE_PATH).to.be.a.string; + }); + }); + + describe('getLocalizedContent', () => { + it('clones, not modify, the original content data', () => { + contentLib.getLocalizedContent(); + expect(typeof content.backgrounds.backgrounds062014.beach.text).to.equal('function'); + }); + }); +}); diff --git a/website/server/controllers/api-v3/content.js b/website/server/controllers/api-v3/content.js index 15cbf282c8..6dbb6137f4 100644 --- a/website/server/controllers/api-v3/content.js +++ b/website/server/controllers/api-v3/content.js @@ -1,70 +1,11 @@ -import _ from 'lodash'; -import fsCallback from 'fs'; -import path from 'path'; -import util from 'util'; -import logger from '../../libs/logger'; +import nconf from 'nconf'; import { langCodes } from '../../libs/i18n'; -import common from '../../../common'; +import { CONTENT_CACHE_PATH, getLocalizedContent } from '../../libs/content'; -// Transform fs methods that accept callbacks in ones that return promises -const fs = { - readFile: util.promisify(fsCallback.readFile).bind(fsCallback), - writeFile: util.promisify(fsCallback.writeFile).bind(fsCallback), - stat: util.promisify(fsCallback.stat).bind(fsCallback), - mkdir: util.promisify(fsCallback.mkdir).bind(fsCallback), -}; +const IS_PROD = nconf.get('IS_PROD'); const api = {}; -function walkContent (obj, lang) { - _.each(obj, (item, key, source) => { - if (_.isPlainObject(item) || _.isArray(item)) { - walkContent(item, lang); - } else if (_.isFunction(item) && item.i18nLangFunc) { - source[key] = item(lang); - } - }); -} - -// After the getContent route is called the first time for a certain language -// the response is saved on disk and subsequentially served -// directly from there to reduce computation. -// Example: if `cachedContentResponses.en` is true it means that the response is cached -const cachedContentResponses = {}; - -// Language key set to true while the cache file is being written -const cacheBeingWritten = {}; - -_.each(langCodes, code => { - cachedContentResponses[code] = false; - cacheBeingWritten[code] = false; -}); - - -const CONTENT_CACHE_PATH = path.join(__dirname, '/../../../../content_cache/'); - -async function saveContentToDisk (language, content) { - try { - cacheBeingWritten[language] = true; - - // check if the directory exists, if it doesn't an error is thrown - await fs.stat(CONTENT_CACHE_PATH); - await fs.writeFile(`${CONTENT_CACHE_PATH}${language}.json`, content, 'utf8'); - - cacheBeingWritten[language] = false; - cachedContentResponses[language] = true; - } catch (err) { - // the directory doesn't exists, create it and retry - if (err.code === 'ENOENT' && err.syscall === 'stat') { - await fs.mkdir(CONTENT_CACHE_PATH); - saveContentToDisk(language, content); - } else { - cacheBeingWritten[language] = false; - logger.error(err); - } - } -} - /** * @api {get} /api/v3/content Get all available content objects * @apiDescription Does not require authentication. @@ -118,33 +59,21 @@ api.getContent = { noLanguage: true, async handler (req, res) { let language = 'en'; - const proposedLang = req.query.language && req.query.language.toString(); + const proposedLang = req.query.language; - if (proposedLang in cachedContentResponses) { + if (proposedLang && langCodes.includes(proposedLang)) { language = proposedLang; } - let content; + if (IS_PROD) { + res.sendFile(`${CONTENT_CACHE_PATH}${language}.json`); + } else { + res.set({ + 'Content-Type': 'application/json', + }); - // is the content response for this language cached? - if (cachedContentResponses[language] === true) { - content = await fs.readFile(`${CONTENT_CACHE_PATH}${language}.json`, 'utf8'); - } else { // generate the response - content = _.cloneDeep(common.content); - walkContent(content, language); - content = JSON.stringify(content); - } - - res.set({ - 'Content-Type': 'application/json', - }); - - const jsonResString = `{"success": true, "data": ${content}}`; - res.status(200).send(jsonResString); - - // save the file in background unless it's already cached or being written right now - if (cachedContentResponses[language] !== true && cacheBeingWritten[language] !== true) { - saveContentToDisk(language, content); + const jsonResString = getLocalizedContent(language); + res.status(200).send(jsonResString); } }, }; diff --git a/website/server/libs/content.js b/website/server/libs/content.js new file mode 100644 index 0000000000..134d87f007 --- /dev/null +++ b/website/server/libs/content.js @@ -0,0 +1,21 @@ +import _ from 'lodash'; +import path from 'path'; +import common from '../../common'; + +export const CONTENT_CACHE_PATH = path.join(__dirname, '/../../../content_cache/'); + +function walkContent (obj, lang) { + _.each(obj, (item, key, source) => { + if (_.isPlainObject(item) || _.isArray(item)) { + walkContent(item, lang); + } else if (_.isFunction(item) && item.i18nLangFunc) { + source[key] = item(lang); + } + }); +} + +export function getLocalizedContent (langCode) { + const contentClone = _.cloneDeep(common.content); + walkContent(contentClone, langCode); + return `{"success": true, "data": ${JSON.stringify(contentClone)}}`; +}