Content API Cache improvements (#12020)

* content api improvements

* add content cache to build step

* add tests

.nodemonignore

@@ -1,4 +1,6 @@
 node_modules/**
+content_cache
+content_cache/**
 website/client/**
 test/**
 .git/**

gulp/gulp-build.js

@@ -1,7 +1,7 @@
 import gulp from 'gulp';
 import babel from 'gulp-babel';
 
-gulp.task('build:src', () => gulp.src('website/server/**/*.js')
+gulp.task('build:server', () => gulp.src('website/server/**/*.js')
   .pipe(babel())
   .pipe(gulp.dest('website/transpiled-babel/')));
 
@@ -9,11 +9,11 @@ gulp.task('build:common', () => gulp.src('website/common/script/**/*.js')
   .pipe(babel())
   .pipe(gulp.dest('website/common/transpiled-babel/')));
 
-gulp.task('build:server', gulp.series('build:src', 'build:common', done => done()));
+gulp.task('build:prod', gulp.parallel(
-
-gulp.task('build:prod', gulp.series(
   'build:server',
+  'build:common',
   'apidoc',
+  'content:cache',
   done => done(),
 ));
 

gulp/gulp-content.js

@@ -0,0 +1,35 @@
+import gulp from 'gulp';
+import fs from 'fs';
+import clean from 'rimraf';
+import { CONTENT_CACHE_PATH, getLocalizedContent } from '../website/server/libs/content';
+import { langCodes } from '../website/server/libs/i18n';
+
+gulp.task('content:cache:clean', done => {
+  clean(CONTENT_CACHE_PATH, done);
+});
+
+// TODO parallelize, use gulp file helpers
+gulp.task('content:cache', gulp.series('content:cache:clean', done => {
+  try {
+    // create the cache folder (if it doesn't exist)
+    try {
+      fs.mkdirSync(CONTENT_CACHE_PATH);
+    } catch (err) {
+      if (err.code !== 'EEXIST') throw err;
+    }
+
+    // clone the content for each language and save
+    // localize it
+    // save the result
+    langCodes.forEach(langCode => {
+      fs.writeFileSync(
+        `${CONTENT_CACHE_PATH}${langCode}.json`,
+        getLocalizedContent(langCode),
+        'utf8',
+      );
+    });
+    done();
+  } catch (err) {
+    done(err);
+  }
+}));

gulpfile.js

@@ -13,8 +13,16 @@ const gulp = require('gulp');
 
 if (process.env.NODE_ENV === 'production') { // eslint-disable-line no-process-env
   require('./gulp/gulp-apidoc'); // eslint-disable-line global-require
+  require('./gulp/gulp-content'); // eslint-disable-line global-require
   require('./gulp/gulp-build'); // eslint-disable-line global-require
 } else {
-  require('glob').sync('./gulp/gulp-*').forEach(require); // eslint-disable-line global-require
+  require('./gulp/gulp-apidoc'); // eslint-disable-line global-require
+  require('./gulp/gulp-content'); // eslint-disable-line global-require
+  require('./gulp/gulp-build'); // eslint-disable-line global-require
+  require('./gulp/gulp-console'); // eslint-disable-line global-require
+  require('./gulp/gulp-sprites'); // eslint-disable-line global-require
+  require('./gulp/gulp-start'); // eslint-disable-line global-require
+  require('./gulp/gulp-tests'); // eslint-disable-line global-require
+  require('./gulp/gulp-transifex-test'); // eslint-disable-line global-require
   require('gulp').task('default', gulp.series('test')); // eslint-disable-line global-require
 }

test/api/unit/libs/content.test.js

@@ -0,0 +1,17 @@
+import * as contentLib from '../../../../website/server/libs/content';
+import content from '../../../../website/common/script/content';
+
+describe('contentLib', () => {
+  describe('CONTENT_CACHE_PATH', () => {
+    it('exports CONTENT_CACHE_PATH', () => {
+      expect(contentLib.CONTENT_CACHE_PATH).to.be.a.string;
+    });
+  });
+
+  describe('getLocalizedContent', () => {
+    it('clones, not modify, the original content data', () => {
+      contentLib.getLocalizedContent();
+      expect(typeof content.backgrounds.backgrounds062014.beach.text).to.equal('function');
+    });
+  });
+});

website/server/controllers/api-v3/content.js

@@ -1,70 +1,11 @@
-import _ from 'lodash';
+import nconf from 'nconf';
-import fsCallback from 'fs';
-import path from 'path';
-import util from 'util';
-import logger from '../../libs/logger';
 import { langCodes } from '../../libs/i18n';
-import common from '../../../common';
+import { CONTENT_CACHE_PATH, getLocalizedContent } from '../../libs/content';
 
-// Transform fs methods that accept callbacks in ones that return promises
+const IS_PROD = nconf.get('IS_PROD');
-const fs = {
-  readFile: util.promisify(fsCallback.readFile).bind(fsCallback),
-  writeFile: util.promisify(fsCallback.writeFile).bind(fsCallback),
-  stat: util.promisify(fsCallback.stat).bind(fsCallback),
-  mkdir: util.promisify(fsCallback.mkdir).bind(fsCallback),
-};
 
 const api = {};
 
-function walkContent (obj, lang) {
-  _.each(obj, (item, key, source) => {
-    if (_.isPlainObject(item) || _.isArray(item)) {
-      walkContent(item, lang);
-    } else if (_.isFunction(item) && item.i18nLangFunc) {
-      source[key] = item(lang);
-    }
-  });
-}
-
-// After the getContent route is called the first time for a certain language
-// the response is saved on disk and subsequentially served
-// directly from there to reduce computation.
-// Example: if `cachedContentResponses.en` is true it means that the response is cached
-const cachedContentResponses = {};
-
-// Language key set to true while the cache file is being written
-const cacheBeingWritten = {};
-
-_.each(langCodes, code => {
-  cachedContentResponses[code] = false;
-  cacheBeingWritten[code] = false;
-});
-
-
-const CONTENT_CACHE_PATH = path.join(__dirname, '/../../../../content_cache/');
-
-async function saveContentToDisk (language, content) {
-  try {
-    cacheBeingWritten[language] = true;
-
-    // check if the directory exists, if it doesn't an error is thrown
-    await fs.stat(CONTENT_CACHE_PATH);
-    await fs.writeFile(`${CONTENT_CACHE_PATH}${language}.json`, content, 'utf8');
-
-    cacheBeingWritten[language] = false;
-    cachedContentResponses[language] = true;
-  } catch (err) {
-    // the directory doesn't exists, create it and retry
-    if (err.code === 'ENOENT' && err.syscall === 'stat') {
-      await fs.mkdir(CONTENT_CACHE_PATH);
-      saveContentToDisk(language, content);
-    } else {
-      cacheBeingWritten[language] = false;
-      logger.error(err);
-    }
-  }
-}
-
 /**
  * @api {get} /api/v3/content Get all available content objects
  * @apiDescription Does not require authentication.
@@ -118,33 +59,21 @@ api.getContent = {
   noLanguage: true,
   async handler (req, res) {
     let language = 'en';
-    const proposedLang = req.query.language && req.query.language.toString();
+    const proposedLang = req.query.language;
 
-    if (proposedLang in cachedContentResponses) {
+    if (proposedLang && langCodes.includes(proposedLang)) {
       language = proposedLang;
     }
 
-    let content;
+    if (IS_PROD) {
+      res.sendFile(`${CONTENT_CACHE_PATH}${language}.json`);
+    } else {
+      res.set({
+        'Content-Type': 'application/json',
+      });
 
-    // is the content response for this language cached?
+      const jsonResString = getLocalizedContent(language);
-    if (cachedContentResponses[language] === true) {
+      res.status(200).send(jsonResString);
-      content = await fs.readFile(`${CONTENT_CACHE_PATH}${language}.json`, 'utf8');
-    } else { // generate the response
-      content = _.cloneDeep(common.content);
-      walkContent(content, language);
-      content = JSON.stringify(content);
-    }
-
-    res.set({
-      'Content-Type': 'application/json',
-    });
-
-    const jsonResString = `{"success": true, "data": ${content}}`;
-    res.status(200).send(jsonResString);
-
-    // save the file in background unless it's already cached or being written right now
-    if (cachedContentResponses[language] !== true && cacheBeingWritten[language] !== true) {
-      saveContentToDisk(language, content);
     }
   },
 };

website/server/libs/content.js

@@ -0,0 +1,21 @@
+import _ from 'lodash';
+import path from 'path';
+import common from '../../common';
+
+export const CONTENT_CACHE_PATH = path.join(__dirname, '/../../../content_cache/');
+
+function walkContent (obj, lang) {
+  _.each(obj, (item, key, source) => {
+    if (_.isPlainObject(item) || _.isArray(item)) {
+      walkContent(item, lang);
+    } else if (_.isFunction(item) && item.i18nLangFunc) {
+      source[key] = item(lang);
+    }
+  });
+}
+
+export function getLocalizedContent (langCode) {
+  const contentClone = _.cloneDeep(common.content);
+  walkContent(contentClone, langCode);
+  return `{"success": true, "data": ${JSON.stringify(contentClone)}}`;
+}