Scripts October 2018 (#10741)

* chore(scripts): BTS Challenge archive and username email jobbing

* refactor(migration): use batching and sendTxn

* fix(script): introduce delay for batching

* fix(migration): correct import, fix delay promise, slower batching

* fix(migration): add daterange

* WIP(script): deletion helper for GDPR

* fix(script): address code comments

* refactor(script): use for loop

* fix(script-runner): bad catch syntax

* fix(script-runner): oops I did it again

* fix(lint): name functions

config.json.example

@@ -39,6 +39,7 @@
     "NEW_RELIC_API_KEY":"NEW_RELIC_API_KEY",
     "GA_ID": "GA_ID",
     "AMPLITUDE_KEY": "AMPLITUDE_KEY",
+    "AMPLITUDE_SECRET": "AMPLITUDE_SECRET",
     "AMAZON_PAYMENTS": {
         "SELLER_ID": "SELLER_ID",
         "CLIENT_ID": "CLIENT_ID",

database_reports/20181001_backtoschool_challenge.js

@@ -0,0 +1,48 @@
+import monk from 'monk';
+import nconf from 'nconf';
+
+/*
+ * Output data on users who completed all the To-Do tasks in the 2018 Back-to-School Challenge.
+ * User ID,Profile Name
+ */
+const CONNECTION_STRING = nconf.get('MIGRATION_CONNECT_STRING');
+const CHALLENGE_ID = '0acb1d56-1660-41a4-af80-9259f080b62b';
+
+let dbUsers = monk(CONNECTION_STRING).get('users', { castIds: false });
+let dbTasks = monk(CONNECTION_STRING).get('tasks', { castIds: false });
+
+function usersReport() {
+  console.info('User ID,Profile Name');
+  let userCount = 0;
+
+  dbUsers.find(
+    {challenges: CHALLENGE_ID},
+    {fields:
+      {_id: 1, 'profile.name': 1}
+    },
+  ).each((user, {close, pause, resume}) => {
+    pause();
+    userCount++;
+    let completedTodos = 0;
+    return dbTasks.find(
+      {
+        userId: user._id,
+        'challenge.id': CHALLENGE_ID,
+        type: 'todo',
+      },
+      {fields: {completed: 1}}
+    ).each((task) => {
+      if (task.completed) completedTodos++;
+    }).then(() => {
+      if (completedTodos >= 7) {
+        console.info(`${user._id},${user.profile.name}`);
+      }
+      resume();
+    });
+  }).then(() => {
+    console.info(`${userCount} users reviewed`);
+    return process.exit(0);
+  });
+}
+
+module.exports = usersReport;

migrations/migration-runner.js

@@ -18,4 +18,11 @@ setUpServer();
 
 // Replace this with your migration
 const processUsers = require('./users/takeThis.js');
-processUsers();
+processUsers()
+  .then(function success () {
+    process.exitCode = 0;
+  })
+  .catch(function failure (err) {
+    console.log(err);
+    process.exitCode = 1;
+  });

migrations/users/20181002_username_email.js

@@ -0,0 +1,107 @@
+const MIGRATION_NAME = '20181003_username_email.js';
+let authorName = 'Sabe'; // in case script author needs to know when their ...
+let authorUuid = '7f14ed62-5408-4e1b-be83-ada62d504931'; // ... own data is done
+
+/*
+ * Send emails to eligible users announcing upcoming username changes
+ */
+
+import monk from 'monk';
+import nconf from 'nconf';
+import { sendTxn } from '../../website/server/libs/email';
+const CONNECTION_STRING = nconf.get('MIGRATION_CONNECT_STRING');
+let dbUsers = monk(CONNECTION_STRING).get('users', { castIds: false });
+
+function processUsers (lastId) {
+  // specify a query to limit the affected users (empty for all users):
+  let query = {
+    migration: {$ne: MIGRATION_NAME},
+    'auth.timestamps.loggedin': {$gt: new Date('2018-04-01')},
+  };
+
+  if (lastId) {
+    query._id = {
+      $gt: lastId,
+    };
+  }
+
+  dbUsers.find(query, {
+    sort: {_id: 1},
+    limit: 100,
+    fields: [
+      '_id',
+      'auth',
+      'preferences',
+      'profile',
+    ], // specify fields we are interested in to limit retrieved data (empty if we're not reading data):
+  })
+    .then(updateUsers)
+    .catch((err) => {
+      console.log(err);
+      return exiting(1, `ERROR! ${  err}`);
+    });
+}
+
+let progressCount = 1000;
+let count = 0;
+
+function updateUsers (users) {
+  if (!users || users.length === 0) {
+    console.warn('All appropriate users found and modified.');
+    displayData();
+    return;
+  }
+
+  let userPromises = users.map(updateUser);
+  let lastUser = users[users.length - 1];
+
+  return Promise.all(userPromises)
+    .then(() => delay(7000))
+    .then(() => {
+      processUsers(lastUser._id);
+    });
+}
+
+function updateUser (user) {
+  count++;
+
+  dbUsers.update({_id: user._id}, {$set: {migration: MIGRATION_NAME}});
+
+  sendTxn(
+    user,
+    'username-change',
+    [{name: 'UNSUB_EMAIL_TYPE_URL', content: '/user/settings/notifications?unsubFrom=importantAnnouncements'},
+     {name: 'LOGIN_NAME', content: user.auth.local.username}]
+  );
+
+  if (count % progressCount === 0) console.warn(`${count} ${user._id}`);
+  if (user._id === authorUuid) console.warn(`${authorName} processed`);
+}
+
+function displayData () {
+  console.warn(`\n${count} users processed\n`);
+  return exiting(0);
+}
+
+function delay (t, v) {
+  return new Promise(function batchPause (resolve) {
+    setTimeout(resolve.bind(null, v), t);
+  });
+}
+
+function exiting (code, msg) {
+  code = code || 0; // 0 = success
+  if (code && !msg) {
+    msg = 'ERROR!';
+  }
+  if (msg) {
+    if (code) {
+      console.error(msg);
+    } else      {
+      console.log(msg);
+    }
+  }
+  process.exit(code);
+}
+
+module.exports = processUsers;

scripts/gdpr-delete-users.js

@@ -0,0 +1,82 @@
+/* eslint-disable no-console */
+import axios from 'axios';
+import { model as User } from '../website/server/models/user';
+import nconf from 'nconf';
+
+const AMPLITUDE_KEY = nconf.get('AMPLITUDE_KEY');
+const AMPLITUDE_SECRET = nconf.get('AMPLITUDE_SECRET');
+const BASE_URL = nconf.get('BASE_URL');
+
+async function _deleteAmplitudeData (userId, email) {
+  const response = await axios.post(
+    'https://amplitude.com/api/2/deletions/users',
+    {
+      user_ids: userId, // eslint-disable-line camelcase
+      requester: email,
+    },
+    {
+      auth: {
+        username: AMPLITUDE_KEY,
+        password: AMPLITUDE_SECRET,
+      },
+    }
+  );
+
+  console.log(`${response.status} ${response.statusText}`);
+}
+
+async function _deleteHabiticaData (user) {
+  await User.update(
+    {_id: user._id},
+    {$set: {
+      'auth.local.passwordHashMethod': 'bcrypt',
+      'auth.local.hashed_password': '$2a$10$QDnNh1j1yMPnTXDEOV38xOePEWFd4X8DSYwAM8XTmqmacG5X0DKjW',
+    }}
+  );
+  const response = await axios.delete(
+    `${BASE_URL}/api/v3/user`,
+    {
+      data: {
+        password: 'test',
+      },
+      headers: {
+        'x-api-user': user._id,
+        'x-api-key': user.apiToken,
+      },
+    }
+  );
+
+  console.log(`${response.status} ${response.statusText}`);
+  if (response.status === 200) console.log(`${user._id} removed. Last login: ${user.auth.timestamps.loggedin}`);
+}
+
+async function _processEmailAddress (email) {
+  const emailRegex = new RegExp(`^${email}`, 'i');
+  const users = await User.find({
+    $or: [
+      {'auth.local.email': emailRegex},
+      {'auth.facebook.emails.value': emailRegex},
+      {'auth.google.emails.value': emailRegex},
+    ]},
+  {
+    _id: 1,
+    apiToken: 1,
+    auth: 1,
+  }).exec();
+
+  if (users.length < 1) {
+    console.warn(`No users found with email address ${email}`);
+  } else {
+    for (const user of users) {
+      await _deleteAmplitudeData(user._id, email); // eslint-disable-line no-await-in-loop
+      await _deleteHabiticaData(user); // eslint-disable-line no-await-in-loop
+    }
+  }
+}
+
+function deleteUserData (emails) {
+  const emailPromises = emails.map(_processEmailAddress);
+  return Promise.all(emailPromises);
+}
+
+module.exports = deleteUserData;