Automatically mute users who attempt to post a slur, fixes #8062 (#8177)

* Initial psuedo-code for checking for slurs in messages

* Initial working prototype for blocking posting of slurs. Moved check from group.js to the chat api. Still needs: to permanently revoke chat privileges, to notify the moderators, a better method for checking for the blacklisted words, and a way to get the real list of words to check.

* Permanently revoke chat privileges when attempting to post a slur.

* Removed console logs

* Fixing rebase

* Do not moderate private groups

* Moved slur check to a generic check for banned words function

* Moved list of slurs to a separate file, fixed misplacement of return in ContainsBannedWords() function

* Slurs are blocked in both public and private groups

* Added code to send a slack message for slurs

* Fixed formatting issues

* Incorporated tectContainsBannedWords() function from PR 8197, added an argument to specify the list of banned words to check

* Added initial tests for blocking slurs and revoking chat priviliges

* Uncommented line to save revoked privileges

* Check that privileges are revoked in private groups

* Moved code to email/slack mods to chat api file

* Switched to BadRequest instead of NotFound error

* Restore chat privileges after test

* Using official placeholder slur

* Fixed line to export sendSubscriptionNotification function for slack

* Replaced muteUser function in user methods with a single line in the chat controller file

* Reset chatRevoked flag to false in a single line

* Switched method of setting chatRevoked flag so that it is updated locally and in the database

* First attempt at the muteUser function: revokes user's chat privileges and notifies moderators

* Manual merge for cherry-pick

* Initial working prototype for blocking posting of slurs. Moved check from group.js to the chat api. Still needs: to permanently revoke chat privileges, to notify the moderators, a better method for checking for the blacklisted words, and a way to get the real list of words to check.

* Permanently revoke chat privileges when attempting to post a slur.

* Removed console logs

* Created report to be sent to moderators via email

* Do not moderate private groups

* Moved slur check to a generic check for banned words function

* Moved list of slurs to a separate file, fixed misplacement of return in ContainsBannedWords() function

* Slurs are blocked in both public and private groups

* Added code to send a slack message for slurs

* Fixed formatting issues

* Incorporated tectContainsBannedWords() function from PR 8197, added an argument to specify the list of banned words to check

* Added initial tests for blocking slurs and revoking chat priviliges

* Uncommented line to save revoked privileges

* Check that privileges are revoked in private groups

* Moved code to email/slack mods to chat api file

* Switched to BadRequest instead of NotFound error

* Restore chat privileges after test

* Using official placeholder slur

* Fixed line to export sendSubscriptionNotification function for slack

* Replaced muteUser function in user methods with a single line in the chat controller file

* Reset chatRevoked flag to false in a single line

* Switched method of setting chatRevoked flag so that it is updated locally and in the database

* Removed some code that got re-added after rebase

* Tests for automatic slur muting pass but are incomplete (do not check that chatRevoked flag is true)

* Moved list of banned slurs to server side

* Added warning to bannedSlurs file

* Test chat privileges revoked when posting slur in public chat

* Fix issues left over after rebase (I hope)

* Added code to test for revoked chat privileges after posting a slur in a private group

* Moved banned slur message into locales message

* Added new code to check for banned slurs (parallels banned words code)

* Fixed AUTHOR_MOTAL_URL in sendTxn for slur blocking

* Added tests that email sent on attempted slur in chat post

* Created context for slur-related-tests, fixed sandboxing of email. Successfully tests that email.sendTxn is called, but the email content test fails

* commented out slack (for now) and cleaned up tests of sending email

* Successfully tests that slur-report-to-mods email is sent

* Slack message is sent, and testing works, but some user variables seem to only work when found in chat.js and passed to slack

* Made some fixes for lint, but not sure what to do about the camel case requirement fail, since that's how they're defined in other slack calls

* Slack tests pass, skipped camelcase check around those code blocks

* Fixed InternalServerError caused by slack messaging

* Updated chat privileges revoked error

* fix(locale): typo correction

test/api/v3/integration/chat/POST-chat.test.js

@@ -10,11 +10,17 @@ import {
   TAVERN_ID,
 } from '../../../../../website/server/models/group';
 import { v4 as generateUUID } from 'uuid';
+import * as email from '../../../../../website/server/libs/email';
+import { IncomingWebhook } from '@slack/client';
+import nconf from 'nconf';
+
+const BASE_URL = nconf.get('BASE_URL');
 
 describe('POST /chat', () => {
   let user, groupWithChat, member, additionalMember;
   let testMessage = 'Test Message';
   let testBannedWordMessage = 'TEST_PLACEHOLDER_SWEAR_WORD_HERE';
+  let testSlurMessage = 'message with TEST_PLACEHOLDER_SLUR_WORD_HERE';
 
   before(async () => {
     let { group, groupLeader, members } = await createAndPopulateGroup({
@@ -166,6 +172,114 @@ describe('POST /chat', () => {
     });
   });
 
+  context('banned slur', () => {
+    beforeEach(() => {
+      sandbox.spy(email, 'sendTxn');
+      sandbox.stub(IncomingWebhook.prototype, 'send');
+    });
+
+    afterEach(() => {
+      sandbox.restore();
+    });
+
+    it('errors and revokes privileges when chat message contains a banned slur', async () => {
+      await expect(user.post(`/groups/${groupWithChat._id}/chat`, { message: testSlurMessage})).to.eventually.be.rejected.and.eql({
+        code: 400,
+        error: 'BadRequest',
+        message: t('bannedSlurUsed'),
+      });
+
+      // Email sent to mods
+      await sleep(0.5);
+      expect(email.sendTxn).to.be.calledOnce;
+      expect(email.sendTxn.args[0][1]).to.be.eql('slur-report-to-mods');
+
+      // Slack message to mods
+      expect(IncomingWebhook.prototype.send).to.be.calledOnce;
+      /* eslint-disable camelcase */
+      expect(IncomingWebhook.prototype.send).to.be.calledWith({
+        text: `${user.profile.name} (${user.id}) tried to post a slur`,
+        attachments: [{
+          fallback: 'Slur Message',
+          color: 'danger',
+          author_name: `${user.profile.name} - ${user.auth.local.email} - ${user._id}`,
+          title: 'Slur in Test Guild',
+          title_link: `${BASE_URL}/#/options/groups/guilds/${groupWithChat.id}`,
+          text: testSlurMessage,
+          // footer: sandbox.match(/<.*?groupId=group-id&chatId=chat-id\|Flag this message>/),
+          mrkdwn_in: [
+            'text',
+          ],
+        }],
+      });
+      /* eslint-enable camelcase */
+
+      // Chat privileges are revoked
+      await expect(user.post(`/groups/${groupWithChat._id}/chat`, { message: testMessage})).to.eventually.be.rejected.and.eql({
+        code: 401,
+        error: 'NotAuthorized',
+        message: t('chatPrivilegesRevoked'),
+      });
+
+      // Restore chat privileges to continue testing
+      user.flags.chatRevoked = false;
+      await user.update({'flags.chatRevoked': false});
+    });
+
+    it('does not allow slurs in private groups', async () => {
+      let { group, members } = await createAndPopulateGroup({
+        groupDetails: {
+          name: 'Party',
+          type: 'party',
+          privacy: 'private',
+        },
+        members: 1,
+      });
+
+      await expect(members[0].post(`/groups/${group._id}/chat`, { message: testSlurMessage})).to.eventually.be.rejected.and.eql({
+        code: 400,
+        error: 'BadRequest',
+        message: t('bannedSlurUsed'),
+      });
+
+      // Email sent to mods
+      await sleep(0.5);
+      expect(email.sendTxn).to.be.calledThrice;
+      expect(email.sendTxn.args[2][1]).to.be.eql('slur-report-to-mods');
+
+      // Slack message to mods
+      expect(IncomingWebhook.prototype.send).to.be.calledOnce;
+      /* eslint-disable camelcase */
+      expect(IncomingWebhook.prototype.send).to.be.calledWith({
+        text: `${members[0].profile.name} (${members[0].id}) tried to post a slur`,
+        attachments: [{
+          fallback: 'Slur Message',
+          color: 'danger',
+          author_name: `${members[0].profile.name} - ${members[0].auth.local.email} - ${members[0]._id}`,
+          title: 'Slur in Party - (private party)',
+          title_link: undefined,
+          text: testSlurMessage,
+          // footer: sandbox.match(/<.*?groupId=group-id&chatId=chat-id\|Flag this message>/),
+          mrkdwn_in: [
+            'text',
+          ],
+        }],
+      });
+      /* eslint-enable camelcase */
+
+      // Chat privileges are revoked
+      await expect(members[0].post(`/groups/${groupWithChat._id}/chat`, { message: testMessage})).to.eventually.be.rejected.and.eql({
+        code: 401,
+        error: 'NotAuthorized',
+        message: t('chatPrivilegesRevoked'),
+      });
+
+      // Restore chat privileges to continue testing
+      members[0].flags.chatRevoked = false;
+      await members[0].update({'flags.chatRevoked': false});
+    });
+  });
+
   it('does not error when sending a message to a private guild with a user with revoked chat', async () => {
     let { group, members } = await createAndPopulateGroup({
       groupDetails: {

website/common/locales/en/groups.json

@@ -22,6 +22,7 @@
   "communityGuidelinesRead1": "Please read our",
   "communityGuidelinesRead2": "before chatting.",
   "bannedWordUsed": "Oops! Looks like this post contains a swearword, religious oath, or reference to an addictive substance or adult topic. Habitica has users from all backgrounds, so we keep our chat very clean. Feel free to edit your message so you can post it!",
+  "bannedSlurUsed": "Your post contained inappropriate language, and your chat privileges have been revoked.",
   "party": "Party",
   "createAParty": "Create A Party",
   "updatedParty": "Party settings updated.",

website/server/bannedSlurs.js

@@ -0,0 +1,56 @@
+/* eslint-disable no-multiple-empty-lines */
+
+// CONTENT WARNING:
+// This file contains slurs, swear words, religious oaths, and words related to addictive substance and adult topics.
+// Do not read this file if you do not want to be exposed to those words.
+// The words are stored in an array called `bannedSlurs` which is then exported with `module.exports = bannedSlurs;`
+// This file does not contain any other code.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+let bannedSlurs = ['TEST_PLACEHOLDER_SLUR_WORD_HERE'];
+
+module.exports = bannedSlurs;

website/server/controllers/api-v3/chat.js

@@ -15,6 +15,7 @@ import nconf from 'nconf';
 import Bluebird from 'bluebird';
 import bannedWords from '../../libs/bannedWords';
 import { TAVERN_ID } from '../../models/group';
+import bannedSlurs from '../../bannedSlurs';
 
 const FLAG_REPORT_EMAILS = nconf.get('FLAG_REPORT_EMAIL').split(',').map((email) => {
   return { email, canSend: true };
@@ -53,6 +54,44 @@ async function getAuthorEmailFromMessage (message) {
   }
 }
 
+// @TODO: Probably move this to a library
+function matchExact (r, str) {
+  let match = str.match(r);
+  return match !== null && match[0] !== null;
+}
+
+let bannedWordRegexs = [];
+for (let i = 0; i < bannedWords.length; i += 1) {
+  let word = bannedWords[i];
+  let regEx = new RegExp(`\\b([^a-z]+)?${word.toLowerCase()}([^a-z]+)?\\b`);
+  bannedWordRegexs.push(regEx);
+}
+
+function textContainsBannedWords (message) {
+  for (let i = 0; i < bannedWordRegexs.length; i += 1) {
+    let regEx = bannedWordRegexs[i];
+    if (matchExact(regEx, message.toLowerCase())) return true;
+  }
+
+  return false;
+}
+
+let bannedSlurRegexs = [];
+for (let i = 0; i < bannedSlurs.length; i += 1) {
+  let word = bannedSlurs[i];
+  let regEx = new RegExp(`\\b([^a-z]+)?${word.toLowerCase()}([^a-z]+)?\\b`);
+  bannedSlurRegexs.push(regEx);
+}
+
+function textContainsBannedSlur (message) {
+  for (let i = 0; i < bannedSlurRegexs.length; i += 1) {
+    let regEx = bannedSlurRegexs[i];
+    if (matchExact(regEx, message.toLowerCase())) return true;
+  }
+
+  return false;
+}
+
 /**
  * @api {get} /api/v3/groups/:groupId/chat Get chat messages from a group
  * @apiName GetChat
@@ -85,28 +124,6 @@ api.getChat = {
   },
 };
 
-// @TODO: Probably move this to a library
-function matchExact (r, str) {
-  let match = str.match(r);
-  return match !== null && match[0] !== null;
-}
-
-let bannedWordRegexs = [];
-for (let i = 0; i < bannedWords.length; i += 1) {
-  let word = bannedWords[i];
-  let regEx = new RegExp(`\\b([^a-z]+)?${word.toLowerCase()}([^a-z]+)?\\b`);
-  bannedWordRegexs.push(regEx);
-}
-
-function textContainsBannedWords (message) {
-  for (let i = 0; i < bannedWordRegexs.length; i += 1) {
-    let regEx = bannedWordRegexs[i];
-    if (matchExact(regEx, message.toLowerCase())) return true;
-  }
-
-  return false;
-}
-
 /**
  * @api {post} /api/v3/groups/:groupId/chat Post chat message to a group
  * @apiName PostChat
@@ -139,6 +156,44 @@ api.postChat = {
 
     let group = await Group.getGroup({user, groupId});
 
+    // Check message for banned slurs
+    if (textContainsBannedSlur(req.body.message)) {
+      let message = req.body.message;
+      user.flags.chatRevoked = true;
+      await user.save();
+
+      // Email the mods
+      let authorEmail = getUserInfo(user, ['email']).email;
+      let groupUrl = getGroupUrl(group);
+
+      let report =  [
+        {name: 'MESSAGE_TIME', content: (new Date()).toString()},
+        {name: 'MESSAGE_TEXT', content: message},
+
+        {name: 'AUTHOR_USERNAME', content: user.profile.name},
+        {name: 'AUTHOR_UUID', content: user._id},
+        {name: 'AUTHOR_EMAIL', content: authorEmail},
+        {name: 'AUTHOR_MODAL_URL', content: `/static/front/#?memberId=${user._id}`},
+
+        {name: 'GROUP_NAME', content: group.name},
+        {name: 'GROUP_TYPE', content: group.type},
+        {name: 'GROUP_ID', content: group._id},
+        {name: 'GROUP_URL', content: groupUrl},
+      ];
+
+      sendTxn(FLAG_REPORT_EMAILS, 'slur-report-to-mods', report);
+
+      // Slack the mods
+      slack.sendSlurNotification({
+        authorEmail,
+        author: user,
+        group,
+        message,
+      });
+
+      throw new BadRequest(res.t('bannedSlurUsed'));
+    }
+
     if (!group) throw new NotFound(res.t('groupNotFound'));
     if (group.privacy !== 'private' && user.flags.chatRevoked) {
       throw new NotAuthorized(res.t('chatPrivilegesRevoked'));

website/server/libs/slack.js

@@ -93,3 +93,49 @@ module.exports = {
   sendFlagNotification,
   sendSubscriptionNotification,
 };
+
+function sendSlurNotification ({
+  authorEmail,
+  author,
+  group,
+  message,
+}) {
+  if (!SLACK_FLAGGING_URL) {
+    return;
+  }
+  let titleLink;
+  let authorName;
+  let title = `Slur in ${group.name}`;
+  let text = `${author.profile.name} (${author._id}) tried to post a slur`;
+
+  if (group.id === TAVERN_ID) {
+    titleLink = `${BASE_URL}/#/options/groups/tavern`;
+  } else if (group.privacy === 'public') {
+    titleLink = `${BASE_URL}/#/options/groups/guilds/${group.id}`;
+  } else {
+    title += ` - (${group.privacy} ${group.type})`;
+  }
+
+  authorName = `${author.profile.name} - ${authorEmail} - ${author.id}`;
+
+  flagSlack.send({
+    text,
+    attachments: [{
+      fallback: 'Slur Message',
+      color: 'danger',
+      author_name: authorName,
+      title,
+      title_link: titleLink,
+      text: message,
+      // What to replace the footer with?
+      // footer: `<${SLACK_FLAGGING_FOOTER_LINK}?groupId=${group.id}&chatId=${message.id}|Flag this message>`,
+      mrkdwn_in: [
+        'text',
+      ],
+    }],
+  });
+}
+
+module.exports = {
+  sendFlagNotification, sendSubscriptionNotification, sendSlurNotification,
+};

website/server/models/user/methods.js

@@ -1,11 +1,13 @@
 import moment from 'moment';
 import common from '../../../common';
+
 import Bluebird from 'bluebird';
 import {
   chatDefaults,
   TAVERN_ID,
   model as Group,
 } from '../group';
+
 import { defaults, map, flatten, flow, compact, uniq, partialRight } from 'lodash';
 import { model as UserNotification } from '../userNotification';
 import schema from './schema';