Move Chat to Model (#9703)

* Began moving group chat to separate model

* Fixed lint issue

* Updated delete chat with new model

* Updated flag chat to support model

* Updated like chat to use model

* Fixed duplicate code and chat messages

* Added note about concat chat

* Updated clear flags to user new model

* Updated more chat checks when loading get group

* Fixed spell test and back save

* Moved get chat to json method

* Updated flagging with new chat model

* Added missing await

* Fixed chat user styles. Fixed spell group test

* Added new model to quest chat and group plan chat

* Removed extra timestamps. Added limit check for group plans

* Updated tests

* Synced id fields

* Fixed id creation

* Add meta and fixed tests

* Fixed group quest accept test

* Updated puppeteer

* Added migration

* Export vars

* Updated comments

migrations/groups/migrate-chat.js

@@ -0,0 +1,52 @@
+// @migrationName = 'MigrateGroupChat';
+// @authorName = 'TheHollidayInn'; // in case script author needs to know when their ...
+// @authorUuid = ''; // ... own data is done
+
+
+/*
+ * This migration move ass chat off of groups and into their own model
+ */
+
+import { model as Group } from '../../website/server/models/group';
+import { model as Chat } from '../../website/server/models/chat';
+
+async function moveGroupChatToModel (skip = 0) {
+  const groups = await Group.find({})
+    .limit(50)
+    .skip(skip)
+    .sort({ _id: -1 })
+    .exec();
+
+  if (groups.length === 0) {
+    console.log('End of groups');
+    process.exit();
+  }
+
+  const promises = groups.map(group => {
+    const chatpromises = group.chat.map(message => {
+      const newChat = new Chat();
+      Object.assign(newChat, message);
+      newChat._id = message.id;
+      newChat.groupId = group._id;
+
+      return newChat.save();
+    });
+
+    group.chat = [];
+    chatpromises.push(group.save());
+
+    return chatpromises;
+  });
+
+
+  const reducedPromises = promises.reduce((acc, curr) => {
+    acc = acc.concat(curr);
+    return acc;
+  }, []);
+
+  console.log(reducedPromises);
+  await Promise.all(reducedPromises);
+  moveGroupChatToModel(skip + 50);
+}
+
+module.exports = moveGroupChatToModel;

migrations/migration-runner.js

@@ -17,5 +17,5 @@ function setUpServer () {
 setUpServer();
 
 // Replace this with your migration
-const processUsers = require('./20180125_clean_new_notifications.js');
+const processUsers = require('./groups/migrate-chat.js');
 processUsers();

package-lock.json

@@ -5631,7 +5631,7 @@
       "dependencies": {
         "onetime": {
           "version": "1.1.0",
-          "resolved": "http://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz",
+          "resolved": "https://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz",
           "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k="
         }
       }
@@ -6369,7 +6369,7 @@
     },
     "event-stream": {
       "version": "3.3.4",
-      "resolved": "http://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz",
+      "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz",
       "integrity": "sha1-SrTJoPWlTbkzi0w02Gv86PSzVXE=",
       "requires": {
         "duplexer": "0.1.1",
@@ -15754,7 +15754,7 @@
     },
     "pinkie-promise": {
       "version": "2.0.1",
-      "resolved": "http://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz",
       "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=",
       "requires": {
         "pinkie": "2.0.4"
@@ -17822,14 +17822,14 @@
       "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
     },
     "puppeteer": {
-      "version": "1.2.0",
+      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.3.0.tgz",
-      "integrity": "sha512-4sY/6mB7+kNPGAzPGKq65tH0VG3ohUEkXHuOReB9K/tw3m1TqifYmxnMR/uDeci/UPwyk5K1gWYh8rw0U0Zscw==",
+      "integrity": "sha512-wx10aPQPpGJVxdB6yoDSLm9p4rCwARUSLMVV0bx++owuqkvviXKyiFM3EWsywaFmjOKNPXacIjplF7xhHiFP3w==",
       "dev": true,
       "requires": {
         "debug": "2.6.9",
         "extract-zip": "1.6.6",
-        "https-proxy-agent": "2.2.0",
+        "https-proxy-agent": "2.2.1",
         "mime": "1.6.0",
         "progress": "2.0.0",
         "proxy-from-env": "1.0.0",
@@ -17847,9 +17847,9 @@
           }
         },
         "https-proxy-agent": {
-          "version": "2.2.0",
+          "version": "2.2.1",
-          "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.0.tgz",
+          "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz",
-          "integrity": "sha512-uUWcfXHvy/dwfM9bqa6AozvAjS32dZSTUYd/4SEpYKRg6LEcPLshksnQYRudM9AyNvUARMfAg5TLjUDyX/K4vA==",
+          "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==",
           "dev": true,
           "requires": {
             "agent-base": "4.2.0",
@@ -18822,9 +18822,9 @@
       }
     },
     "sass-loader": {
-      "version": "6.0.7",
+      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-6.0.7.tgz",
+      "resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-7.0.1.tgz",
-      "integrity": "sha512-JoiyD00Yo1o61OJsoP2s2kb19L1/Y2p3QFcCdWdF6oomBGKVYuZyqHWemRBfQ2uGYsk+CH3eCguXNfpjzlcpaA==",
+      "integrity": "sha512-MeVVJFejJELlAbA7jrRchi88PGP6U9yIfqyiG+bBC4a9s2PX+ulJB9h8bbEohtPBfZmlLhNZ0opQM9hovRXvlw==",
       "requires": {
         "clone-deep": "2.0.2",
         "loader-utils": "1.1.0",

package.json

@@ -177,7 +177,7 @@
     "mocha": "^5.0.5",
     "monk": "^6.0.5",
     "nightwatch": "^0.9.20",
-    "puppeteer": "^1.2.0",
+    "puppeteer": "^1.3.0",
     "require-again": "^2.0.0",
     "selenium-server": "^3.11.0",
     "sinon": "^4.5.0",

test/api/v3/integration/chat/DELETE-chat_id.test.js

@@ -53,16 +53,26 @@ describe('DELETE /groups/:groupId/chat/:chatId', () => {
 
     it('allows creator to delete a their message', async () => {
       await user.del(`/groups/${groupWithChat._id}/chat/${nextMessage.id}`);
-      let messages = await user.get(`/groups/${groupWithChat._id}/chat/`);
+
-      expect(messages).is.an('array');
+      const returnedMessages = await user.get(`/groups/${groupWithChat._id}/chat/`);
-      expect(messages).to.not.include(nextMessage);
+      const messageFromUser = returnedMessages.find(returnedMessage => {
+        return returnedMessage.id === nextMessage.id;
+      });
+
+      expect(returnedMessages).is.an('array');
+      expect(messageFromUser).to.not.exist;
     });
 
     it('allows admin to delete another user\'s message', async () => {
       await admin.del(`/groups/${groupWithChat._id}/chat/${nextMessage.id}`);
-      let messages = await user.get(`/groups/${groupWithChat._id}/chat/`);
+
-      expect(messages).is.an('array');
+      const returnedMessages = await user.get(`/groups/${groupWithChat._id}/chat/`);
-      expect(messages).to.not.include(nextMessage);
+      const messageFromUser = returnedMessages.find(returnedMessage => {
+        return returnedMessage.id === nextMessage.id;
+      });
+
+      expect(returnedMessages).is.an('array');
+      expect(messageFromUser).to.not.exist;
     });
 
     it('returns empty when previous message parameter is passed and the last message was deleted', async () => {
@@ -71,9 +81,9 @@ describe('DELETE /groups/:groupId/chat/:chatId', () => {
     });
 
     it('returns the update chat when previous message parameter is passed and the chat is updated', async () => {
-      let deleteResult = await user.del(`/groups/${groupWithChat._id}/chat/${nextMessage.id}?previousMsg=${message.id}`);
+      const updatedChat = await user.del(`/groups/${groupWithChat._id}/chat/${nextMessage.id}?previousMsg=${message.id}`);
 
-      expect(deleteResult[0].id).to.eql(message.id);
+      expect(updatedChat[0].id).to.eql(message.id);
     });
   });
 });

test/api/v3/integration/chat/GET-chat.test.js

@@ -23,14 +23,14 @@ describe('GET /groups/:groupId/chat', () => {
         privacy: 'public',
       }, {
         chat: [
-          {text: 'Hello', flags: {}},
+          {text: 'Hello', flags: {}, id: 1},
-          {text: 'Welcome to the Guild', flags: {}},
+          {text: 'Welcome to the Guild', flags: {}, id: 2},
         ],
       });
     });
 
     it('returns Guild chat', async () => {
-      let chat = await user.get(`/groups/${group._id}/chat`);
+      const chat = await user.get(`/groups/${group._id}/chat`);
 
       expect(chat).to.eql(group.chat);
     });

test/api/v3/integration/chat/POST-chat.test.js

@@ -381,9 +381,11 @@ describe('POST /chat', () => {
   });
 
   it('creates a chat', async () => {
-    let message = await user.post(`/groups/${groupWithChat._id}/chat`, { message: testMessage});
+    const newMessage = await user.post(`/groups/${groupWithChat._id}/chat`, { message: testMessage});
+    const groupMessages = await user.get(`/groups/${groupWithChat._id}/chat`);
 
-    expect(message.message.id).to.exist;
+    expect(newMessage.message.id).to.exist;
+    expect(groupMessages[0].id).to.exist;
   });
 
   it('creates a chat with user styles', async () => {

test/api/v3/integration/quests/POST-groups_groupId_quests_accept.test.js

@@ -4,6 +4,7 @@ import {
   generateUser,
   sleep,
 } from '../../../../helpers/api-v3-integration.helper';
+import { model as Chat } from '../../../../../website/server/models/chat';
 
 describe('POST /groups/:groupId/quests/accept', () => {
   const PET_QUEST = 'whale';
@@ -155,10 +156,11 @@ describe('POST /groups/:groupId/quests/accept', () => {
       // quest will start after everyone has accepted
       await partyMembers[1].post(`/groups/${questingGroup._id}/quests/accept`);
 
-      await questingGroup.sync();
+      const groupChat = await Chat.find({ groupId: questingGroup._id }).exec();
-      expect(questingGroup.chat[0].text).to.exist;
+
-      expect(questingGroup.chat[0]._meta).to.exist;
+      expect(groupChat[0].text).to.exist;
-      expect(questingGroup.chat[0]._meta).to.have.all.keys(['participatingMembers']);
+      expect(groupChat[0]._meta).to.exist;
+      expect(groupChat[0]._meta).to.have.all.keys(['participatingMembers']);
 
       let returnedGroup = await leader.get(`/groups/${questingGroup._id}`);
       expect(returnedGroup.chat[0]._meta).to.be.undefined;

test/api/v3/integration/quests/POST-groups_groupId_quests_force-start.test.js

@@ -4,6 +4,7 @@ import {
   generateUser,
   sleep,
 } from '../../../../helpers/api-v3-integration.helper';
+import { model as Chat } from '../../../../../website/server/models/chat';
 
 describe('POST /groups/:groupId/quests/force-start', () => {
   const PET_QUEST = 'whale';
@@ -241,11 +242,13 @@ describe('POST /groups/:groupId/quests/force-start', () => {
 
       await questingGroup.sync();
 
-      expect(questingGroup.chat[0].text).to.exist;
+      const groupChat = await Chat.find({ groupId: questingGroup._id }).exec();
-      expect(questingGroup.chat[0]._meta).to.exist;
-      expect(questingGroup.chat[0]._meta).to.have.all.keys(['participatingMembers']);
 
-      let returnedGroup = await leader.get(`/groups/${questingGroup._id}`);
+      expect(groupChat[0].text).to.exist;
+      expect(groupChat[0]._meta).to.exist;
+      expect(groupChat[0]._meta).to.have.all.keys(['participatingMembers']);
+
+      const returnedGroup = await leader.get(`/groups/${questingGroup._id}`);
       expect(returnedGroup.chat[0]._meta).to.be.undefined;
     });
   });

test/api/v3/integration/quests/POST-groups_groupId_quests_invite.test.js

@@ -5,6 +5,7 @@ import {
 } from '../../../../helpers/api-v3-integration.helper';
 import { v4 as generateUUID } from 'uuid';
 import { quests as questScrolls } from '../../../../../website/common/script/content';
+import { model as Chat } from '../../../../../website/server/models/chat';
 
 describe('POST /groups/:groupId/quests/invite/:questKey', () => {
   let questingGroup;
@@ -199,11 +200,11 @@ describe('POST /groups/:groupId/quests/invite/:questKey', () => {
 
       await groupLeader.post(`/groups/${group._id}/quests/invite/${PET_QUEST}`);
 
-      await group.sync();
+      const groupChat = await Chat.find({ groupId: group._id }).exec();
 
-      expect(group.chat[0].text).to.exist;
+      expect(groupChat[0].text).to.exist;
-      expect(group.chat[0]._meta).to.exist;
+      expect(groupChat[0]._meta).to.exist;
-      expect(group.chat[0]._meta).to.have.all.keys(['participatingMembers']);
+      expect(groupChat[0]._meta).to.have.all.keys(['participatingMembers']);
 
       let returnedGroup = await groupLeader.get(`/groups/${group._id}`);
       expect(returnedGroup.chat[0]._meta).to.be.undefined;

test/api/v3/integration/quests/POST-groups_groupid_quests_abort.test.js

@@ -90,7 +90,7 @@ describe('POST /groups/:groupId/quests/abort', () => {
     await partyMembers[0].post(`/groups/${questingGroup._id}/quests/accept`);
     await partyMembers[1].post(`/groups/${questingGroup._id}/quests/accept`);
 
-    let stub = sandbox.stub(Group.prototype, 'sendChat');
+    let stub = sandbox.spy(Group.prototype, 'sendChat');
 
     let res = await leader.post(`/groups/${questingGroup._id}/quests/abort`);
     await Promise.all([

test/api/v3/integration/quests/POST-groups_groupid_quests_reject.test.js

@@ -5,6 +5,7 @@ import {
   sleep,
 } from '../../../../helpers/api-v3-integration.helper';
 import { v4 as generateUUID } from 'uuid';
+import { model as Chat } from '../../../../../website/server/models/chat';
 
 describe('POST /groups/:groupId/quests/reject', () => {
   let questingGroup;
@@ -185,11 +186,12 @@ describe('POST /groups/:groupId/quests/reject', () => {
       await leader.post(`/groups/${questingGroup._id}/quests/invite/${PET_QUEST}`);
       await partyMembers[0].post(`/groups/${questingGroup._id}/quests/accept`);
       await partyMembers[1].post(`/groups/${questingGroup._id}/quests/reject`);
-      await questingGroup.sync();
 
-      expect(questingGroup.chat[0].text).to.exist;
+      const groupChat = await Chat.find({ groupId: questingGroup._id }).exec();
-      expect(questingGroup.chat[0]._meta).to.exist;
+
-      expect(questingGroup.chat[0]._meta).to.have.all.keys(['participatingMembers']);
+      expect(groupChat[0].text).to.exist;
+      expect(groupChat[0]._meta).to.exist;
+      expect(groupChat[0]._meta).to.have.all.keys(['participatingMembers']);
 
       let returnedGroup = await leader.get(`/groups/${questingGroup._id}`);
       expect(returnedGroup.chat[0]._meta).to.be.undefined;

test/api/v3/integration/user/POST-user_class_cast_spellId.test.js

@@ -180,11 +180,13 @@ describe('POST /user/class/cast/:spellId', () => {
       members: 1,
     });
     await groupLeader.update({'stats.mp': 200, 'stats.class': 'wizard', 'stats.lvl': 13});
+
     await groupLeader.post('/user/class/cast/earth');
     await sleep(1);
-    await group.sync();
+    const groupMessages = await groupLeader.get(`/groups/${group._id}/chat`);
-    expect(group.chat[0]).to.exist;
+
-    expect(group.chat[0].uuid).to.equal('system');
+    expect(groupMessages[0]).to.exist;
+    expect(groupMessages[0].uuid).to.equal('system');
   });
 
   it('Ethereal Surge does not recover mp of other mages', async () => {
@@ -226,7 +228,7 @@ describe('POST /user/class/cast/:spellId', () => {
     await groupLeader.post('/user/class/cast/earth', {quantity: 2});
 
     await sleep(1);
-    await group.sync();
+    group = await groupLeader.get(`/groups/${group._id}`);
 
     expect(group.chat[0]).to.exist;
     expect(group.chat[0].uuid).to.equal('system');

test/api/v3/unit/models/group.test.js

@@ -182,7 +182,7 @@ describe('Group Model', () => {
           await party.startQuest(questLeader);
           await party.save();
 
-          sendChatStub = sandbox.stub(Group.prototype, 'sendChat');
+          sendChatStub = sandbox.spy(Group.prototype, 'sendChat');
         });
 
         afterEach(() => sendChatStub.restore());
@@ -378,7 +378,7 @@ describe('Group Model', () => {
           await party.startQuest(questLeader);
           await party.save();
 
-          sendChatStub = sandbox.stub(Group.prototype, 'sendChat');
+          sendChatStub = sandbox.spy(Group.prototype, 'sendChat');
         });
 
         afterEach(() => sendChatStub.restore());
@@ -918,21 +918,8 @@ describe('Group Model', () => {
         sandbox.spy(User, 'update');
       });
 
-      it('puts message at top of chat array', () => {
-        let oldMessage = {
-          text: 'a message',
-        };
-        party.chat.push(oldMessage, oldMessage, oldMessage);
-
-        party.sendChat('a new message', {_id: 'user-id', profile: { name: 'user name' }});
-
-        expect(party.chat).to.have.a.lengthOf(4);
-        expect(party.chat[0].text).to.eql('a new message');
-        expect(party.chat[0].uuid).to.eql('user-id');
-      });
-
       it('formats message', () => {
-        party.sendChat('a new message', {
+        const chatMessage = party.sendChat('a new message', {
           _id: 'user-id',
           profile: { name: 'user name' },
           contributor: {
@@ -947,11 +934,11 @@ describe('Group Model', () => {
           },
         });
 
-        let chat = party.chat[0];
+        const chat = chatMessage;
 
         expect(chat.text).to.eql('a new message');
         expect(validator.isUUID(chat.id)).to.eql(true);
-        expect(chat.timestamp).to.be.a('number');
+        expect(chat.timestamp).to.be.a('date');
         expect(chat.likes).to.eql({});
         expect(chat.flags).to.eql({});
         expect(chat.flagCount).to.eql(0);
@@ -962,13 +949,11 @@ describe('Group Model', () => {
       });
 
       it('formats message as system if no user is passed in', () => {
-        party.sendChat('a system message');
+        const chat = party.sendChat('a system message');
-
-        let chat = party.chat[0];
 
         expect(chat.text).to.eql('a system message');
         expect(validator.isUUID(chat.id)).to.eql(true);
-        expect(chat.timestamp).to.be.a('number');
+        expect(chat.timestamp).to.be.a('date');
         expect(chat.likes).to.eql({});
         expect(chat.flags).to.eql({});
         expect(chat.flagCount).to.eql(0);

website/server/controllers/api-v3/chat.js

@@ -1,12 +1,12 @@
 import { authWithHeaders } from '../../middlewares/auth';
 import { model as Group } from '../../models/group';
 import { model as User } from '../../models/user';
+import { model as Chat } from '../../models/chat';
 import {
   BadRequest,
   NotFound,
   NotAuthorized,
 } from '../../libs/errors';
-import _ from 'lodash';
 import { removeFromArray } from '../../libs/collectionManipulators';
 import { getUserInfo, getGroupUrl, sendTxn } from '../../libs/email';
 import slack from '../../libs/slack';
@@ -70,10 +70,12 @@ api.getChat = {
     let validationErrors = req.validationErrors();
     if (validationErrors) throw validationErrors;
 
-    let group = await Group.getGroup({user, groupId: req.params.groupId, fields: 'chat'});
+    const groupId = req.params.groupId;
+    let group = await Group.getGroup({user, groupId, fields: 'chat'});
     if (!group) throw new NotFound(res.t('groupNotFound'));
 
-    res.respond(200, Group.toJSONCleanChat(group, user).chat);
+    const groupChat = await Group.toJSONCleanChat(group, user);
+    res.respond(200, groupChat.chat);
   },
 };
 
@@ -164,35 +166,35 @@ api.postChat = {
       }
     }
 
-    let lastClientMsg = req.query.previousMsg;
+    const chatRes = await Group.toJSONCleanChat(group, user);
+    const lastClientMsg = req.query.previousMsg;
     chatUpdated = lastClientMsg && group.chat && group.chat[0] && group.chat[0].id !== lastClientMsg ? true : false;
 
     if (group.checkChatSpam(user)) {
       throw new NotAuthorized(res.t('messageGroupChatSpam'));
     }
 
-    let newChatMessage = group.sendChat(req.body.message, user);
+    const newChatMessage = group.sendChat(req.body.message, user);
-
+    let toSave = [newChatMessage.save()];
-    let toSave = [group.save()];
 
     if (group.type === 'party') {
-      user.party.lastMessageSeen = group.chat[0].id;
+      user.party.lastMessageSeen = newChatMessage.id;
       toSave.push(user.save());
     }
 
-    let [savedGroup] = await Promise.all(toSave);
+    await Promise.all(toSave);
 
-    // realtime chat is only enabled for private groups (for now only for parties)
+    // @TODO: rethink if we want real-time
-    if (savedGroup.privacy === 'private' && savedGroup.type === 'party') {
+    if (group.privacy === 'private' && group.type === 'party') {
       // req.body.pusherSocketId is sent from official clients to identify the sender user's real time socket
       // see https://pusher.com/docs/server_api_guide/server_excluding_recipients
-      pusher.trigger(`presencegroup${savedGroup._id}`, 'newchat', newChatMessage, req.body.pusherSocketId);
+      pusher.trigger(`presence-group-${group._id}`, 'new-chat', newChatMessage, req.body.pusherSocketId);
     }
 
     if (chatUpdated) {
-      res.respond(200, {chat: Group.toJSONCleanChat(savedGroup, user).chat});
+      res.respond(200, {chat: chatRes.chat});
     } else {
-      res.respond(200, {message: savedGroup.chat[0]});
+      res.respond(200, {message: newChatMessage});
     }
 
     group.sendGroupChatReceivedWebhooks(newChatMessage);
@@ -233,22 +235,16 @@ api.likeChat = {
     let group = await Group.getGroup({user, groupId});
     if (!group) throw new NotFound(res.t('groupNotFound'));
 
-    let message = _.find(group.chat, {id: req.params.chatId});
+    let message = await Chat.findOne({id: req.params.chatId}).exec();
     if (!message) throw new NotFound(res.t('messageGroupChatNotFound'));
-    // TODO correct this error type
+    // @TODO correct this error type
     if (message.uuid === user._id) throw new NotFound(res.t('messageGroupChatLikeOwnMessage'));
 
-    let update = {$set: {}};
-
     if (!message.likes) message.likes = {};
-
     message.likes[user._id] = !message.likes[user._id];
-    update.$set[`chat.$.likes.${user._id}`] = message.likes[user._id];
+    message.markModified('likes');
+    await message.save();
 
-    await Group.update(
-      {_id: group._id, 'chat.id': message.id},
-      update
-    ).exec();
     res.respond(200, message); // TODO what if the message is flagged and shouldn't be returned?
   },
 };
@@ -334,15 +330,11 @@ api.clearChatFlags = {
     });
     if (!group) throw new NotFound(res.t('groupNotFound'));
 
-    let message = _.find(group.chat, {id: chatId});
+    let message = await Chat.findOne({id: chatId}).exec();
     if (!message) throw new NotFound(res.t('messageGroupChatNotFound'));
 
     message.flagCount = 0;
-
+    await message.save();
-    await Group.update(
-      {_id: group._id, 'chat.id': message.id},
-      {$set: {'chat.$.flagCount': message.flagCount}}
-    ).exec();
 
     let adminEmailContent = getUserInfo(user, ['email']).email;
     let authorEmail = getAuthorEmailFromMessage(message);
@@ -466,25 +458,22 @@ api.deleteChat = {
     let group = await Group.getGroup({user, groupId, fields: 'chat'});
     if (!group) throw new NotFound(res.t('groupNotFound'));
 
-    let message = _.find(group.chat, {id: chatId});
+    let message = await Chat.findOne({id: chatId}).exec();
     if (!message) throw new NotFound(res.t('messageGroupChatNotFound'));
 
     if (user._id !== message.uuid && !user.contributor.admin) {
       throw new NotAuthorized(res.t('onlyCreatorOrAdminCanDeleteChat'));
     }
 
-    let lastClientMsg = req.query.previousMsg;
+    const chatRes = await Group.toJSONCleanChat(group, user);
-    let chatUpdated = lastClientMsg && group.chat && group.chat[0] && group.chat[0].id !== lastClientMsg ? true : false;
+    const lastClientMsg = req.query.previousMsg;
+    const chatUpdated = lastClientMsg && group.chat && group.chat[0] && group.chat[0].id !== lastClientMsg ? true : false;
 
-    await Group.update(
+    await Chat.remove({_id: message._id}).exec();
-      {_id: group._id},
-      {$pull: {chat: {id: chatId}}}
-    ).exec();
 
     if (chatUpdated) {
-      let chatRes = Group.toJSONCleanChat(group, user).chat;
+      removeFromArray(chatRes.chat, {id: chatId});
-      removeFromArray(chatRes, {id: chatId});
+      res.respond(200, chatRes.chat);
-      res.respond(200, chatRes);
     } else {
       res.respond(200, {});
     }

website/server/controllers/api-v3/groups.js

@@ -392,7 +392,7 @@ api.getGroup = {
       throw new NotFound(res.t('groupNotFound'));
     }
 
-    let groupJson = Group.toJSONCleanChat(group, user);
+    let groupJson = await Group.toJSONCleanChat(group, user);
 
     if (groupJson.leader === user._id) {
       groupJson.purchased.plan = group.purchased.plan.toObject();
@@ -456,7 +456,7 @@ api.updateGroup = {
     _.assign(group, _.merge(group.toObject(), Group.sanitizeUpdate(req.body)));
 
     let savedGroup = await group.save();
-    let response = Group.toJSONCleanChat(savedGroup, user);
+    let response = await Group.toJSONCleanChat(savedGroup, user);
 
     // If the leader changed fetch new data, otherwise use authenticated user
     if (response.leader !== user._id) {
@@ -625,7 +625,7 @@ api.joinGroup = {
 
     promises = await Promise.all(promises);
 
-    let response = Group.toJSONCleanChat(promises[0], user);
+    let response = await Group.toJSONCleanChat(promises[0], user);
     let leader = await User.findById(response.leader).select(nameFields).exec();
     if (leader) {
       response.leader = leader.toJSON({minimize: true});

website/server/controllers/api-v3/quests.js

@@ -421,7 +421,8 @@ api.abortQuest = {
     if (user._id !== group.leader && user._id !== group.quest.leader) throw new NotAuthorized(res.t('onlyLeaderAbortQuest'));
 
     let questName = questScrolls[group.quest.key].text('en');
-    group.sendChat(`\`${user.profile.name} aborted the party quest ${questName}.\``);
+    const newChatMessage = group.sendChat(`\`${user.profile.name} aborted the party quest ${questName}.\``);
+    await newChatMessage.save();
 
     let memberUpdates = User.update({
       'party._id': groupId,

website/server/controllers/api-v3/tasks/groups.js

@@ -195,13 +195,15 @@ api.assignTask = {
 
     if (canNotEditTasks(group, user, assignedUserId)) throw new NotAuthorized(res.t('onlyGroupLeaderCanEditTasks'));
 
+    let promises = [];
+
     // User is claiming the task
     if (user._id === assignedUserId) {
       let message = res.t('userIsClamingTask', {username: user.profile.name, task: task.text});
-      group.sendChat(message);
+      const newMessage = group.sendChat(message);
+      promises.push(newMessage.save());
     }
 
-    let promises = [];
     promises.push(group.syncTask(task, assignedUser));
     promises.push(group.save());
     await Promise.all(promises);

website/server/controllers/api-v3/user/spells.js

@@ -130,8 +130,8 @@ api.castSpell = {
 
       if (party && !spell.silent) {
         let message = `\`${user.profile.name} casts ${spell.text()}${targetType === 'user' ? ` on ${partyMembers.profile.name}` : ' for the party'}.\``;
-        party.sendChat(message);
+        const newChatMessage = party.sendChat(message);
-        await party.save();
+        await newChatMessage.save();
       }
     }
   },

website/server/libs/chat/group-chat.js

@@ -0,0 +1,24 @@
+import { model as Chat } from '../../models/chat';
+import { MAX_CHAT_COUNT, MAX_SUBBED_GROUP_CHAT_COUNT } from '../../models/group';
+
+// @TODO: Don't use this method when the group can be saved.
+export async function getGroupChat (group) {
+  const maxChatCount = group.isSubscribed() ? MAX_SUBBED_GROUP_CHAT_COUNT : MAX_CHAT_COUNT;
+
+  const groupChat = await Chat.find({groupId: group._id})
+    .limit(maxChatCount)
+    .sort('-timestamp')
+    .exec();
+
+  // @TODO: Concat old chat to keep continuity of chat stored on group object
+  const currentGroupChat = group.chat || [];
+  const concatedGroupChat = groupChat.concat(currentGroupChat);
+
+  group.chat = concatedGroupChat.reduce((previous, current) => {
+    const foundMessage = previous.find(message => {
+      return message.id === current.id;
+    });
+    if (!foundMessage) previous.push(current);
+    return previous;
+  }, []);
+}

website/server/libs/chatReporting/groupChatReporter.js

@@ -1,4 +1,3 @@
-import find from 'lodash/find';
 import nconf from 'nconf';
 
 import ChatReporter from './chatReporter';
@@ -9,6 +8,7 @@ import {
 import { getGroupUrl, sendTxn } from '../email';
 import slack from '../slack';
 import { model as Group } from '../../models/group';
+import { model as Chat } from '../../models/chat';
 
 const COMMUNITY_MANAGER_EMAIL = nconf.get('EMAILS:COMMUNITY_MANAGER_EMAIL');
 const FLAG_REPORT_EMAILS = nconf.get('FLAG_REPORT_EMAIL').split(',').map((email) => {
@@ -37,7 +37,7 @@ export default class GroupChatReporter extends ChatReporter {
     });
     if (!group) throw new NotFound(this.res.t('groupNotFound'));
 
-    let message = find(group.chat, {id: this.req.params.chatId});
+    const message = await Chat.findOne({id: this.req.params.chatId}).exec();
     if (!message) throw new NotFound(this.res.t('messageGroupChatNotFound'));
     if (message.uuid === 'system') throw new BadRequest(this.res.t('messageCannotFlagSystemMessages', {communityManagerEmail: COMMUNITY_MANAGER_EMAIL}));
 
@@ -68,13 +68,12 @@ export default class GroupChatReporter extends ChatReporter {
   }
 
   async flagGroupMessage (group, message) {
-    let update = {$set: {}};
     // Log user ids that have flagged the message
     if (!message.flags) message.flags = {};
     // TODO fix error type
     if (message.flags[this.user._id] && !this.user.contributor.admin) throw new NotFound(this.res.t('messageGroupChatFlagAlreadyReported'));
     message.flags[this.user._id] = true;
-    update.$set[`chat.$.flags.${this.user._id}`] = true;
+    message.markModified('flags');
 
     // Log total number of flags (publicly viewable)
     if (!message.flagCount) message.flagCount = 0;
@@ -84,12 +83,8 @@ export default class GroupChatReporter extends ChatReporter {
     } else {
       message.flagCount++;
     }
-    update.$set['chat.$.flagCount'] = message.flagCount;
 
-    await Group.update(
+    await message.save();
-      {_id: group._id, 'chat.id': message.id},
-      update
-    ).exec();
   }
 
   async flag () {

website/server/models/chat.js

@@ -0,0 +1,26 @@
+import mongoose from 'mongoose';
+import baseModel from '../libs/baseModel';
+
+const schema = new mongoose.Schema({
+  timestamp: Date,
+  user: String,
+  text: String,
+  contributor: {type: mongoose.Schema.Types.Mixed},
+  backer: {type: mongoose.Schema.Types.Mixed},
+  uuid: String,
+  id: String,
+  groupId: {type: String, ref: 'Group'},
+  flags: {type: mongoose.Schema.Types.Mixed, default: {}},
+  flagCount: {type: Number, default: 0},
+  likes: {type: mongoose.Schema.Types.Mixed},
+  userStyles: {type: mongoose.Schema.Types.Mixed},
+  _meta: {type: mongoose.Schema.Types.Mixed},
+}, {
+  minimize: false, // Allow for empty flags to be saved
+});
+
+schema.plugin(baseModel, {
+  noSet: ['_id'],
+});
+
+export const model = mongoose.model('Chat', schema);

website/server/models/group.js

@@ -7,6 +7,7 @@ import {
 import shared from '../../common';
 import _  from 'lodash';
 import { model as Challenge} from './challenge';
+import { model as Chat } from './chat';
 import * as Tasks from './task';
 import validator from 'validator';
 import { removeFromArray } from '../libs/collectionManipulators';
@@ -30,6 +31,7 @@ import {
 } from './subscriptionPlan';
 import amazonPayments from '../libs/payments/amazon';
 import stripePayments from '../libs/payments/stripe';
+import { getGroupChat } from '../libs/chat/group-chat';
 import { model as UserNotification } from './userNotification';
 
 const questScrolls = shared.content.quests;
@@ -57,6 +59,9 @@ export const SPAM_MESSAGE_LIMIT = 2;
 export const SPAM_WINDOW_LENGTH = 60000; // 1 minute
 export const SPAM_MIN_EXEMPT_CONTRIB_LEVEL = 4;
 
+export const MAX_CHAT_COUNT = 200;
+export const MAX_SUBBED_GROUP_CHAT_COUNT = 400;
+
 export let schema = new Schema({
   name: {type: String, required: true},
   summary: {type: String, maxlength: MAX_SUMMARY_SIZE_FOR_GUILDS},
@@ -319,7 +324,13 @@ schema.statics.getGroups = async function getGroups (options = {}) {
 // unless the user is an admin or said chat is posted by that user
 // Not putting into toJSON because there we can't access user
 // It also removes the _meta field that can be stored inside a chat message
-schema.statics.toJSONCleanChat = function groupToJSONCleanChat (group, user) {
+schema.statics.toJSONCleanChat = async function groupToJSONCleanChat (group, user) {
+  // @TODO: Adding this here for support the old chat, but we should depreciate accessing chat like this
+  // Also only return chat if requested, eventually we don't want to return chat here
+  if (group && group.chat) {
+    await getGroupChat(group);
+  }
+
   let toJSON = group.toJSON();
 
   if (!user.contributor.admin) {
@@ -451,8 +462,10 @@ schema.methods.getMemberCount = async function getMemberCount () {
 };
 
 export function chatDefaults (msg, user) {
-  let message = {
+  const id = shared.uuid();
-    id: shared.uuid(),
+  const message = {
+    id,
+    _id: id,
     text: msg,
     timestamp: Number(new Date()),
     likes: {},
@@ -518,23 +531,25 @@ function setUserStyles (newMessage, user) {
   }
 
   newMessage.userStyles = userStyles;
+  newMessage.markModified('userStyles');
 }
 
 schema.methods.sendChat = function sendChat (message, user, metaData) {
   let newMessage = chatDefaults(message, user);
+  let newChatMessage = new Chat();
+  newChatMessage = Object.assign(newChatMessage, newMessage);
+  newChatMessage.groupId = this._id;
 
-  if (user) setUserStyles(newMessage, user);
+  if (user) setUserStyles(newChatMessage, user);
 
   // Optional data stored in the chat message but not returned
   // to the users that can be stored for debugging purposes
   if (metaData) {
-    newMessage._meta = metaData;
+    newChatMessage._meta = metaData;
   }
 
-  this.chat.unshift(newMessage);
+  // @TODO: Completely remove the code below after migration
-
+  // this.chat.unshift(newMessage);
-  const MAX_CHAT_COUNT = 200;
-  const MAX_SUBBED_GROUP_CHAT_COUNT = 400;
 
   let maxCount = MAX_CHAT_COUNT;
 
@@ -546,7 +561,7 @@ schema.methods.sendChat = function sendChat (message, user, metaData) {
 
   // do not send notifications for guilds with more than 5000 users and for the tavern
   if (NO_CHAT_NOTIFICATIONS.indexOf(this._id) !== -1 || this.memberCount > LARGE_GROUP_COUNT_MESSAGE_CUTOFF) {
-    return;
+    return newChatMessage;
   }
 
   // Kick off chat notifications in the background.
@@ -591,7 +606,7 @@ schema.methods.sendChat = function sendChat (message, user, metaData) {
     pusher.trigger(`presence-group-${this._id}`, 'new-chat', newMessage);
   }
 
-  return newMessage;
+  return newChatMessage;
 };
 
 schema.methods.startQuest = async function startQuest (user) {
@@ -710,9 +725,11 @@ schema.methods.startQuest = async function startQuest (user) {
         });
     });
   });
-  this.sendChat(`\`Your quest, ${quest.text('en')}, has started.\``, null, {
+  const newMessage = this.sendChat(`\`Your quest, ${quest.text('en')}, has started.\``, null, {
     participatingMembers: this.getParticipatingQuestMembers().join(', '),
   });
+
+  await newMessage.save();
 };
 
 schema.methods.sendGroupChatReceivedWebhooks = function sendGroupChatReceivedWebhooks (chat) {
@@ -905,19 +922,22 @@ schema.methods._processBossQuest = async function processBossQuest (options) {
   let updates = {
     $inc: {'stats.hp': down},
   };
+  const promises = [];
 
   group.quest.progress.hp -= progress.up;
   // TODO Create a party preferred language option so emits like this can be localized. Suggestion: Always display the English version too. Or, if English is not displayed to the players, at least include it in a new field in the chat object that's visible in the database - essential for admins when troubleshooting quests!
   let playerAttack = `${user.profile.name} attacks ${quest.boss.name('en')} for ${progress.up.toFixed(1)} damage.`;
   let bossAttack = CRON_SAFE_MODE || CRON_SEMI_SAFE_MODE ? `${quest.boss.name('en')} does not attack, because it respects the fact that there are some bugs\` \`post-maintenance and it doesn't want to hurt anyone unfairly. It will continue its rampage soon!` : `${quest.boss.name('en')} attacks party for ${Math.abs(down).toFixed(1)} damage.`;
   // TODO Consider putting the safe mode boss attack message in an ENV var
-  group.sendChat(`\`${playerAttack}\` \`${bossAttack}\``);
+  const groupMessage = group.sendChat(`\`${playerAttack}\` \`${bossAttack}\``);
+  promises.push(groupMessage.save());
 
   // If boss has Rage, increment Rage as well
   if (quest.boss.rage) {
     group.quest.progress.rage += Math.abs(down);
     if (group.quest.progress.rage >= quest.boss.rage.value) {
-      group.sendChat(quest.boss.rage.effect('en'));
+      const rageMessage = group.sendChat(quest.boss.rage.effect('en'));
+      promises.push(rageMessage.save());
       group.quest.progress.rage = 0;
 
       // TODO To make Rage effects more expandable, let's turn these into functions in quest.boss.rage
@@ -944,13 +964,15 @@ schema.methods._processBossQuest = async function processBossQuest (options) {
 
   // Boss slain, finish quest
   if (group.quest.progress.hp <= 0) {
-    group.sendChat(`\`You defeated ${quest.boss.name('en')}! Questing party members receive the rewards of victory.\``);
+    const questFinishChat = group.sendChat(`\`You defeated ${quest.boss.name('en')}! Questing party members receive the rewards of victory.\``);
+    promises.push(questFinishChat.save());
 
     // Participants: Grant rewards & achievements, finish quest
     await group.finishQuest(shared.content.quests[group.quest.key]);
   }
 
-  return await group.save();
+  promises.unshift(group.save());
+  return await Promise.all(promises);
 };
 
 schema.methods._processCollectionQuest = async function processCollectionQuest (options) {
@@ -995,18 +1017,24 @@ schema.methods._processCollectionQuest = async function processCollectionQuest (
   }, []);
 
   foundText = foundText.join(', ');
-  group.sendChat(`\`${user.profile.name} found ${foundText}.\``);
+  const foundChat = group.sendChat(`\`${user.profile.name} found ${foundText}.\``);
   group.markModified('quest.progress.collect');
 
   // Still needs completing
-  if (_.find(quest.collect, (v, k) => {
+  const needsCompleted = _.find(quest.collect, (v, k) => {
     return group.quest.progress.collect[k] < v.count;
-  })) return await group.save();
+  });
+
+  if (needsCompleted) {
+    return await Promise.all([group.save(), foundChat.save()]);
+  }
 
   await group.finishQuest(quest);
-  group.sendChat('`All items found! Party has received their rewards.`');
+  const allItemsFoundChat = group.sendChat('`All items found! Party has received their rewards.`');
 
-  return await group.save();
+  const promises = [group.save(), foundChat.save(), allItemsFoundChat.save()];
+
+  return await Promise.all(promises);
 };
 
 schema.statics.processQuestProgress = async function processQuestProgress (user, progress) {
@@ -1060,8 +1088,11 @@ schema.statics.tavernBoss = async function tavernBoss (user, progress) {
 
   let quest = shared.content.quests[tavern.quest.key];
 
+  const chatPromises = [];
+
   if (tavern.quest.progress.hp <= 0) {
-    tavern.sendChat(quest.completionChat('en'));
+    const completeChat = tavern.sendChat(quest.completionChat('en'));
+    chatPromises.push(completeChat.save());
     await tavern.finishQuest(quest);
     _.assign(tavernQuest, {extra: null});
     return tavern.save();
@@ -1089,10 +1120,12 @@ schema.statics.tavernBoss = async function tavernBoss (user, progress) {
       }
 
       if (!scene) {
-        tavern.sendChat(`\`${quest.boss.name('en')} tries to unleash ${quest.boss.rage.title('en')} but is too tired.\``);
+        const tiredChat = tavern.sendChat(`\`${quest.boss.name('en')} tries to unleash ${quest.boss.rage.title('en')} but is too tired.\``);
+        chatPromises.push(tiredChat.save());
         tavern.quest.progress.rage = 0; // quest.boss.rage.value;
       } else {
-        tavern.sendChat(quest.boss.rage[scene]('en'));
+        const rageChat = tavern.sendChat(quest.boss.rage[scene]('en'));
+        chatPromises.push(rageChat.save());
         tavern.quest.extra.worldDmg[scene] = true;
         tavern.markModified('quest.extra.worldDmg');
         tavern.quest.progress.rage = 0;
@@ -1103,7 +1136,8 @@ schema.statics.tavernBoss = async function tavernBoss (user, progress) {
     }
 
     if (quest.boss.desperation && tavern.quest.progress.hp < quest.boss.desperation.threshold && !tavern.quest.extra.desperate) {
-      tavern.sendChat(quest.boss.desperation.text('en'));
+      const progressChat = tavern.sendChat(quest.boss.desperation.text('en'));
+      chatPromises.push(progressChat.save());
       tavern.quest.extra.desperate = true;
       tavern.quest.extra.def = quest.boss.desperation.def;
       tavern.quest.extra.str = quest.boss.desperation.str;
@@ -1111,7 +1145,9 @@ schema.statics.tavernBoss = async function tavernBoss (user, progress) {
     }
 
     _.assign(tavernQuest, tavern.quest.toObject());
-    return tavern.save();
+
+    chatPromises.unshift(tavern.save());
+    return Promise.all(chatPromises);
   }
 };