Misc Webhooks Fixes (#12038)

* fix(webhooks): don t parse response as json

* upgrade got to version 10

* remove old header

* fix tests

* fix email auth

* add migration

* update email error

* split migration in two

migrations/archive/2020/20200402_webhooks_add_protocol.js

@@ -0,0 +1,69 @@
+/* eslint-disable no-console */
+const MIGRATION_NAME = '20200402_webhooks_add_protocol';
+import { model as User } from '../../../website/server/models/user';
+
+const progressCount = 1000;
+let count = 0;
+
+async function updateUser (user) {
+  count++;
+
+  const set = {
+    migration: MIGRATION_NAME,
+  };
+
+  if (user && user.webhooks && user.webhooks.length > 0) {
+    user.webhooks.forEach(webhook => {
+      // Make sure the protocol is set and valid
+      if (webhook.url.startsWith('ftp')) {
+        webhook.url = webhook.url.replace('ftp', 'https');
+      }
+
+      if (!webhook.url.startsWith('http://') && !webhook.url.startsWith('https://')) {
+        // the default in got 9 was https
+        // see https://github.com/sindresorhus/got/commit/92bc8082137d7d085750359bbd76c801e213d7d2#diff-0730bb7c2e8f9ea2438b52e419dd86c9L111
+        webhook.url = `https://${webhook.url}`;
+      }
+    });
+
+    set.webhooks = user.webhooks;
+  }
+
+  if (count % progressCount === 0) console.warn(`${count} ${user._id}`);
+
+  return await User.update({ _id: user._id }, { $set: set }).exec();
+}
+
+module.exports = async function processUsers () {
+  let query = {
+    migration: { $ne: MIGRATION_NAME },
+    webhooks: { $exists: true, $not: { $size: 0 } },
+  };
+
+  const fields = {
+    _id: 1,
+    webhooks: 1,
+  };
+
+  while (true) { // eslint-disable-line no-constant-condition
+    const users = await User // eslint-disable-line no-await-in-loop
+      .find(query)
+      .limit(250)
+      .sort({_id: 1})
+      .select(fields)
+      .lean()
+      .exec();
+
+    if (users.length === 0) {
+      console.warn('All appropriate users found and modified.');
+      console.warn(`\n${count} users processed\n`);
+      break;
+    } else {
+      query._id = {
+        $gt: users[users.length - 1]._id,
+      };
+    }
+
+    await Promise.all(users.map(updateUser)); // eslint-disable-line no-await-in-loop
+  }
+};

migrations/archive/2020/20200402_webhooks_reenable.js

@@ -0,0 +1,63 @@
+/* eslint-disable no-console */
+const MIGRATION_NAME = '20200402_webhooks_reenable';
+import { model as User } from '../../../website/server/models/user';
+
+const progressCount = 1000;
+let count = 0;
+
+async function updateUser (user) {
+  count++;
+
+  const set = {
+    migration: MIGRATION_NAME,
+  };
+
+  if (user && user.webhooks && user.webhooks.length > 0) {
+    user.webhooks.forEach(webhook => {
+      // Re-enable webhooks disabled because of too many failures
+      if (webhook.enabled === false && webhook.lastFailureAt === null) {
+        webhook.enabled = true;
+      }
+    });
+
+    set.webhooks = user.webhooks;
+  }
+
+  if (count % progressCount === 0) console.warn(`${count} ${user._id}`);
+
+  return await User.update({ _id: user._id }, { $set: set }).exec();
+}
+
+module.exports = async function processUsers () {
+  let query = {
+    migration: { $ne: MIGRATION_NAME },
+    webhooks: { $exists: true, $not: { $size: 0 } },
+  };
+
+  const fields = {
+    _id: 1,
+    webhooks: 1,
+  };
+
+  while (true) { // eslint-disable-line no-constant-condition
+    const users = await User // eslint-disable-line no-await-in-loop
+      .find(query)
+      .limit(250)
+      .sort({_id: 1})
+      .select(fields)
+      .lean()
+      .exec();
+
+    if (users.length === 0) {
+      console.warn('All appropriate users found and modified.');
+      console.warn(`\n${count} users processed\n`);
+      break;
+    } else {
+      query._id = {
+        $gt: users[users.length - 1]._id,
+      };
+    }
+
+    await Promise.all(users.map(updateUser)); // eslint-disable-line no-await-in-loop
+  }
+};

package-lock.json

@@ -1360,9 +1360,9 @@
       }
     },
     "@sindresorhus/is": {
-      "version": "0.14.0",
+      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz",
+      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.0.tgz",
-      "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ=="
+      "integrity": "sha512-lXKXfypKo644k4Da4yXkPCrwcvn6SlUW2X2zFbuflKHNjf0w9htru01bo26uMhleMXsDmnZ12eJLdrAZa9MANg=="
     },
     "@sinonjs/commons": {
       "version": "1.6.0",
@@ -1479,11 +1479,22 @@
       }
     },
     "@szmarczak/http-timer": {
-      "version": "1.1.2",
+      "version": "4.0.5",
-      "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz",
+      "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.5.tgz",
-      "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==",
+      "integrity": "sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ==",
       "requires": {
-        "defer-to-connect": "^1.0.1"
+        "defer-to-connect": "^2.0.0"
+      }
+    },
+    "@types/cacheable-request": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.1.tgz",
+      "integrity": "sha512-ykFq2zmBGOCbpIXtoVbz4SKY5QriWPh3AjyU4G74RYbtt5yOc5OfaY75ftjg7mikMOla1CTGpX3lLbuJh8DTrQ==",
+      "requires": {
+        "@types/http-cache-semantics": "*",
+        "@types/keyv": "*",
+        "@types/node": "*",
+        "@types/responselike": "*"
       }
     },
     "@types/color-name": {
@@ -1507,6 +1518,19 @@
         "@types/node": "*"
       }
     },
+    "@types/http-cache-semantics": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.0.tgz",
+      "integrity": "sha512-c3Xy026kOF7QOTn00hbIllV1dLR9hG9NkSrLQgCVs8NF6sBU+VGWjD3wLPhmh1TYAc7ugCFsvHYMN4VcBN1U1A=="
+    },
+    "@types/keyv": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.1.tgz",
+      "integrity": "sha512-MPtoySlAZQ37VoLaPcTHCu1RWJ4llDkULYZIzOYxlhxBqYPB0RsRlmMU0R6tahtFe27mIdkHV+551ZWV4PLmVw==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/minimatch": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
@@ -1523,6 +1547,14 @@
       "integrity": "sha512-ce5d3q03Ex0sy4R14722Rmt6MT07Ua+k4FwDfdcToYJcMKNtRVQvJ6JCAPdAmAnbRb6CsX6aYb9m96NGod9uTw==",
       "optional": true
     },
+    "@types/responselike": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz",
+      "integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "abbrev": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@@ -3134,18 +3166,27 @@
         "unset-value": "^1.0.0"
       }
     },
+    "cacheable-lookup": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-2.0.1.tgz",
+      "integrity": "sha512-EMMbsiOTcdngM/K6gV/OxF2x0t07+vMOWxZNSCRQMjO2MY2nhZQ6OYhOOpyQrbhqsgtvKGI7hcq6xjnA92USjg==",
+      "requires": {
+        "@types/keyv": "^3.1.1",
+        "keyv": "^4.0.0"
+      }
+    },
     "cacheable-request": {
-      "version": "6.1.0",
+      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.1.tgz",
-      "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==",
+      "integrity": "sha512-lt0mJ6YAnsrBErpTMWeu5kl/tg9xMAWjavYTN6VQXM1A/teBITuNcccXsCxF0tDQQJf9DfAaX5O4e0zp0KlfZw==",
       "requires": {
         "clone-response": "^1.0.2",
         "get-stream": "^5.1.0",
         "http-cache-semantics": "^4.0.0",
-        "keyv": "^3.0.0",
+        "keyv": "^4.0.0",
         "lowercase-keys": "^2.0.0",
         "normalize-url": "^4.1.0",
-        "responselike": "^1.0.2"
+        "responselike": "^2.0.0"
       },
       "dependencies": {
         "get-stream": {
@@ -3160,6 +3201,14 @@
           "version": "2.0.0",
           "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
           "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="
+        },
+        "responselike": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
+          "integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==",
+          "requires": {
+            "lowercase-keys": "^2.0.0"
+          }
         }
       }
     },
@@ -4183,6 +4232,7 @@
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz",
       "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=",
+      "optional": true,
       "requires": {
         "mimic-response": "^1.0.0"
       }
@@ -4360,9 +4410,9 @@
       "integrity": "sha1-vLgrqnKtebQmp2cy8aga1t8m1oQ="
     },
     "defer-to-connect": {
-      "version": "1.0.2",
+      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.0.tgz",
-      "integrity": "sha512-k09hcQcTDY+cwgiwa6PYKLm3jlagNzQ+RSvhjzESOGOx+MNOuXkxTfEvPrO1IOQ81tArCFYQgi631clB70RpQw=="
+      "integrity": "sha512-bYL2d05vOSf1JEZNx5vSAtPuBMkX8K9EUutg7zlKvTqKXHt7RhWJFbmd7qakVuf13i+IkGmp6FwSsONOf6VYIg=="
     },
     "define-properties": {
       "version": "1.1.3",
@@ -6754,6 +6804,7 @@
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz",
       "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==",
+      "optional": true,
       "requires": {
         "pump": "^3.0.0"
       }
@@ -7024,21 +7075,82 @@
       }
     },
     "got": {
-      "version": "9.6.0",
+      "version": "10.7.0",
-      "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz",
+      "resolved": "https://registry.npmjs.org/got/-/got-10.7.0.tgz",
-      "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==",
+      "integrity": "sha512-aWTDeNw9g+XqEZNcTjMMZSy7B7yE9toWOFYip7ofFTLleJhvZwUxxTxkTpKvF+p1SAA4VHmuEy7PiHTHyq8tJg==",
       "requires": {
-        "@sindresorhus/is": "^0.14.0",
+        "@sindresorhus/is": "^2.0.0",
-        "@szmarczak/http-timer": "^1.1.2",
+        "@szmarczak/http-timer": "^4.0.0",
-        "cacheable-request": "^6.0.0",
+        "@types/cacheable-request": "^6.0.1",
-        "decompress-response": "^3.3.0",
+        "cacheable-lookup": "^2.0.0",
+        "cacheable-request": "^7.0.1",
+        "decompress-response": "^5.0.0",
         "duplexer3": "^0.1.4",
-        "get-stream": "^4.1.0",
+        "get-stream": "^5.0.0",
-        "lowercase-keys": "^1.0.1",
+        "lowercase-keys": "^2.0.0",
-        "mimic-response": "^1.0.1",
+        "mimic-response": "^2.1.0",
-        "p-cancelable": "^1.0.0",
+        "p-cancelable": "^2.0.0",
-        "to-readable-stream": "^1.0.0",
+        "p-event": "^4.0.0",
-        "url-parse-lax": "^3.0.0"
+        "responselike": "^2.0.0",
+        "to-readable-stream": "^2.0.0",
+        "type-fest": "^0.10.0"
+      },
+      "dependencies": {
+        "decompress-response": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-5.0.0.tgz",
+          "integrity": "sha512-TLZWWybuxWgoW7Lykv+gq9xvzOsUjQ9tF09Tj6NSTYGMTCHNXzrPnD6Hi+TgZq19PyTAGH4Ll/NIM/eTGglnMw==",
+          "requires": {
+            "mimic-response": "^2.0.0"
+          }
+        },
+        "get-stream": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
+          "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==",
+          "requires": {
+            "pump": "^3.0.0"
+          }
+        },
+        "lowercase-keys": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
+          "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="
+        },
+        "mimic-response": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz",
+          "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA=="
+        },
+        "p-event": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/p-event/-/p-event-4.1.0.tgz",
+          "integrity": "sha512-4vAd06GCsgflX4wHN1JqrMzBh/8QZ4j+rzp0cd2scXRwuBEv+QR3wrVA5aLhWDLw4y2WgDKvzWF3CCLmVM1UgA==",
+          "requires": {
+            "p-timeout": "^2.0.1"
+          }
+        },
+        "p-timeout": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-2.0.1.tgz",
+          "integrity": "sha512-88em58dDVB/KzPEx1X0N3LwFfYZPyDc4B6eF38M1rk9VTZMbxXXgjugz8mmwpS9Ox4BDZ+t6t3QP5+/gazweIA==",
+          "requires": {
+            "p-finally": "^1.0.0"
+          }
+        },
+        "responselike": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
+          "integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==",
+          "requires": {
+            "lowercase-keys": "^2.0.0"
+          }
+        },
+        "type-fest": {
+          "version": "0.10.0",
+          "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.10.0.tgz",
+          "integrity": "sha512-EUV9jo4sffrwlg8s0zDhP0T2WD3pru5Xi0+HTE3zTUmBaZNhfkite9PdSJwdXLwPVW0jnAHT56pZHIOYckPEiw=="
+        }
       }
     },
     "graceful-fs": {
@@ -7575,9 +7687,9 @@
       }
     },
     "http-cache-semantics": {
-      "version": "4.0.3",
+      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
-      "integrity": "sha512-TcIMG3qeVLgDr1TEd2XvHaTnMPwYQUQMIBLy+5pLSDKYFc7UIqj39w8EGzZkaxoLv/l2K8HaI0t5AVA+YYgUew=="
+      "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ=="
     },
     "http-errors": {
       "version": "1.7.2",
@@ -8681,7 +8793,8 @@
     "json-buffer": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz",
-      "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg="
+      "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=",
+      "optional": true
     },
     "json-content-demux": {
       "version": "0.1.3",
@@ -8825,11 +8938,18 @@
       }
     },
     "keyv": {
-      "version": "3.1.0",
+      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.0.0.tgz",
-      "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==",
+      "integrity": "sha512-U7ioE8AimvRVLfw4LffyOIRhL2xVgmE8T22L6i0BucSnBUyv4w+I7VN/zVZwRKHOI6ZRUcdMdWHQ8KSUvGpEog==",
       "requires": {
-        "json-buffer": "3.0.0"
+        "json-buffer": "3.0.1"
+      },
+      "dependencies": {
+        "json-buffer": {
+          "version": "3.0.1",
+          "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
+          "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
+        }
       }
     },
     "kind-of": {
@@ -10707,9 +10827,9 @@
       }
     },
     "p-cancelable": {
-      "version": "1.1.0",
+      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.0.0.tgz",
-      "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw=="
+      "integrity": "sha512-wvPXDmbMmu2ksjkB4Z3nZWTSkJEb9lqVdMaCKpZUGJG9TMiNp9XcbG3fn9fPKjem04fJMJnXoyFPk2FmgiaiNg=="
     },
     "p-defer": {
       "version": "1.0.0",
@@ -11254,7 +11374,8 @@
     "prepend-http": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz",
-      "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc="
+      "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=",
+      "optional": true
     },
     "pretty-bytes": {
       "version": "5.3.0",
@@ -11867,6 +11988,7 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz",
       "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=",
+      "optional": true,
       "requires": {
         "lowercase-keys": "^1.0.0"
       }
@@ -13177,9 +13299,9 @@
       }
     },
     "to-readable-stream": {
-      "version": "1.0.0",
+      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-2.1.0.tgz",
-      "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q=="
+      "integrity": "sha512-o3Qa6DGg1CEXshSdvWNX2sN4QHqg03SPq7U6jPXRahlQdl5dK8oXjkU/2/sGrnOZKeGV1zLSO8qPwyKklPPE7w=="
     },
     "to-regex": {
       "version": "3.0.2",
@@ -13783,6 +13905,7 @@
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz",
       "integrity": "sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=",
+      "optional": true,
       "requires": {
         "prepend-http": "^2.0.0"
       }

package.json

@@ -30,7 +30,7 @@
     "express-basic-auth": "^1.1.5",
     "express-validator": "^5.2.0",
     "glob": "^7.1.6",
-    "got": "^9.0.0",
+    "got": "^10.7.0",
     "gulp": "^4.0.0",
     "gulp-babel": "^8.0.0",
     "gulp-imagemin": "^6.2.0",

test/api/unit/libs/email.test.js

@@ -121,8 +121,7 @@ describe('emails', () => {
 
       sendTxnEmail(mailingInfo, emailType);
       expect(got.post).to.be.calledWith('undefined/job', sinon.match({
-        json: true,
+        json: {
-        body: {
           data: {
             emailType: sinon.match.same(emailType),
             to: sinon.match(value => Array.isArray(value) && value[0].name === mailingInfo.name, 'matches mailing info array'),
@@ -154,8 +153,7 @@ describe('emails', () => {
 
       sendTxnEmail(mailingInfo, emailType);
       expect(got.post).to.be.calledWith('undefined/job', sinon.match({
-        json: true,
+        json: {
-        body: {
           data: {
             emailType: sinon.match.same(emailType),
             to: sinon.match(val => val[0]._id === mailingInfo._id),
@@ -177,8 +175,7 @@ describe('emails', () => {
 
       sendTxnEmail(mailingInfo, emailType, variables);
       expect(got.post).to.be.calledWith('undefined/job', sinon.match({
-        json: true,
+        json: {
-        body: {
           data: {
             variables: sinon.match(value => value[0].name === 'BASE_URL', 'matches variables'),
             personalVariables: sinon.match(value => value[0].rcpt === mailingInfo.email

test/api/unit/libs/webhooks.test.js

@@ -101,8 +101,7 @@ describe('webhooks', () => {
       expect(WebhookSender.defaultTransformData).to.be.calledOnce;
       expect(got.post).to.be.calledOnce;
       expect(got.post).to.be.calledWithMatch('http://custom-url.com', {
-        json: true,
+        json: body,
-        body,
       });
     });
 
@@ -122,7 +121,7 @@ describe('webhooks', () => {
       expect(sendWebhook.attachDefaultData).to.be.calledOnce;
       expect(got.post).to.be.calledOnce;
       expect(got.post).to.be.calledWithMatch('http://custom-url.com', {
-        json: true,
+        json: body,
       });
 
       expect(body).to.eql({
@@ -153,8 +152,7 @@ describe('webhooks', () => {
       expect(WebhookSender.defaultTransformData).to.not.be.called;
       expect(got.post).to.be.calledOnce;
       expect(got.post).to.be.calledWithMatch('http://custom-url.com', {
-        json: true,
+        json: {
-        body: {
           foo: 'bar',
           baz: 'biz',
         },
@@ -271,8 +269,7 @@ describe('webhooks', () => {
 
       expect(got.post).to.be.calledOnce;
       expect(got.post).to.be.calledWithMatch('http://custom-url.com', {
-        body,
+        json: body,
-        json: true,
       });
     });
 
@@ -292,8 +289,7 @@ describe('webhooks', () => {
 
       expect(got.post).to.be.calledOnce;
       expect(got.post).to.be.calledWithMatch('http://custom-url.com', {
-        body,
+        json: body,
-        json: true,
       });
     });
 
@@ -316,12 +312,10 @@ describe('webhooks', () => {
 
       expect(got.post).to.be.calledTwice;
       expect(got.post).to.be.calledWithMatch('http://custom-url.com', {
-        body,
+        json: body,
-        json: true,
       });
       expect(got.post).to.be.calledWithMatch('http://other-url.com', {
-        body,
+        json: body,
-        json: true,
       });
     });
 
@@ -351,8 +345,7 @@ describe('webhooks', () => {
 
         expect(got.post).to.be.calledOnce;
         expect(got.post).to.be.calledWithMatch('http://custom-url.com', {
-          json: true,
+          json: body,
-          body,
         });
 
         await sleep(0.1);
@@ -368,8 +361,7 @@ describe('webhooks', () => {
 
         expect(got.post).to.be.calledOnce;
         expect(got.post).to.be.calledWithMatch('http://custom-url.com', {
-          json: true,
+          json: body,
-          body,
         });
 
         await sleep(0.1);
@@ -459,8 +451,7 @@ describe('webhooks', () => {
 
       expect(got.post).to.be.calledOnce;
       expect(got.post).to.be.calledWithMatch(webhooks[0].url, {
-        json: true,
+        json: {
-        body: {
           type: 'scored',
           webhookType: 'taskActivity',
           user: {
@@ -497,8 +488,7 @@ describe('webhooks', () => {
 
       expect(got.post).to.be.calledOnce;
       expect(got.post).to.be.calledWithMatch('http://global-activity.com', {
-        json: true,
+        json: {
-        body: {
           type: 'scored',
           webhookType: 'taskActivity',
           user: {
@@ -551,8 +541,7 @@ describe('webhooks', () => {
 
         expect(got.post).to.be.calledOnce;
         expect(got.post).to.be.calledWithMatch(webhooks[0].url, {
-          json: true,
+          json: {
-          body: {
             type,
             webhookType: 'taskActivity',
             user: {
@@ -592,8 +581,7 @@ describe('webhooks', () => {
 
         expect(got.post).to.be.calledOnce;
         expect(got.post).to.be.calledWithMatch(webhooks[0].url, {
-          json: true,
+          json: {
-          body: {
             webhookType: 'taskActivity',
             user: {
               _id: user._id,
@@ -633,8 +621,7 @@ describe('webhooks', () => {
 
         expect(got.post).to.be.calledOnce;
         expect(got.post).to.be.calledWithMatch(webhooks[2].url, {
-          json: true,
+          json: {
-          body: {
             type,
             webhookType: 'userActivity',
             user: {
@@ -680,8 +667,7 @@ describe('webhooks', () => {
 
         expect(got.post).to.be.calledOnce;
         expect(got.post).to.be.calledWithMatch(webhooks[1].url, {
-          json: true,
+          json: {
-          body: {
             type,
             webhookType: 'questActivity',
             user: {
@@ -727,8 +713,7 @@ describe('webhooks', () => {
 
       expect(got.post).to.be.calledOnce;
       expect(got.post).to.be.calledWithMatch(webhooks[webhooks.length - 1].url, {
-        json: true,
+        json: {
-        body: {
           webhookType: 'groupChatReceived',
           user: {
             _id: user._id,

website/server/libs/email.js

@@ -133,9 +133,9 @@ export async function sendTxn (mailingInfoArray, emailType, variables, personalV
     return got.post(`${EMAIL_SERVER.url}/job`, {
       retry: 5, // retry the http request to the email server 5 times
       timeout: 60000, // wait up to 60s before timing out
-      auth: `${EMAIL_SERVER.auth.user}:${EMAIL_SERVER.auth.password}`,
+      username: EMAIL_SERVER.auth.user,
-      json: true,
+      password: EMAIL_SERVER.auth.password,
-      body: {
+      json: {
         type: 'email',
         data: {
           emailType,
@@ -149,7 +149,7 @@ export async function sendTxn (mailingInfoArray, emailType, variables, personalV
           backoff: { delay: 10 * 60 * 1000, type: 'fixed' },
         },
       },
-    }).catch(err => logger.error(err));
+    }).json().catch(err => logger.error(err, 'Error while sending an email.'));
   }
 
   return null;

website/server/libs/webhook.js

@@ -13,10 +13,10 @@ function sendWebhook (webhook, body, user) {
   const { url, lastFailureAt } = webhook;
 
   got.post(url, {
-    body,
+    json: body,
-    json: true,
     timeout: 30000, // wait up to 30s before timing out
     retry: 3, // retry the request up to 3 times
+  // Not calling .json() to parse the response because we simply ignore it
   }).catch(webhookErr => {
     // Log the error
     logger.error(webhookErr, 'Error while sending a webhook request.');

website/server/models/webhook.js

@@ -58,6 +58,8 @@ export const schema = new Schema({
     required: true,
     validate: [v => validator.isURL(v, {
       require_tld: !!IS_PRODUCTION, // eslint-disable-line camelcase
+      require_protocol: true, // TODO migrate existing ones
+      protocols: ['http', 'https'],
     }), shared.i18n.t('invalidUrl')],
   },
   enabled: { $type: Boolean, required: true, default: true },