Organize directories

apps/metamigo-worker/tasks/notify-webhook.ts

@@ -0,0 +1,57 @@
+import Wreck from "@hapi/wreck";
+import * as R from "remeda";
+import { withDb, AppDatabase } from "../db";
+import logger from "../logger";
+
+export interface WebhookOptions {
+  webhookId: string;
+  payload: any;
+}
+
+const notifyWebhooksTask = async (options: WebhookOptions): Promise<void> =>
+  withDb(async (db: AppDatabase) => {
+    const { webhookId, payload } = options;
+
+    const webhook = await db.webhooks.findById({ id: webhookId });
+    if (!webhook) {
+      logger.debug(
+        { webhookId },
+        "notify-webhook: no webhook registered with id"
+      );
+      return;
+    }
+
+    const { endpointUrl, httpMethod, headers } = webhook;
+    const headersFormatted = R.reduce(
+      headers || [],
+      (acc: any, h: any) => {
+        acc[h.header] = h.value;
+        return acc;
+      },
+      {}
+    );
+
+    const wreck = Wreck.defaults({
+      json: true,
+      headers: headersFormatted,
+    });
+
+    // http errors will bubble up and cause the job to fail and be retried
+    try {
+      logger.debug(
+        { webhookId, endpointUrl, httpMethod },
+        "notify-webhook: notifying"
+      );
+      await (httpMethod === "post"
+        ? wreck.post(endpointUrl, { payload })
+        : wreck.put(endpointUrl, { payload }));
+    } catch (error: any) {
+      logger.error(
+        { webhookId, error: error.output },
+        "notify-webhook failed with this error"
+      );
+      throw new Error(`webhook failed webhookId=${webhookId}`);
+    }
+  });
+
+export default notifyWebhooksTask;

apps/metamigo-worker/tasks/signal-message.ts

@@ -0,0 +1,76 @@
+/* eslint-disable camelcase */
+import { withDb, AppDatabase } from "../db";
+import workerUtils from "../utils";
+
+interface WebhookPayload {
+  to: string;
+  from: string;
+  message_id: string;
+  sent_at: string;
+  message: string;
+  attachment: string;
+  filename: string;
+  mime_type: string;
+}
+
+interface SignalMessageTaskOptions {
+  id: string;
+  source: string;
+  timestamp: string;
+  message: string;
+  attachments: unknown[];
+  signalBotId: string;
+}
+
+const formatPayload = (
+  messageInfo: SignalMessageTaskOptions
+): WebhookPayload => {
+  const { id, source, message, timestamp } = messageInfo;
+
+  return {
+    to: "16464229653",
+    from: source,
+    message_id: id,
+    sent_at: timestamp,
+    message,
+    attachment: "",
+    filename: "test.png",
+    mime_type: "image/png",
+  };
+};
+
+const notifyWebhooks = async (
+  db: AppDatabase,
+  messageInfo: SignalMessageTaskOptions
+) => {
+  const { id: messageID, signalBotId } = messageInfo;
+  const webhooks = await db.webhooks.findAllByBackendId("signal", signalBotId);
+  if (webhooks && webhooks.length === 0) return;
+
+  webhooks.forEach(({ id }) => {
+    const payload = formatPayload(messageInfo);
+    console.log({ payload });
+    workerUtils.addJob(
+      "notify-webhook",
+      {
+        payload,
+        webhookId: id,
+      },
+      {
+        // this de-deduplicates the job
+        jobKey: `webhook-${id}-message-${messageID}`,
+      }
+    );
+  });
+};
+
+const signalMessageTask = async (
+  options: SignalMessageTaskOptions
+): Promise<void> => {
+  console.log(options);
+  withDb(async (db: AppDatabase) => {
+    await notifyWebhooks(db, options);
+  });
+};
+
+export default signalMessageTask;

apps/metamigo-worker/tasks/signald-message.ts

@@ -0,0 +1,87 @@
+/* eslint-disable camelcase */
+import logger from "../logger";
+import { IncomingMessagev1 } from "@digiresilience/node-signald/dist/generated";
+import { withDb, AppDatabase } from "../db";
+import workerUtils from "../utils";
+
+interface WebhookPayload {
+  to: string;
+  from: string;
+  message_id: string;
+  sent_at: string;
+  message: string;
+  attachment: string | null;
+  filename: string | null;
+  mime_type: string | null;
+}
+
+interface SignaldMessageTaskOptions {
+  message: IncomingMessagev1;
+  botId: string;
+  botPhoneNumber: string;
+}
+
+const formatPayload = (opts: SignaldMessageTaskOptions): WebhookPayload => {
+  const { botId, botPhoneNumber, message } = opts;
+  const { source, timestamp, data_message: dataMessage } = message;
+
+  const { number }: any = source;
+
+  const { body, attachments }: any = dataMessage;
+
+  return {
+    to: botPhoneNumber,
+    from: number,
+    message_id: `${botId}-${timestamp}`,
+    sent_at: `${timestamp}`,
+    message: body,
+    attachment: null,
+    filename: null,
+    mime_type: null,
+  };
+};
+
+const notifyWebhooks = async (
+  db: AppDatabase,
+  messageInfo: SignaldMessageTaskOptions
+) => {
+  const {
+    botId,
+    message: { timestamp },
+  } = messageInfo;
+  const webhooks = await db.webhooks.findAllByBackendId("signal", botId);
+  if (webhooks && webhooks.length === 0) {
+    logger.debug({ botId }, "no webhooks registered for signal bot");
+    return;
+  }
+
+  webhooks.forEach(({ id }) => {
+    const payload = formatPayload(messageInfo);
+    logger.debug(
+      { payload },
+      "formatted signal bot payload for notify-webhook"
+    );
+    workerUtils.addJob(
+      "notify-webhook",
+      {
+        payload,
+        webhookId: id,
+      },
+      {
+        // this de-deduplicates the job
+        jobKey: `webhook-${id}-message-${botId}-${timestamp}`,
+      }
+    );
+  });
+};
+
+const signaldMessageTask = async (
+  options: SignaldMessageTaskOptions
+): Promise<void> => {
+  console.log(options);
+  withDb(async (db: AppDatabase) => {
+    await notifyWebhooks(db, options);
+  });
+};
+
+export default signaldMessageTask;

apps/metamigo-worker/tasks/twilio-recording.ts

@@ -0,0 +1,101 @@
+import Wreck from "@hapi/wreck";
+import { withDb, AppDatabase } from "../db";
+import { twilioClientFor } from "../common";
+import { CallInstance } from "twilio/lib/rest/api/v2010/account/call";
+import workerUtils from "../utils";
+
+interface WebhookPayload {
+  startTime: string;
+  endTime: string;
+  to: string;
+  from: string;
+  duration: string;
+  callSid: string;
+  recording: string;
+  mimeType: string;
+}
+
+const getTwilioRecording = async (url: string) => {
+  try {
+    const { payload } = await Wreck.get(url);
+    return { recording: payload as Buffer };
+  } catch (error: any) {
+    console.error(error.output);
+    return { error: error.output };
+  }
+};
+
+const formatPayload = (
+  call: CallInstance,
+  recording: Buffer
+): WebhookPayload => {
+  return {
+    startTime: call.startTime.toISOString(),
+    endTime: call.endTime.toISOString(),
+    to: call.toFormatted,
+    from: call.fromFormatted,
+    duration: call.duration,
+    callSid: call.sid,
+    recording: recording.toString("base64"),
+    mimeType: "audio/mpeg",
+  };
+};
+
+const notifyWebhooks = async (
+  db: AppDatabase,
+  voiceLineId: string,
+  call: CallInstance,
+  recording: Buffer
+) => {
+  const webhooks = await db.webhooks.findAllByBackendId("voice", voiceLineId);
+  if (webhooks && webhooks.length === 0) return;
+
+  webhooks.forEach(({ id }) => {
+    const payload = formatPayload(call, recording);
+    workerUtils.addJob(
+      "notify-webhook",
+      {
+        payload,
+        webhookId: id,
+      },
+      {
+        // this de-depuplicates the job
+        jobKey: `webhook-${id}-call-${call.sid}`,
+      }
+    );
+  });
+};
+
+interface TwilioRecordingTaskOptions {
+  accountSid: string;
+  callSid: string;
+  recordingSid: string;
+  voiceLineId: string;
+}
+
+const twilioRecordingTask = async (
+  options: TwilioRecordingTaskOptions
+): Promise<void> =>
+  withDb(async (db: AppDatabase) => {
+    const { voiceLineId, accountSid, callSid, recordingSid } = options;
+
+    const voiceLine = await db.voiceLines.findById({ id: voiceLineId });
+    if (!voiceLine) return;
+
+    const provider = await db.voiceProviders.findByTwilioAccountSid(accountSid);
+    if (!provider) return;
+
+    const client = twilioClientFor(provider);
+    const meta = await client.recordings(recordingSid).fetch();
+
+    const mp3Url = "https://api.twilio.com/" + meta.uri.slice(0, -4) + "mp3";
+    const { recording, error } = await getTwilioRecording(mp3Url);
+    if (error) {
+      throw new Error(`failed to get recording for call ${callSid}`);
+    }
+
+    const call = await client.calls(callSid).fetch();
+    await notifyWebhooks(db, voiceLineId, call, recording!);
+  });
+
+export default twilioRecordingTask;

apps/metamigo-worker/tasks/voice-line-audio-update.ts

@@ -0,0 +1,48 @@
+import { createHash } from "crypto";
+import { withDb, AppDatabase } from "../db";
+import { convert } from "../lib/media-convert";
+
+interface VoiceLineAudioUpdateTaskOptions {
+  voiceLineId: string;
+}
+
+const sha1sum = (v: any) => {
+  const shasum = createHash("sha1");
+  shasum.update(v);
+  return shasum.digest("hex");
+};
+
+const voiceLineAudioUpdateTask = async (
+  payload: VoiceLineAudioUpdateTaskOptions
+): Promise<void> =>
+  withDb(async (db: AppDatabase) => {
+    const { voiceLineId } = payload;
+    const voiceLine = await db.voiceLines.findById({ id: voiceLineId });
+    if (!voiceLine) return;
+    if (!voiceLine?.promptAudio?.["audio/webm"]) return;
+
+    const webm = Buffer.from(voiceLine.promptAudio["audio/webm"], "base64");
+    const webmSha1 = sha1sum(webm);
+
+    if (
+      voiceLine.promptAudio.checksum &&
+      voiceLine.promptAudio.checksum === webmSha1
+    ) {
+      // already converted
+      return;
+    }
+
+    const mp3 = await convert(webm);
+    await db.voiceLines.updateById(
+      { id: voiceLine.id },
+      {
+        promptAudio: {
+          ...voiceLine.promptAudio,
+          "audio/mpeg": mp3.toString("base64"),
+          checksum: webmSha1,
+        },
+      }
+    );
+  });
+
+export default voiceLineAudioUpdateTask;

apps/metamigo-worker/tasks/voice-line-delete.ts

@@ -0,0 +1,41 @@
+import Twilio from "twilio";
+import config from "config";
+import { withDb, AppDatabase } from "../db";
+
+interface VoiceLineDeleteTaskOptions {
+  voiceLineId: string;
+  providerId: string;
+  providerLineSid: string;
+}
+
+const voiceLineDeleteTask = async (
+  payload: VoiceLineDeleteTaskOptions
+): Promise<void> =>
+  withDb(async (db: AppDatabase) => {
+    const { voiceLineId, providerId, providerLineSid } = payload;
+    const provider = await db.voiceProviders.findById({ id: providerId });
+    if (!provider) return;
+
+    const { accountSid, apiKeySid, apiKeySecret } = provider.credentials;
+    if (!accountSid || !apiKeySid || !apiKeySecret)
+      throw new Error(
+        `twilio provider ${provider.name} does not have credentials`
+      );
+
+    const client = Twilio(apiKeySid, apiKeySecret, {
+      accountSid,
+    });
+
+    const number = await client.incomingPhoneNumbers(providerLineSid).fetch();
+    if (
+      number &&
+      number.voiceUrl ===
+      `${config.frontend.url}/api/v1/voice/twilio/record/${voiceLineId}`
+    )
+      await client.incomingPhoneNumbers(providerLineSid).update({
+        voiceUrl: "",
+        voiceMethod: "POST",
+      });
+  });
+
+export default voiceLineDeleteTask;

apps/metamigo-worker/tasks/voice-line-provider-update.ts

@@ -0,0 +1,38 @@
+import Twilio from "twilio";
+import config from "config";
+import { withDb, AppDatabase } from "../db";
+
+interface VoiceLineUpdateTaskOptions {
+  voiceLineId: string;
+}
+
+const voiceLineUpdateTask = async (
+  payload: VoiceLineUpdateTaskOptions
+): Promise<void> =>
+  withDb(async (db: AppDatabase) => {
+    const { voiceLineId } = payload;
+    const voiceLine = await db.voiceLines.findById({ id: voiceLineId });
+    if (!voiceLine) return;
+
+    const provider = await db.voiceProviders.findById({
+      id: voiceLine.providerId,
+    });
+    if (!provider) return;
+
+    const { accountSid, apiKeySid, apiKeySecret } = provider.credentials;
+    if (!accountSid || !apiKeySid || !apiKeySecret)
+      throw new Error(
+        `twilio provider ${provider.name} does not have credentials`
+      );
+
+    const client = Twilio(apiKeySid, apiKeySecret, {
+      accountSid,
+    });
+
+    await client.incomingPhoneNumbers(voiceLine.providerLineSid).update({
+      voiceUrl: `${config.frontend.url}/api/v1/voice/twilio/record/${voiceLineId}`,
+      voiceMethod: "POST",
+    });
+  });
+
+export default voiceLineUpdateTask;

apps/metamigo-worker/tasks/whatsapp-message.ts

@@ -0,0 +1,94 @@
+/* eslint-disable camelcase */
+import { withDb, AppDatabase } from "../db";
+import workerUtils from "../utils";
+
+interface WebhookPayload {
+  to: string;
+  from: string;
+  message_id: string;
+  sent_at: string;
+  message: string;
+  attachment: string;
+  filename: string;
+  mime_type: string;
+}
+
+interface WhatsappMessageTaskOptions {
+  waMessageId: string;
+  waMessage: string;
+  waTimestamp: string;
+  attachment: string;
+  filename: string;
+  mimetype: string;
+  botPhoneNumber: string;
+  whatsappBotId: string;
+}
+
+const formatPayload = (
+  messageInfo: WhatsappMessageTaskOptions
+): WebhookPayload => {
+  const {
+    waMessageId,
+    waMessage,
+    waTimestamp,
+    attachment,
+    filename,
+    mimetype,
+    botPhoneNumber,
+  } = messageInfo;
+  const parsedMessage = JSON.parse(waMessage);
+  const message = parsedMessage.message?.conversation ?? 
+    parsedMessage.message?.extendedTextMessage?.text ?? 
+    parsedMessage.message?.imageMessage?.caption ??
+    parsedMessage.message?.videoMessage?.caption;
+
+  return {
+    to: botPhoneNumber,
+    from: parsedMessage.key.remoteJid,
+    message_id: waMessageId,
+    sent_at: waTimestamp,
+    message,
+    attachment,
+    filename,
+    mime_type: mimetype,
+  };
+};
+
+const notifyWebhooks = async (
+  db: AppDatabase,
+  messageInfo: WhatsappMessageTaskOptions
+) => {
+  const { waMessageId, whatsappBotId } = messageInfo;
+  const webhooks = await db.webhooks.findAllByBackendId(
+    "whatsapp",
+    whatsappBotId
+  );
+  if (webhooks && webhooks.length === 0) return;
+
+  webhooks.forEach(({ id }) => {
+    const payload = formatPayload(messageInfo);
+    console.log({ payload });
+    workerUtils.addJob(
+      "notify-webhook",
+      {
+        payload,
+        webhookId: id,
+      },
+      {
+        // this de-deduplicates the job
+        jobKey: `webhook-${id}-message-${waMessageId}`,
+      }
+    );
+  });
+};
+
+const whatsappMessageTask = async (
+  options: WhatsappMessageTaskOptions
+): Promise<void> => {
+  console.log(options);
+  withDb(async (db: AppDatabase) => {
+    await notifyWebhooks(db, options);
+  });
+};
+
+export default whatsappMessageTask;