sr2/link-stack
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Bridge worker updates

Browse source
This commit is contained in:
Darren Clarke 2024-04-21 16:59:50 +02:00
parent a445762a37
commit f93c4ad317
33 changed files with 17584 additions and 161 deletions
Download patch file Download diff file Expand all files Collapse all files

30
apps/bridge-worker/tasks/notify-webhook.ts → apps/bridge-worker/tasks/common/notify-webhook.ts
View file

@ -1,7 +1,7 @@
import Wreck from "@hapi/wreck";
import * as R from "remeda";
import { withDb, AppDatabase } from "../db";
import logger from "../logger";
import { withDb, AppDatabase } from "../../lib/db";
// import logger from "../logger";
export interface WebhookOptions {
webhookId: string;
@ -14,10 +14,10 @@ const notifyWebhooksTask = async (options: WebhookOptions): Promise<void> =>
const webhook = await db.webhooks.findById({ id: webhookId });
if (!webhook) {
logger.debug(
{ webhookId },
"notify-webhook: no webhook registered with id"
);
// logger.debug(
// { webhookId },
// "notify-webhook: no webhook registered with id",
// );
return;
}
@ -28,7 +28,7 @@ const notifyWebhooksTask = async (options: WebhookOptions): Promise<void> =>
acc[h.header] = h.value;
return acc;
},
{}
{},
);
const wreck = Wreck.defaults({
@ -38,18 +38,18 @@ const notifyWebhooksTask = async (options: WebhookOptions): Promise<void> =>
// http errors will bubble up and cause the job to fail and be retried
try {
logger.debug(
{ webhookId, endpointUrl, httpMethod },
"notify-webhook: notifying"
);
// logger.debug(
// { webhookId, endpointUrl, httpMethod },
// "notify-webhook: notifying",
// );
await (httpMethod === "post"
? wreck.post(endpointUrl, { payload })
: wreck.put(endpointUrl, { payload }));
} catch (error: any) {
logger.error(
{ webhookId, error: error.output },
"notify-webhook failed with this error"
);
// logger.error(
// { webhookId, error: error.output },
// "notify-webhook failed with this error",
// );
throw new Error(`webhook failed webhookId=${webhookId}`);
}
});

12
apps/bridge-worker/tasks/facebook/receive-facebook-message.ts Normal file
View file

@ -0,0 +1,12 @@
interface ReceiveFacebookMessageTaskOptions {}
const receiveFacebookMessageTask = async (
options: ReceiveFacebookMessageTaskOptions,
): Promise<void> => {
console.log(options);
// withDb(async (db: AppDatabase) => {
// await notifyWebhooks(db, options);
// });
};
export default receiveFacebookMessageTask;

12
apps/bridge-worker/tasks/facebook/send-facebook-message.ts Normal file
View file

@ -0,0 +1,12 @@
interface SendFacebookMessageTaskOptions {}
const sendFacebookMessageTask = async (
options: SendFacebookMessageTaskOptions,
): Promise<void> => {
console.log(options);
// withDb(async (db: AppDatabase) => {
// await notifyWebhooks(db, options);
// });
};
export default sendFacebookMessageTask;

15
apps/bridge-worker/tasks/import-label-studio.ts → apps/bridge-worker/tasks/leafcutter/import-label-studio.ts
View file

@ -1,10 +1,11 @@
/* eslint-disable camelcase */
import { convert } from "html-to-text";
import fetch from "node-fetch";
import { URLSearchParams } from "url";
import { withDb, AppDatabase } from "../db";
import { loadConfig } from "@digiresilience/bridge-config";
import { tagMap } from "../lib/tag-map";
import { withDb, AppDatabase } from "../../lib/db";
// import { loadConfig } from "@digiresilience/bridge-config";
import { tagMap } from "../../lib/tag-map";
const config: any = {};
type FormattedZammadTicket = {
data: Record<string, unknown>;
@ -17,7 +18,7 @@ const getZammadTickets = async (
): Promise<[boolean, FormattedZammadTicket[]]> => {
const {
leafcutter: { zammadApiUrl, zammadApiKey, contributorName, contributorId },
} = await loadConfig();
} = config;
const headers = { Authorization: `Token ${zammadApiKey}` };
let shouldContinue = false;
const docs = [];
@ -161,7 +162,7 @@ const fetchFromZammad = async (
const sendToLabelStudio = async (tickets: FormattedZammadTicket[]) => {
const {
leafcutter: { labelStudioApiUrl, labelStudioApiKey },
} = await loadConfig();
} = config;
const headers = {
Authorization: `Token ${labelStudioApiKey}`,
@ -185,7 +186,7 @@ const importLabelStudioTask = async (): Promise<void> => {
withDb(async (db: AppDatabase) => {
const {
leafcutter: { contributorName },
} = await loadConfig();
} = config;
const settingName = `${contributorName}ImportLabelStudioTask`;
const res: any = await db.settings.findByName(settingName);
const startTimestamp = res?.value?.minUpdatedTimestamp

14
apps/bridge-worker/tasks/import-leafcutter.ts → apps/bridge-worker/tasks/leafcutter/import-leafcutter.ts
View file

@ -1,8 +1,9 @@
/* eslint-disable camelcase */
import fetch from "node-fetch";
import { URLSearchParams } from "url";
import { withDb, AppDatabase } from "../db";
import { loadConfig } from "@digiresilience/bridge-config";
import { withDb, AppDatabase } from "../../lib/db";
// import { loadConfig } from "@digiresilience/bridge-config";
const config: any = {};
type LabelStudioTicket = {
id: string;
@ -32,8 +33,7 @@ const getLabelStudioTickets = async (
): Promise<LabelStudioTicket[]> => {
const {
leafcutter: { labelStudioApiUrl, labelStudioApiKey },
} = await loadConfig();
} = config;
const headers = {
Authorization: `Token ${labelStudioApiKey}`,
Accept: "application/json",
@ -90,7 +90,7 @@ const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
opensearchUsername,
opensearchPassword,
},
} = await loadConfig();
} = config;
console.log({ tickets });
const filteredTickets = tickets.filter((ticket) => ticket.is_labeled);
@ -148,7 +148,7 @@ const importLeafcutterTask = async (): Promise<void> => {
withDb(async (db: AppDatabase) => {
const {
leafcutter: { contributorName },
} = await loadConfig();
} = config;
const settingName = `${contributorName}ImportLeafcutterTask`;
const res: any = await db.settings.findByName(settingName);
const startTimestamp = res?.value?.minUpdatedTimestamp

29
apps/bridge-worker/tasks/signald-message.ts → apps/bridge-worker/tasks/signal/signald-message.ts
View file

@ -1,8 +1,10 @@
/* eslint-disable camelcase */
import logger from "../logger";
import { IncomingMessagev1 } from "@digiresilience/node-signald/build/main/generated";
import { withDb, AppDatabase } from "../db";
import workerUtils from "../utils";
// import logger from "../logger";
// import { IncomingMessagev1 } from "@digiresilience/node-signald/build/main/generated";
import { withDb, AppDatabase } from "../../lib/db";
import workerUtils from "../../lib/utils";
type IncomingMessagev1 = any;
interface WebhookPayload {
to: string;
@ -25,7 +27,8 @@ interface SignaldMessageTaskOptions {
}
const formatPayload = (opts: SignaldMessageTaskOptions): WebhookPayload => {
const { botId, botPhoneNumber, message, attachment, filename, mimetype } = opts;
const { botId, botPhoneNumber, message, attachment, filename, mimetype } =
opts;
const { source, timestamp, data_message: dataMessage } = message;
const { number }: any = source;
@ -46,7 +49,7 @@ const formatPayload = (opts: SignaldMessageTaskOptions): WebhookPayload => {
const notifyWebhooks = async (
db: AppDatabase,
messageInfo: SignaldMessageTaskOptions
messageInfo: SignaldMessageTaskOptions,
) => {
const {
botId,
@ -54,16 +57,16 @@ const notifyWebhooks = async (
} = messageInfo;
const webhooks = await db.webhooks.findAllByBackendId("signal", botId);
if (webhooks && webhooks.length === 0) {
logger.debug({ botId }, "no webhooks registered for signal bot");
// logger.debug({ botId }, "no webhooks registered for signal bot");
return;
}
webhooks.forEach(({ id }) => {
const payload = formatPayload(messageInfo);
logger.debug(
{ payload },
"formatted signal bot payload for notify-webhook"
);
// logger.debug(
// { payload },
// "formatted signal bot payload for notify-webhook",
// );
workerUtils.addJob(
"notify-webhook",
{
@ -73,13 +76,13 @@ const notifyWebhooks = async (
{
// this de-deduplicates the job
jobKey: `webhook-${id}-message-${botId}-${timestamp}`,
}
},
);
});
};
const signaldMessageTask = async (
options: SignaldMessageTaskOptions
options: SignaldMessageTaskOptions,
): Promise<void> => {
console.log(options);
withDb(async (db: AppDatabase) => {

14
apps/bridge-worker/tasks/twilio-recording.ts → apps/bridge-worker/tasks/voice/twilio-recording.ts
View file

@ -1,8 +1,8 @@
import Wreck from "@hapi/wreck";
import { withDb, AppDatabase } from "../db";
import { twilioClientFor } from "../common";
import { withDb, AppDatabase } from "../../lib/db";
import { twilioClientFor } from "../../lib/common";
import { CallInstance } from "twilio/lib/rest/api/v2010/account/call";
import workerUtils from "../utils";
import workerUtils from "../../lib/utils";
interface WebhookPayload {
startTime: string;
@ -27,7 +27,7 @@ const getTwilioRecording = async (url: string) => {
const formatPayload = (
call: CallInstance,
recording: Buffer
recording: Buffer,
): WebhookPayload => {
return {
startTime: call.startTime.toISOString(),
@ -45,7 +45,7 @@ const notifyWebhooks = async (
db: AppDatabase,
voiceLineId: string,
call: CallInstance,
recording: Buffer
recording: Buffer,
) => {
const webhooks = await db.webhooks.findAllByBackendId("voice", voiceLineId);
if (webhooks && webhooks.length === 0) return;
@ -61,7 +61,7 @@ const notifyWebhooks = async (
{
// this de-depuplicates the job
jobKey: `webhook-${id}-call-${call.sid}`,
}
},
);
});
};
@ -74,7 +74,7 @@ interface TwilioRecordingTaskOptions {
}
const twilioRecordingTask = async (
options: TwilioRecordingTaskOptions
options: TwilioRecordingTaskOptions,
): Promise<void> =>
withDb(async (db: AppDatabase) => {
const { voiceLineId, accountSid, callSid, recordingSid } = options;

8
apps/bridge-worker/tasks/voice-line-audio-update.ts → apps/bridge-worker/tasks/voice/voice-line-audio-update.ts
View file

@ -1,6 +1,6 @@
import { createHash } from "crypto";
import { withDb, AppDatabase } from "../db";
import { convert } from "../lib/media-convert";
import { withDb, AppDatabase } from "../../lib/db";
import { convert } from "../../lib/media-convert";
interface VoiceLineAudioUpdateTaskOptions {
voiceLineId: string;
@ -13,7 +13,7 @@ const sha1sum = (v: any) => {
};
const voiceLineAudioUpdateTask = async (
payload: VoiceLineAudioUpdateTaskOptions
payload: VoiceLineAudioUpdateTaskOptions,
): Promise<void> =>
withDb(async (db: AppDatabase) => {
const { voiceLineId } = payload;
@ -41,7 +41,7 @@ const voiceLineAudioUpdateTask = async (
"audio/mpeg": mp3.toString("base64"),
checksum: webmSha1,
},
}
},
);
});

6
apps/bridge-worker/tasks/voice-line-delete.ts → apps/bridge-worker/tasks/voice/voice-line-delete.ts
View file

@ -1,6 +1,8 @@
import Twilio from "twilio";
import config from "@digiresilience/bridge-config";
import { withDb, AppDatabase } from "../db";
// import config from "@digiresilience/bridge-config";
import { withDb, AppDatabase } from "../../lib/db";
const config: any = {};
interface VoiceLineDeleteTaskOptions {
voiceLineId: string;

6
apps/bridge-worker/tasks/voice-line-provider-update.ts → apps/bridge-worker/tasks/voice/voice-line-provider-update.ts
View file

@ -1,6 +1,8 @@
import Twilio from "twilio";
import config from "@digiresilience/bridge-config";
import { withDb, AppDatabase } from "../db";
// import config from "@digiresilience/bridge-config";
import { withDb, AppDatabase } from "../../lib/db";
const config: any = {};
interface VoiceLineUpdateTaskOptions {
voiceLineId: string;

19
apps/bridge-worker/tasks/whatsapp-message.ts → apps/bridge-worker/tasks/whatsapp/receive-whatsapp-message.ts
View file

@ -1,6 +1,6 @@
/* eslint-disable camelcase */
import { withDb, AppDatabase } from "../db";
import workerUtils from "../utils";
import { withDb, AppDatabase } from "../../lib/db";
import workerUtils from "../../lib/utils";
interface WebhookPayload {
to: string;
@ -25,7 +25,7 @@ interface WhatsappMessageTaskOptions {
}
const formatPayload = (
messageInfo: WhatsappMessageTaskOptions
messageInfo: WhatsappMessageTaskOptions,
): WebhookPayload => {
const {
waMessageId,
@ -37,8 +37,9 @@ const formatPayload = (
botPhoneNumber,
} = messageInfo;
const parsedMessage = JSON.parse(waMessage);
const message = parsedMessage.message?.conversation ??
parsedMessage.message?.extendedTextMessage?.text ??
const message =
parsedMessage.message?.conversation ??
parsedMessage.message?.extendedTextMessage?.text ??
parsedMessage.message?.imageMessage?.caption ??
parsedMessage.message?.videoMessage?.caption;
@ -56,12 +57,12 @@ const formatPayload = (
const notifyWebhooks = async (
db: AppDatabase,
messageInfo: WhatsappMessageTaskOptions
messageInfo: WhatsappMessageTaskOptions,
) => {
const { waMessageId, whatsappBotId } = messageInfo;
const webhooks = await db.webhooks.findAllByBackendId(
"whatsapp",
whatsappBotId
whatsappBotId,
);
if (webhooks && webhooks.length === 0) return;
@ -77,13 +78,13 @@ const notifyWebhooks = async (
{
// this de-deduplicates the job
jobKey: `webhook-${id}-message-${waMessageId}`,
}
},
);
});
};
const whatsappMessageTask = async (
options: WhatsappMessageTaskOptions
options: WhatsappMessageTaskOptions,
): Promise<void> => {
console.log(options);
withDb(async (db: AppDatabase) => {