Bridge worker updates

apps/bridge-worker/tasks/leafcutter/import-label-studio.ts

@@ -0,0 +1,209 @@
+/* eslint-disable camelcase */
+import { convert } from "html-to-text";
+import { URLSearchParams } from "url";
+import { withDb, AppDatabase } from "../../lib/db";
+// import { loadConfig } from "@digiresilience/bridge-config";
+import { tagMap } from "../../lib/tag-map";
+
+const config: any = {};
+
+type FormattedZammadTicket = {
+  data: Record<string, unknown>;
+  predictions: Record<string, unknown>[];
+};
+
+const getZammadTickets = async (
+  page: number,
+  minUpdatedTimestamp: Date,
+): Promise<[boolean, FormattedZammadTicket[]]> => {
+  const {
+    leafcutter: { zammadApiUrl, zammadApiKey, contributorName, contributorId },
+  } = config;
+  const headers = { Authorization: `Token ${zammadApiKey}` };
+  let shouldContinue = false;
+  const docs = [];
+  const ticketsQuery = new URLSearchParams({
+    expand: "true",
+    sort_by: "updated_at",
+    order_by: "asc",
+    query: "state.name: closed",
+    per_page: "25",
+    page: `${page}`,
+  });
+  const rawTickets = await fetch(
+    `${zammadApiUrl}/tickets/search?${ticketsQuery}`,
+    { headers },
+  );
+  const tickets: any = await rawTickets.json();
+  console.log({ tickets });
+  if (!tickets || tickets.length === 0) {
+    return [shouldContinue, docs];
+  }
+
+  for await (const ticket of tickets) {
+    const { id: source_id, created_at, updated_at, close_at } = ticket;
+    const source_created_at = new Date(created_at);
+    const source_updated_at = new Date(updated_at);
+    const source_closed_at = new Date(close_at);
+    shouldContinue = true;
+
+    if (source_closed_at <= minUpdatedTimestamp) {
+      console.log(`Skipping ticket`, {
+        source_id,
+        source_updated_at,
+        source_closed_at,
+        minUpdatedTimestamp,
+      });
+      continue;
+    }
+
+    console.log(`Processing ticket`, {
+      source_id,
+      source_updated_at,
+      source_closed_at,
+      minUpdatedTimestamp,
+    });
+
+    const rawArticles = await fetch(
+      `${zammadApiUrl}/ticket_articles/by_ticket/${source_id}`,
+      { headers },
+    );
+    const articles: any = await rawArticles.json();
+    let articleText = "";
+
+    for (const article of articles) {
+      const { content_type: contentType, body } = article;
+
+      if (contentType === "text/html") {
+        const cleanArticleText = convert(body);
+        articleText += cleanArticleText + "\n\n";
+      } else {
+        articleText += body + "\n\n";
+      }
+    }
+
+    const tagsQuery = new URLSearchParams({
+      object: "Ticket",
+      o_id: source_id,
+    });
+
+    const rawTags = await fetch(`${zammadApiUrl}/tags?${tagsQuery}`, {
+      headers,
+    });
+    const { tags }: any = await rawTags.json();
+    const transformedTags = [];
+    for (const tag of tags) {
+      const outputs = tagMap[tag];
+      if (outputs) {
+        transformedTags.push(...outputs);
+      }
+    }
+
+    const doc: FormattedZammadTicket = {
+      data: {
+        ticket: articleText,
+        contributor_id: contributorId,
+        source_id,
+        source_closed_at,
+        source_created_at,
+        source_updated_at,
+      },
+      predictions: [],
+    };
+
+    const result = transformedTags.map((tag) => {
+      return {
+        type: "choices",
+        value: {
+          choices: [tag.value],
+        },
+        to_name: "ticket",
+        from_name: tag.field,
+      };
+    });
+
+    if (result.length > 0) {
+      doc.predictions.push({
+        model_version: `${contributorName}TranslatorV1`,
+        result,
+      });
+    }
+
+    docs.push(doc);
+  }
+
+  return [shouldContinue, docs];
+};
+
+const fetchFromZammad = async (
+  minUpdatedTimestamp: Date,
+): Promise<FormattedZammadTicket[]> => {
+  const pages = [...Array.from({ length: 10000 }).keys()];
+  const allTickets: FormattedZammadTicket[] = [];
+
+  for await (const page of pages) {
+    const [shouldContinue, tickets] = await getZammadTickets(
+      page + 1,
+      minUpdatedTimestamp,
+    );
+
+    if (!shouldContinue) {
+      break;
+    }
+
+    if (tickets.length > 0) {
+      allTickets.push(...tickets);
+    }
+  }
+
+  return allTickets;
+};
+
+const sendToLabelStudio = async (tickets: FormattedZammadTicket[]) => {
+  const {
+    leafcutter: { labelStudioApiUrl, labelStudioApiKey },
+  } = config;
+
+  const headers = {
+    Authorization: `Token ${labelStudioApiKey}`,
+    "Content-Type": "application/json",
+    Accept: "application/json",
+  };
+
+  for await (const ticket of tickets) {
+    const res = await fetch(`${labelStudioApiUrl}/projects/1/import`, {
+      method: "POST",
+      headers,
+      body: JSON.stringify([ticket]),
+    });
+    const importResult = await res.json();
+
+    console.log(JSON.stringify(importResult, undefined, 2));
+  }
+};
+
+const importLabelStudioTask = async (): Promise<void> => {
+  withDb(async (db: AppDatabase) => {
+    const {
+      leafcutter: { contributorName },
+    } = config;
+    const settingName = `${contributorName}ImportLabelStudioTask`;
+    const res: any = await db.settings.findByName(settingName);
+    const startTimestamp = res?.value?.minUpdatedTimestamp
+      ? new Date(res.value.minUpdatedTimestamp as string)
+      : new Date("2023-03-01");
+    const tickets = await fetchFromZammad(startTimestamp);
+
+    if (tickets.length > 0) {
+      await sendToLabelStudio(tickets);
+      const lastTicket = tickets.pop();
+      const newLastTimestamp = lastTicket.data.source_closed_at;
+      console.log({ newLastTimestamp });
+      await db.settings.upsert(settingName, {
+        minUpdatedTimestamp: newLastTimestamp,
+      });
+    }
+  });
+};
+
+export default importLabelStudioTask;