link-stack/apps/bridge-worker/tasks/import-label-studio.ts

/* eslint-disable camelcase */
import { convert } from "html-to-text";
import fetch from "node-fetch";
import { URLSearchParams } from "url";
import { withDb, AppDatabase } from "../db";
import { loadConfig } from "@digiresilience/bridge-config";
import { tagMap } from "../lib/tag-map";

type FormattedZammadTicket = {
  data: Record<string, unknown>;
  predictions: Record<string, unknown>[];
};

const getZammadTickets = async (
  page: number,
  minUpdatedTimestamp: Date,
): Promise<[boolean, FormattedZammadTicket[]]> => {
  const {
    leafcutter: { zammadApiUrl, zammadApiKey, contributorName, contributorId },
  } = await loadConfig();
  const headers = { Authorization: `Token ${zammadApiKey}` };
  let shouldContinue = false;
  const docs = [];
  const ticketsQuery = new URLSearchParams({
    expand: "true",
    sort_by: "updated_at",
    order_by: "asc",
    query: "state.name: closed",
    per_page: "25",
    page: `${page}`,
  });
  const rawTickets = await fetch(
    `${zammadApiUrl}/tickets/search?${ticketsQuery}`,
    { headers },
  );
  const tickets: any = await rawTickets.json();
  console.log({ tickets });
  if (!tickets || tickets.length === 0) {
    return [shouldContinue, docs];
  }

  for await (const ticket of tickets) {
    const { id: source_id, created_at, updated_at, close_at } = ticket;
    const source_created_at = new Date(created_at);
    const source_updated_at = new Date(updated_at);
    const source_closed_at = new Date(close_at);
    shouldContinue = true;

    if (source_closed_at <= minUpdatedTimestamp) {
      console.log(`Skipping ticket`, {
        source_id,
        source_updated_at,
        source_closed_at,
        minUpdatedTimestamp,
      });
      continue;
    }

    console.log(`Processing ticket`, {
      source_id,
      source_updated_at,
      source_closed_at,
      minUpdatedTimestamp,
    });

    const rawArticles = await fetch(
      `${zammadApiUrl}/ticket_articles/by_ticket/${source_id}`,
      { headers },
    );
    const articles: any = await rawArticles.json();
    let articleText = "";

    for (const article of articles) {
      const { content_type: contentType, body } = article;

      if (contentType === "text/html") {
        const cleanArticleText = convert(body);
        articleText += cleanArticleText + "\n\n";
      } else {
        articleText += body + "\n\n";
      }
    }

    const tagsQuery = new URLSearchParams({
      object: "Ticket",
      o_id: source_id,
    });

    const rawTags = await fetch(`${zammadApiUrl}/tags?${tagsQuery}`, {
      headers,
    });
    const { tags }: any = await rawTags.json();
    const transformedTags = [];
    for (const tag of tags) {
      const outputs = tagMap[tag];
      if (outputs) {
        transformedTags.push(...outputs);
      }
    }

    const doc: FormattedZammadTicket = {
      data: {
        ticket: articleText,
        contributor_id: contributorId,
        source_id,
        source_closed_at,
        source_created_at,
        source_updated_at,
      },
      predictions: [],
    };

    const result = transformedTags.map((tag) => {
      return {
        type: "choices",
        value: {
          choices: [tag.value],
        },
        to_name: "ticket",
        from_name: tag.field,
      };
    });

    if (result.length > 0) {
      doc.predictions.push({
        model_version: `${contributorName}TranslatorV1`,
        result,
      });
    }

    docs.push(doc);
  }

  return [shouldContinue, docs];
};

const fetchFromZammad = async (
  minUpdatedTimestamp: Date,
): Promise<FormattedZammadTicket[]> => {
  const pages = [...Array.from({ length: 10000 }).keys()];
  const allTickets: FormattedZammadTicket[] = [];

  for await (const page of pages) {
    const [shouldContinue, tickets] = await getZammadTickets(
      page + 1,
      minUpdatedTimestamp,
    );

    if (!shouldContinue) {
      break;
    }

    if (tickets.length > 0) {
      allTickets.push(...tickets);
    }
  }

  return allTickets;
};

const sendToLabelStudio = async (tickets: FormattedZammadTicket[]) => {
  const {
    leafcutter: { labelStudioApiUrl, labelStudioApiKey },
  } = await loadConfig();

  const headers = {
    Authorization: `Token ${labelStudioApiKey}`,
    "Content-Type": "application/json",
    Accept: "application/json",
  };

  for await (const ticket of tickets) {
    const res = await fetch(`${labelStudioApiUrl}/projects/1/import`, {
      method: "POST",
      headers,
      body: JSON.stringify([ticket]),
    });
    const importResult = await res.json();

    console.log(JSON.stringify(importResult, undefined, 2));
  }
};

const importLabelStudioTask = async (): Promise<void> => {
  withDb(async (db: AppDatabase) => {
    const {
      leafcutter: { contributorName },
    } = await loadConfig();
    const settingName = `${contributorName}ImportLabelStudioTask`;
    const res: any = await db.settings.findByName(settingName);
    const startTimestamp = res?.value?.minUpdatedTimestamp
      ? new Date(res.value.minUpdatedTimestamp as string)
      : new Date("2023-03-01");
    const tickets = await fetchFromZammad(startTimestamp);

    if (tickets.length > 0) {
      await sendToLabelStudio(tickets);
      const lastTicket = tickets.pop();
      const newLastTimestamp = lastTicket.data.source_closed_at;
      console.log({ newLastTimestamp });
      await db.settings.upsert(settingName, {
        minUpdatedTimestamp: newLastTimestamp,
      });
    }
  });
};

export default importLabelStudioTask;