/* eslint-disable camelcase */ import { convert } from "html-to-text"; import fetch from "node-fetch"; import { URLSearchParams } from "url"; import { withDb, AppDatabase } from "../db"; import { loadConfig } from "@digiresilience/metamigo-config"; import { tagMap } from "../lib/tag-map"; type FormattedZammadTicket = { data: Record, predictions: Record[]; }; const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promise<[boolean, FormattedZammadTicket[]]> => { const { leafcutter: { zammadApiUrl, zammadApiKey, contributorName, contributorId } } = await loadConfig(); const headers = { Authorization: `Token ${zammadApiKey}` }; let shouldContinue = false; const docs = []; const ticketsQuery = new URLSearchParams({ "expand": "true", "sort_by": "updated_at", "order_by": "asc", "query": "state.name: closed", "per_page": "25", "page": `${page}`, }); const rawTickets = await fetch(`${zammadApiUrl}/tickets/search?${ticketsQuery}`, { headers } ); const tickets: any = await rawTickets.json(); console.log({ tickets }); if (!tickets || tickets.length === 0) { return [shouldContinue, docs]; } for await (const ticket of tickets) { const { id: source_id, created_at, updated_at, close_at } = ticket; const source_created_at = new Date(created_at); const source_updated_at = new Date(updated_at); const source_closed_at = new Date(close_at); shouldContinue = true; if (source_closed_at <= minUpdatedTimestamp) { console.log(`Skipping ticket`, { source_id, source_updated_at, source_closed_at, minUpdatedTimestamp }); continue; } console.log(`Processing ticket`, { source_id, source_updated_at, source_closed_at, minUpdatedTimestamp }); const rawArticles = await fetch(`${zammadApiUrl}/ticket_articles/by_ticket/${source_id}`, { headers } ); const articles: any = await rawArticles.json(); let articleText = ""; for (const article of articles) { const { content_type: contentType, body } = article; if (contentType === "text/html") { const cleanArticleText = convert(body); articleText += cleanArticleText + "\n\n"; } else { articleText += body + "\n\n"; } } const tagsQuery = new URLSearchParams({ object: "Ticket", o_id: source_id, }); const rawTags = await fetch(`${zammadApiUrl}/tags?${tagsQuery}`, { headers }); const { tags }: any = await rawTags.json(); const transformedTags = []; for (const tag of tags) { const outputs = tagMap[tag]; if (outputs) { transformedTags.push(...outputs); } } const doc: FormattedZammadTicket = { data: { ticket: articleText, contributor_id: contributorId, source_id, source_closed_at, source_created_at, source_updated_at, }, predictions: [] }; const result = transformedTags.map((tag) => { return { type: "choices", value: { choices: [tag.value], }, to_name: "ticket", from_name: tag.field, }; }); if (result.length > 0) { doc.predictions.push({ model_version: `${contributorName}TranslatorV1`, result, }); } docs.push(doc); } return [shouldContinue, docs]; }; const fetchFromZammad = async (minUpdatedTimestamp: Date): Promise => { const pages = [...Array.from({ length: 10000 }).keys()]; const allTickets: FormattedZammadTicket[] = []; for await (const page of pages) { const [shouldContinue, tickets] = await getZammadTickets(page + 1, minUpdatedTimestamp); if (!shouldContinue) { break; } if (tickets.length > 0) { allTickets.push(...tickets); } } return allTickets; }; const sendToLabelStudio = async (tickets: FormattedZammadTicket[]) => { const { leafcutter: { labelStudioApiUrl, labelStudioApiKey } } = await loadConfig(); const headers = { Authorization: `Token ${labelStudioApiKey}`, "Content-Type": "application/json", Accept: "application/json", }; for await (const ticket of tickets) { const res = await fetch(`${labelStudioApiUrl}/projects/1/import`, { method: "POST", headers, body: JSON.stringify([ticket]), }); const importResult = await res.json(); console.log(JSON.stringify(importResult, undefined, 2)); } }; const importLabelStudioTask = async (): Promise => { withDb(async (db: AppDatabase) => { const { leafcutter: { contributorName } } = await loadConfig(); const settingName = `${contributorName}ImportLabelStudioTask`; const res: any = await db.settings.findByName(settingName); const startTimestamp = res?.value?.minUpdatedTimestamp ? new Date(res.value.minUpdatedTimestamp as string) : new Date("2023-03-01"); const tickets = await fetchFromZammad(startTimestamp); if (tickets.length > 0) { await sendToLabelStudio(tickets); const lastTicket = tickets.pop(); const newLastTimestamp = lastTicket.data.source_closed_at; console.log({ newLastTimestamp }); await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp }); } }); }; export default importLabelStudioTask;