sr2/link-stack
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Metamigo -> Bridge

Browse source
This commit is contained in:
Darren Clarke 2024-04-21 09:44:30 +02:00
parent 242f3cf6b8
commit a445762a37
145 changed files with 396 additions and 16668 deletions
Download patch file Download diff file Expand all files Collapse all files

176
apps/metamigo-worker/tasks/import-label-studio.ts
View file

@ -1,176 +0,0 @@
/* eslint-disable camelcase */
import { convert } from "html-to-text";
import fetch from "node-fetch";
import { URLSearchParams } from "url";
import { withDb, AppDatabase } from "../db";
import { loadConfig } from "@digiresilience/metamigo-config";
import { tagMap } from "../lib/tag-map";
type FormattedZammadTicket = {
data: Record<string, unknown>,
predictions: Record<string, unknown>[];
};
const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promise<[boolean, FormattedZammadTicket[]]> => {
const { leafcutter: { zammadApiUrl, zammadApiKey, contributorName, contributorId } } = await loadConfig();
const headers = { Authorization: `Token ${zammadApiKey}` };
let shouldContinue = false;
const docs = [];
const ticketsQuery = new URLSearchParams({
"expand": "true",
"sort_by": "updated_at",
"order_by": "asc",
"query": "state.name: closed",
"per_page": "25",
"page": `${page}`,
});
const rawTickets = await fetch(`${zammadApiUrl}/tickets/search?${ticketsQuery}`,
{ headers }
);
const tickets: any = await rawTickets.json();
console.log({ tickets });
if (!tickets || tickets.length === 0) {
return [shouldContinue, docs];
}
for await (const ticket of tickets) {
const { id: source_id, created_at, updated_at, close_at } = ticket;
const source_created_at = new Date(created_at);
const source_updated_at = new Date(updated_at);
const source_closed_at = new Date(close_at);
shouldContinue = true;
if (source_closed_at <= minUpdatedTimestamp) {
console.log(`Skipping ticket`, { source_id, source_updated_at, source_closed_at, minUpdatedTimestamp });
continue;
}
console.log(`Processing ticket`, { source_id, source_updated_at, source_closed_at, minUpdatedTimestamp });
const rawArticles = await fetch(`${zammadApiUrl}/ticket_articles/by_ticket/${source_id}`,
{ headers }
);
const articles: any = await rawArticles.json();
let articleText = "";
for (const article of articles) {
const { content_type: contentType, body } = article;
if (contentType === "text/html") {
const cleanArticleText = convert(body);
articleText += cleanArticleText + "\n\n";
} else {
articleText += body + "\n\n";
}
}
const tagsQuery = new URLSearchParams({
object: "Ticket",
o_id: source_id,
});
const rawTags = await fetch(`${zammadApiUrl}/tags?${tagsQuery}`, { headers });
const { tags }: any = await rawTags.json();
const transformedTags = [];
for (const tag of tags) {
const outputs = tagMap[tag];
if (outputs) {
transformedTags.push(...outputs);
}
}
const doc: FormattedZammadTicket = {
data: {
ticket: articleText,
contributor_id: contributorId,
source_id,
source_closed_at,
source_created_at,
source_updated_at,
},
predictions: []
};
const result = transformedTags.map((tag) => {
return {
type: "choices",
value: {
choices: [tag.value],
},
to_name: "ticket",
from_name: tag.field,
};
});
if (result.length > 0) {
doc.predictions.push({
model_version: `${contributorName}TranslatorV1`,
result,
});
}
docs.push(doc);
}
return [shouldContinue, docs];
};
const fetchFromZammad = async (minUpdatedTimestamp: Date): Promise<FormattedZammadTicket[]> => {
const pages = [...Array.from({ length: 10000 }).keys()];
const allTickets: FormattedZammadTicket[] = [];
for await (const page of pages) {
const [shouldContinue, tickets] = await getZammadTickets(page + 1, minUpdatedTimestamp);
if (!shouldContinue) {
break;
}
if (tickets.length > 0) {
allTickets.push(...tickets);
}
}
return allTickets;
};
const sendToLabelStudio = async (tickets: FormattedZammadTicket[]) => {
const { leafcutter: { labelStudioApiUrl, labelStudioApiKey } } = await loadConfig();
const headers = {
Authorization: `Token ${labelStudioApiKey}`,
"Content-Type": "application/json",
Accept: "application/json",
};
for await (const ticket of tickets) {
const res = await fetch(`${labelStudioApiUrl}/projects/1/import`, {
method: "POST",
headers,
body: JSON.stringify([ticket]),
});
const importResult = await res.json();
console.log(JSON.stringify(importResult, undefined, 2));
}
};
const importLabelStudioTask = async (): Promise<void> => {
withDb(async (db: AppDatabase) => {
const { leafcutter: { contributorName } } = await loadConfig();
const settingName = `${contributorName}ImportLabelStudioTask`;
const res: any = await db.settings.findByName(settingName);
const startTimestamp = res?.value?.minUpdatedTimestamp ? new Date(res.value.minUpdatedTimestamp as string) : new Date("2023-03-01");
const tickets = await fetchFromZammad(startTimestamp);
if (tickets.length > 0) {
await sendToLabelStudio(tickets);
const lastTicket = tickets.pop();
const newLastTimestamp = lastTicket.data.source_closed_at;
console.log({ newLastTimestamp });
await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp });
}
});
};
export default importLabelStudioTask;