Remove leafcutter and related packages
This commit is contained in:
Darren Clarke
parent
48165db6a2
commit
3a1063e40e
264 changed files with 1763 additions and 25062 deletions
|
|
@ -1,195 +0,0 @@
|
|||
/* eslint-disable camelcase */
|
||||
/*
|
||||
import { convert } from "html-to-text";
|
||||
import { URLSearchParams } from "url";
|
||||
import { withDb, AppDatabase } from "../../lib/db.js";
|
||||
// import { loadConfig } from "@digiresilience/bridge-config";
|
||||
import { tagMap } from "../../lib/tag-map.js";
|
||||
|
||||
const config: any = {};
|
||||
|
||||
type FormattedZammadTicket = {
|
||||
data: Record<string, unknown>;
|
||||
predictions: Record<string, unknown>[];
|
||||
};
|
||||
|
||||
const getZammadTickets = async (
|
||||
page: number,
|
||||
minUpdatedTimestamp: Date,
|
||||
): Promise<[boolean, FormattedZammadTicket[]]> => {
|
||||
const {
|
||||
leafcutter: { zammadApiUrl, zammadApiKey, contributorName, contributorId },
|
||||
} = config;
|
||||
const headers = { Authorization: `Token ${zammadApiKey}` };
|
||||
let shouldContinue = false;
|
||||
const docs = [];
|
||||
const ticketsQuery = new URLSearchParams({
|
||||
expand: "true",
|
||||
sort_by: "updated_at",
|
||||
order_by: "asc",
|
||||
query: "state.name: closed",
|
||||
per_page: "25",
|
||||
page: `${page}`,
|
||||
});
|
||||
const rawTickets = await fetch(
|
||||
`${zammadApiUrl}/tickets/search?${ticketsQuery}`,
|
||||
{ headers },
|
||||
);
|
||||
const tickets: any = await rawTickets.json();
|
||||
if (!tickets || tickets.length === 0) {
|
||||
return [shouldContinue, docs];
|
||||
}
|
||||
|
||||
for await (const ticket of tickets) {
|
||||
const { id: source_id, created_at, updated_at, close_at } = ticket;
|
||||
const source_created_at = new Date(created_at);
|
||||
const source_updated_at = new Date(updated_at);
|
||||
const source_closed_at = new Date(close_at);
|
||||
shouldContinue = true;
|
||||
|
||||
if (source_closed_at <= minUpdatedTimestamp) {
|
||||
continue;
|
||||
}
|
||||
const rawArticles = await fetch(
|
||||
`${zammadApiUrl}/ticket_articles/by_ticket/${source_id}`,
|
||||
{ headers },
|
||||
);
|
||||
const articles: any = await rawArticles.json();
|
||||
let articleText = "";
|
||||
|
||||
for (const article of articles) {
|
||||
const { content_type: contentType, body } = article;
|
||||
|
||||
if (contentType === "text/html") {
|
||||
const cleanArticleText = convert(body);
|
||||
articleText += cleanArticleText + "\n\n";
|
||||
} else {
|
||||
articleText += body + "\n\n";
|
||||
}
|
||||
}
|
||||
|
||||
const tagsQuery = new URLSearchParams({
|
||||
object: "Ticket",
|
||||
o_id: source_id,
|
||||
});
|
||||
|
||||
const rawTags = await fetch(`${zammadApiUrl}/tags?${tagsQuery}`, {
|
||||
headers,
|
||||
});
|
||||
const { tags }: any = await rawTags.json();
|
||||
const transformedTags = [];
|
||||
for (const tag of tags) {
|
||||
const outputs = tagMap[tag];
|
||||
if (outputs) {
|
||||
transformedTags.push(...outputs);
|
||||
}
|
||||
}
|
||||
|
||||
const doc: FormattedZammadTicket = {
|
||||
data: {
|
||||
ticket: articleText,
|
||||
contributor_id: contributorId,
|
||||
source_id,
|
||||
source_closed_at,
|
||||
source_created_at,
|
||||
source_updated_at,
|
||||
},
|
||||
predictions: [],
|
||||
};
|
||||
|
||||
const result = transformedTags.map((tag) => {
|
||||
return {
|
||||
type: "choices",
|
||||
value: {
|
||||
choices: [tag.value],
|
||||
},
|
||||
to_name: "ticket",
|
||||
from_name: tag.field,
|
||||
};
|
||||
});
|
||||
|
||||
if (result.length > 0) {
|
||||
doc.predictions.push({
|
||||
model_version: `${contributorName}TranslatorV1`,
|
||||
result,
|
||||
});
|
||||
}
|
||||
|
||||
docs.push(doc);
|
||||
}
|
||||
|
||||
return [shouldContinue, docs];
|
||||
};
|
||||
|
||||
const fetchFromZammad = async (
|
||||
minUpdatedTimestamp: Date,
|
||||
): Promise<FormattedZammadTicket[]> => {
|
||||
const pages = [...Array.from({ length: 10000 }).keys()];
|
||||
const allTickets: FormattedZammadTicket[] = [];
|
||||
|
||||
for await (const page of pages) {
|
||||
const [shouldContinue, tickets] = await getZammadTickets(
|
||||
page + 1,
|
||||
minUpdatedTimestamp,
|
||||
);
|
||||
|
||||
if (!shouldContinue) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (tickets.length > 0) {
|
||||
allTickets.push(...tickets);
|
||||
}
|
||||
}
|
||||
|
||||
return allTickets;
|
||||
};
|
||||
|
||||
const sendToLabelStudio = async (tickets: FormattedZammadTicket[]) => {
|
||||
const {
|
||||
leafcutter: { labelStudioApiUrl, labelStudioApiKey },
|
||||
} = config;
|
||||
|
||||
const headers = {
|
||||
Authorization: `Token ${labelStudioApiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json",
|
||||
};
|
||||
|
||||
for await (const ticket of tickets) {
|
||||
const res = await fetch(`${labelStudioApiUrl}/projects/1/import`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify([ticket]),
|
||||
});
|
||||
const importResult = await res.json();
|
||||
}
|
||||
};
|
||||
*/
|
||||
|
||||
const importLabelStudioTask = async (): Promise<void> => {
|
||||
/*
|
||||
withDb(async (db: AppDatabase) => {
|
||||
const {
|
||||
leafcutter: { contributorName },
|
||||
} = config;
|
||||
const settingName = `${contributorName}ImportLabelStudioTask`;
|
||||
const res: any = await db.settings.findByName(settingName);
|
||||
const startTimestamp = res?.value?.minUpdatedTimestamp
|
||||
? new Date(res.value.minUpdatedTimestamp as string)
|
||||
: new Date("2023-03-01");
|
||||
const tickets = await fetchFromZammad(startTimestamp);
|
||||
|
||||
if (tickets.length > 0) {
|
||||
await sendToLabelStudio(tickets);
|
||||
const lastTicket = tickets.pop();
|
||||
const newLastTimestamp = lastTicket.data.source_closed_at;
|
||||
await db.settings.upsert(settingName, {
|
||||
minUpdatedTimestamp: newLastTimestamp,
|
||||
});
|
||||
}
|
||||
});
|
||||
*/
|
||||
};
|
||||
|
||||
export default importLabelStudioTask;
|
||||
|
|
@ -1,162 +0,0 @@
|
|||
/* eslint-disable camelcase */
|
||||
/*
|
||||
import { URLSearchParams } from "url";
|
||||
import { withDb, AppDatabase } from "../../lib/db.js";
|
||||
import { createLogger } from "@link-stack/logger";
|
||||
|
||||
const logger = createLogger('bridge-worker-import-leafcutter');
|
||||
// import { loadConfig } from "@digiresilience/bridge-config";
|
||||
|
||||
const config: any = {};
|
||||
|
||||
type LabelStudioTicket = {
|
||||
id: string;
|
||||
is_labeled: boolean;
|
||||
annotations: Record<string, unknown>[];
|
||||
data: Record<string, unknown>;
|
||||
updated_at: string;
|
||||
};
|
||||
|
||||
type LeafcutterTicket = {
|
||||
id: string;
|
||||
incident: string[];
|
||||
technology: string[];
|
||||
targeted_group: string[];
|
||||
country: string[];
|
||||
region: string[];
|
||||
continent: string[];
|
||||
date: Date;
|
||||
origin: string;
|
||||
origin_id: string;
|
||||
source_created_at: string;
|
||||
source_updated_at: string;
|
||||
};
|
||||
|
||||
const getLabelStudioTickets = async (
|
||||
page: number,
|
||||
): Promise<LabelStudioTicket[]> => {
|
||||
const {
|
||||
leafcutter: { labelStudioApiUrl, labelStudioApiKey },
|
||||
} = config;
|
||||
const headers = {
|
||||
Authorization: `Token ${labelStudioApiKey}`,
|
||||
Accept: "application/json",
|
||||
};
|
||||
const ticketsQuery = new URLSearchParams({
|
||||
page_size: "50",
|
||||
page: `${page}`,
|
||||
});
|
||||
const res = await fetch(
|
||||
`${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}`,
|
||||
{ headers },
|
||||
);
|
||||
const tasksResult: any = await res.json();
|
||||
|
||||
return tasksResult;
|
||||
};
|
||||
|
||||
const fetchFromLabelStudio = async (
|
||||
minUpdatedTimestamp: Date,
|
||||
): Promise<LabelStudioTicket[]> => {
|
||||
const pages = [...Array.from({ length: 10000 }).keys()];
|
||||
const allDocs: LabelStudioTicket[] = [];
|
||||
|
||||
for await (const page of pages) {
|
||||
const docs = await getLabelStudioTickets(page + 1);
|
||||
|
||||
if (docs && docs.length > 0) {
|
||||
for (const doc of docs) {
|
||||
const updatedAt = new Date(doc.updated_at);
|
||||
if (updatedAt > minUpdatedTimestamp) {
|
||||
allDocs.push(doc);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return allDocs;
|
||||
};
|
||||
|
||||
const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
|
||||
const {
|
||||
leafcutter: {
|
||||
contributorId,
|
||||
opensearchApiUrl,
|
||||
opensearchUsername,
|
||||
opensearchPassword,
|
||||
},
|
||||
} = config;
|
||||
|
||||
const filteredTickets = tickets.filter((ticket) => ticket.is_labeled);
|
||||
const finalTickets: LeafcutterTicket[] = filteredTickets.map((ticket) => {
|
||||
const {
|
||||
id,
|
||||
annotations,
|
||||
data: { source_id, source_created_at, source_updated_at },
|
||||
} = ticket;
|
||||
|
||||
const getTags = (tags: Record<string, any>[], name: string) =>
|
||||
tags
|
||||
.filter((tag) => tag.from_name === name)
|
||||
.map((tag) => tag.value.choices)
|
||||
.flat();
|
||||
|
||||
const allTags = annotations.map(({ result }) => result).flat();
|
||||
const incident = getTags(allTags, "incidentType tag");
|
||||
const technology = getTags(allTags, "platform tag");
|
||||
const country = getTags(allTags, "country tag");
|
||||
const targetedGroup = getTags(allTags, "targetedGroup tag");
|
||||
|
||||
return {
|
||||
id,
|
||||
incident,
|
||||
technology,
|
||||
targeted_group: targetedGroup,
|
||||
country,
|
||||
region: [],
|
||||
continent: [],
|
||||
date: new Date(source_created_at as string),
|
||||
origin: contributorId,
|
||||
origin_id: source_id as string,
|
||||
source_created_at: source_created_at as string,
|
||||
source_updated_at: source_updated_at as string,
|
||||
};
|
||||
});
|
||||
|
||||
logger.info("Sending to Leafcutter");
|
||||
|
||||
const result = await fetch(opensearchApiUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Basic ${Buffer.from(`${opensearchUsername}:${opensearchPassword}`).toString("base64")}`,
|
||||
},
|
||||
body: JSON.stringify({ tickets: finalTickets }),
|
||||
});
|
||||
|
||||
};
|
||||
*/
|
||||
const importLeafcutterTask = async (): Promise<void> => {
|
||||
/*
|
||||
withDb(async (db: AppDatabase) => {
|
||||
const {
|
||||
leafcutter: { contributorName },
|
||||
} = config;
|
||||
const settingName = `${contributorName}ImportLeafcutterTask`;
|
||||
const res: any = await db.settings.findByName(settingName);
|
||||
const startTimestamp = res?.value?.minUpdatedTimestamp
|
||||
? new Date(res.value.minUpdatedTimestamp as string)
|
||||
: new Date("2023-03-01");
|
||||
const newLastTimestamp = new Date();
|
||||
const tickets = await fetchFromLabelStudio(startTimestamp);
|
||||
await sendToLeafcutter(tickets);
|
||||
await db.settings.upsert(settingName, {
|
||||
minUpdatedTimestamp: newLastTimestamp,
|
||||
});
|
||||
});
|
||||
*/
|
||||
};
|
||||
|
||||
export default importLeafcutterTask;
|
||||
Loading…
Add table
Add a link
Reference in a new issue