sr2/link-stack
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Build fixes

Browse source
This commit is contained in:
Darren Clarke 2023-05-25 09:27:26 +00:00
parent 70d97c1342
commit 04ecef98da
9 changed files with 52 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

64
apps/metamigo-worker/tasks/import-leafcutter.ts
View file

@ -5,27 +5,27 @@ import { withDb, AppDatabase } from "../db";
import { loadConfig } from "@digiresilience/metamigo-config";
type LabelStudioTicket = {
id: string
is_labeled: boolean
annotations: Record<string, unknown>[]
data: Record<string, unknown>
id: string;
is_labeled: boolean;
annotations: Record<string, unknown>[];
data: Record<string, unknown>;
updated_at: string;
}
};
type LeafcutterTicket = {
id: string
incident: string[]
technology: string[]
targeted_group: string[]
country: string[]
region: string[]
continent: string[]
date: Date
origin: string
origin_id: string
source_created_at: string
source_updated_at: string
}
id: string;
incident: string[];
technology: string[];
targeted_group: string[];
country: string[];
region: string[];
continent: string[];
date: Date;
origin: string;
origin_id: string;
source_created_at: string;
source_updated_at: string;
};
const getLabelStudioTickets = async (page: number): Promise<LabelStudioTicket[]> => {
const {
@ -43,15 +43,15 @@ const getLabelStudioTickets = async (page: number): Promise<LabelStudioTicket[]>
page_size: "50",
page: `${page}`,
});
console.log({ url: `${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}` })
console.log({ url: `${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}` });
const res = await fetch(`${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}`,
{ headers });
console.log({ res })
const tasksResult = await res.json();
console.log({ res });
const tasksResult: any = await res.json();
console.log({ tasksResult });
return tasksResult;
}
};
const fetchFromLabelStudio = async (minUpdatedTimestamp: Date): Promise<LabelStudioTicket[]> => {
const pages = [...Array.from({ length: 10000 }).keys()];
@ -59,15 +59,15 @@ const fetchFromLabelStudio = async (minUpdatedTimestamp: Date): Promise<LabelStu
for await (const page of pages) {
const docs = await getLabelStudioTickets(page + 1);
console.log({ page, docs })
console.log({ page, docs });
if (docs && docs.length > 0) {
for (const doc of docs) {
const updatedAt = new Date(doc.updated_at);
console.log({ updatedAt, minUpdatedTimestamp });
if (updatedAt > minUpdatedTimestamp) {
console.log(`Adding doc`, { doc })
allDocs.push(doc)
console.log(`Adding doc`, { doc });
allDocs.push(doc);
}
}
} else {
@ -75,9 +75,9 @@ const fetchFromLabelStudio = async (minUpdatedTimestamp: Date): Promise<LabelStu
}
}
console.log({ allDocs })
console.log({ allDocs });
return allDocs;
}
};
const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
const {
@ -89,9 +89,9 @@ const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
}
} = await loadConfig();
console.log({ tickets })
console.log({ tickets });
const filteredTickets = tickets.filter((ticket) => ticket.is_labeled);
console.log({ filteredTickets })
console.log({ filteredTickets });
const finalTickets: LeafcutterTicket[] = filteredTickets.map((ticket) => {
const {
id,
@ -132,7 +132,7 @@ const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
});
console.log("Sending to Leafcutter");
console.log({ finalTickets })
console.log({ finalTickets });
const result = await fetch(opensearchApiUrl, {
method: "POST",
@ -155,9 +155,9 @@ const importLeafcutterTask = async (): Promise<void> => {
const newLastTimestamp = new Date();
console.log({ contributorName, settingName, res, startTimestamp, newLastTimestamp });
const tickets = await fetchFromLabelStudio(startTimestamp);
console.log({ tickets })
console.log({ tickets });
await sendToLeafcutter(tickets);
await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp })
await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp });
});
};