Build fixes
This commit is contained in:
parent
70d97c1342
commit
04ecef98da
9 changed files with 52 additions and 45 deletions
|
|
@ -3,6 +3,7 @@
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"outDir": "build/main",
|
"outDir": "build/main",
|
||||||
"rootDir": "src",
|
"rootDir": "src",
|
||||||
|
"skipLibCheck": true,
|
||||||
"types": ["jest", "node", "long"],
|
"types": ["jest", "node", "long"],
|
||||||
"lib": ["es2020", "DOM"]
|
"lib": ["es2020", "DOM"]
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -4,12 +4,12 @@ import fetch from "node-fetch";
|
||||||
import { URLSearchParams } from "url";
|
import { URLSearchParams } from "url";
|
||||||
import { withDb, AppDatabase } from "../db";
|
import { withDb, AppDatabase } from "../db";
|
||||||
import { loadConfig } from "@digiresilience/metamigo-config";
|
import { loadConfig } from "@digiresilience/metamigo-config";
|
||||||
import { tagMap } from "../lib/tag-map"
|
import { tagMap } from "../lib/tag-map";
|
||||||
|
|
||||||
type FormattedZammadTicket = {
|
type FormattedZammadTicket = {
|
||||||
data: Record<string, unknown>,
|
data: Record<string, unknown>,
|
||||||
predictions: Record<string, unknown>[]
|
predictions: Record<string, unknown>[];
|
||||||
}
|
};
|
||||||
|
|
||||||
const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promise<[boolean, FormattedZammadTicket[]]> => {
|
const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promise<[boolean, FormattedZammadTicket[]]> => {
|
||||||
const { leafcutter: { zammadApiUrl, zammadApiKey, contributorName, contributorId } } = await loadConfig();
|
const { leafcutter: { zammadApiUrl, zammadApiKey, contributorName, contributorId } } = await loadConfig();
|
||||||
|
|
@ -27,8 +27,8 @@ const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promis
|
||||||
const rawTickets = await fetch(`${zammadApiUrl}/tickets/search?${ticketsQuery}`,
|
const rawTickets = await fetch(`${zammadApiUrl}/tickets/search?${ticketsQuery}`,
|
||||||
{ headers }
|
{ headers }
|
||||||
);
|
);
|
||||||
const tickets = await rawTickets.json();
|
const tickets: any = await rawTickets.json();
|
||||||
console.log({ tickets })
|
console.log({ tickets });
|
||||||
if (!tickets || tickets.length === 0) {
|
if (!tickets || tickets.length === 0) {
|
||||||
return [shouldContinue, docs];
|
return [shouldContinue, docs];
|
||||||
}
|
}
|
||||||
|
|
@ -50,7 +50,7 @@ const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promis
|
||||||
const rawArticles = await fetch(`${zammadApiUrl}/ticket_articles/by_ticket/${source_id}`,
|
const rawArticles = await fetch(`${zammadApiUrl}/ticket_articles/by_ticket/${source_id}`,
|
||||||
{ headers }
|
{ headers }
|
||||||
);
|
);
|
||||||
const articles = await rawArticles.json();
|
const articles: any = await rawArticles.json();
|
||||||
let articleText = "";
|
let articleText = "";
|
||||||
|
|
||||||
for (const article of articles) {
|
for (const article of articles) {
|
||||||
|
|
@ -70,7 +70,7 @@ const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promis
|
||||||
});
|
});
|
||||||
|
|
||||||
const rawTags = await fetch(`${zammadApiUrl}/tags?${tagsQuery}`, { headers });
|
const rawTags = await fetch(`${zammadApiUrl}/tags?${tagsQuery}`, { headers });
|
||||||
const { tags } = await rawTags.json();
|
const { tags }: any = await rawTags.json();
|
||||||
const transformedTags = [];
|
const transformedTags = [];
|
||||||
for (const tag of tags) {
|
for (const tag of tags) {
|
||||||
const outputs = tagMap[tag];
|
const outputs = tagMap[tag];
|
||||||
|
|
@ -106,14 +106,14 @@ const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promis
|
||||||
doc.predictions.push({
|
doc.predictions.push({
|
||||||
model_version: `${contributorName}TranslatorV1`,
|
model_version: `${contributorName}TranslatorV1`,
|
||||||
result,
|
result,
|
||||||
})
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
docs.push(doc);
|
docs.push(doc);
|
||||||
}
|
}
|
||||||
|
|
||||||
return [shouldContinue, docs];
|
return [shouldContinue, docs];
|
||||||
}
|
};
|
||||||
|
|
||||||
const fetchFromZammad = async (minUpdatedTimestamp: Date): Promise<FormattedZammadTicket[]> => {
|
const fetchFromZammad = async (minUpdatedTimestamp: Date): Promise<FormattedZammadTicket[]> => {
|
||||||
const pages = [...Array.from({ length: 10000 }).keys()];
|
const pages = [...Array.from({ length: 10000 }).keys()];
|
||||||
|
|
@ -153,7 +153,7 @@ const sendToLabelStudio = async (tickets: FormattedZammadTicket[]) => {
|
||||||
|
|
||||||
console.log(JSON.stringify(importResult, undefined, 2));
|
console.log(JSON.stringify(importResult, undefined, 2));
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
|
|
||||||
const importLabelStudioTask = async (): Promise<void> => {
|
const importLabelStudioTask = async (): Promise<void> => {
|
||||||
withDb(async (db: AppDatabase) => {
|
withDb(async (db: AppDatabase) => {
|
||||||
|
|
@ -167,8 +167,8 @@ const importLabelStudioTask = async (): Promise<void> => {
|
||||||
await sendToLabelStudio(tickets);
|
await sendToLabelStudio(tickets);
|
||||||
const lastTicket = tickets.pop();
|
const lastTicket = tickets.pop();
|
||||||
const newLastTimestamp = lastTicket.data.source_closed_at;
|
const newLastTimestamp = lastTicket.data.source_closed_at;
|
||||||
console.log({ newLastTimestamp })
|
console.log({ newLastTimestamp });
|
||||||
await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp })
|
await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -5,27 +5,27 @@ import { withDb, AppDatabase } from "../db";
|
||||||
import { loadConfig } from "@digiresilience/metamigo-config";
|
import { loadConfig } from "@digiresilience/metamigo-config";
|
||||||
|
|
||||||
type LabelStudioTicket = {
|
type LabelStudioTicket = {
|
||||||
id: string
|
id: string;
|
||||||
is_labeled: boolean
|
is_labeled: boolean;
|
||||||
annotations: Record<string, unknown>[]
|
annotations: Record<string, unknown>[];
|
||||||
data: Record<string, unknown>
|
data: Record<string, unknown>;
|
||||||
updated_at: string;
|
updated_at: string;
|
||||||
}
|
};
|
||||||
|
|
||||||
type LeafcutterTicket = {
|
type LeafcutterTicket = {
|
||||||
id: string
|
id: string;
|
||||||
incident: string[]
|
incident: string[];
|
||||||
technology: string[]
|
technology: string[];
|
||||||
targeted_group: string[]
|
targeted_group: string[];
|
||||||
country: string[]
|
country: string[];
|
||||||
region: string[]
|
region: string[];
|
||||||
continent: string[]
|
continent: string[];
|
||||||
date: Date
|
date: Date;
|
||||||
origin: string
|
origin: string;
|
||||||
origin_id: string
|
origin_id: string;
|
||||||
source_created_at: string
|
source_created_at: string;
|
||||||
source_updated_at: string
|
source_updated_at: string;
|
||||||
}
|
};
|
||||||
|
|
||||||
const getLabelStudioTickets = async (page: number): Promise<LabelStudioTicket[]> => {
|
const getLabelStudioTickets = async (page: number): Promise<LabelStudioTicket[]> => {
|
||||||
const {
|
const {
|
||||||
|
|
@ -43,15 +43,15 @@ const getLabelStudioTickets = async (page: number): Promise<LabelStudioTicket[]>
|
||||||
page_size: "50",
|
page_size: "50",
|
||||||
page: `${page}`,
|
page: `${page}`,
|
||||||
});
|
});
|
||||||
console.log({ url: `${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}` })
|
console.log({ url: `${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}` });
|
||||||
const res = await fetch(`${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}`,
|
const res = await fetch(`${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}`,
|
||||||
{ headers });
|
{ headers });
|
||||||
console.log({ res })
|
console.log({ res });
|
||||||
const tasksResult = await res.json();
|
const tasksResult: any = await res.json();
|
||||||
console.log({ tasksResult });
|
console.log({ tasksResult });
|
||||||
|
|
||||||
return tasksResult;
|
return tasksResult;
|
||||||
}
|
};
|
||||||
|
|
||||||
const fetchFromLabelStudio = async (minUpdatedTimestamp: Date): Promise<LabelStudioTicket[]> => {
|
const fetchFromLabelStudio = async (minUpdatedTimestamp: Date): Promise<LabelStudioTicket[]> => {
|
||||||
const pages = [...Array.from({ length: 10000 }).keys()];
|
const pages = [...Array.from({ length: 10000 }).keys()];
|
||||||
|
|
@ -59,15 +59,15 @@ const fetchFromLabelStudio = async (minUpdatedTimestamp: Date): Promise<LabelStu
|
||||||
|
|
||||||
for await (const page of pages) {
|
for await (const page of pages) {
|
||||||
const docs = await getLabelStudioTickets(page + 1);
|
const docs = await getLabelStudioTickets(page + 1);
|
||||||
console.log({ page, docs })
|
console.log({ page, docs });
|
||||||
|
|
||||||
if (docs && docs.length > 0) {
|
if (docs && docs.length > 0) {
|
||||||
for (const doc of docs) {
|
for (const doc of docs) {
|
||||||
const updatedAt = new Date(doc.updated_at);
|
const updatedAt = new Date(doc.updated_at);
|
||||||
console.log({ updatedAt, minUpdatedTimestamp });
|
console.log({ updatedAt, minUpdatedTimestamp });
|
||||||
if (updatedAt > minUpdatedTimestamp) {
|
if (updatedAt > minUpdatedTimestamp) {
|
||||||
console.log(`Adding doc`, { doc })
|
console.log(`Adding doc`, { doc });
|
||||||
allDocs.push(doc)
|
allDocs.push(doc);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -75,9 +75,9 @@ const fetchFromLabelStudio = async (minUpdatedTimestamp: Date): Promise<LabelStu
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log({ allDocs })
|
console.log({ allDocs });
|
||||||
return allDocs;
|
return allDocs;
|
||||||
}
|
};
|
||||||
|
|
||||||
const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
|
const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
|
||||||
const {
|
const {
|
||||||
|
|
@ -89,9 +89,9 @@ const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
|
||||||
}
|
}
|
||||||
} = await loadConfig();
|
} = await loadConfig();
|
||||||
|
|
||||||
console.log({ tickets })
|
console.log({ tickets });
|
||||||
const filteredTickets = tickets.filter((ticket) => ticket.is_labeled);
|
const filteredTickets = tickets.filter((ticket) => ticket.is_labeled);
|
||||||
console.log({ filteredTickets })
|
console.log({ filteredTickets });
|
||||||
const finalTickets: LeafcutterTicket[] = filteredTickets.map((ticket) => {
|
const finalTickets: LeafcutterTicket[] = filteredTickets.map((ticket) => {
|
||||||
const {
|
const {
|
||||||
id,
|
id,
|
||||||
|
|
@ -132,7 +132,7 @@ const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log("Sending to Leafcutter");
|
console.log("Sending to Leafcutter");
|
||||||
console.log({ finalTickets })
|
console.log({ finalTickets });
|
||||||
|
|
||||||
const result = await fetch(opensearchApiUrl, {
|
const result = await fetch(opensearchApiUrl, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
|
|
@ -155,9 +155,9 @@ const importLeafcutterTask = async (): Promise<void> => {
|
||||||
const newLastTimestamp = new Date();
|
const newLastTimestamp = new Date();
|
||||||
console.log({ contributorName, settingName, res, startTimestamp, newLastTimestamp });
|
console.log({ contributorName, settingName, res, startTimestamp, newLastTimestamp });
|
||||||
const tickets = await fetchFromLabelStudio(startTimestamp);
|
const tickets = await fetchFromLabelStudio(startTimestamp);
|
||||||
console.log({ tickets })
|
console.log({ tickets });
|
||||||
await sendToLeafcutter(tickets);
|
await sendToLeafcutter(tickets);
|
||||||
await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp })
|
await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp });
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,8 @@
|
||||||
"extends": "tsconfig-link",
|
"extends": "tsconfig-link",
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"outDir": "build/main",
|
"outDir": "build/main",
|
||||||
"esModuleInterop": true
|
"esModuleInterop": true,
|
||||||
|
"skipLibCheck": true
|
||||||
},
|
},
|
||||||
"include": ["**/*.ts", "**/.*.ts"],
|
"include": ["**/*.ts", "**/.*.ts"],
|
||||||
"exclude": ["node_modules", "build"]
|
"exclude": ["node_modules", "build"]
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
// @ts-nocheck
|
||||||
import type { AdapterInstance } from "next-auth/adapters";
|
import type { AdapterInstance } from "next-auth/adapters";
|
||||||
import type { NumberSchema, StringSchema, ObjectSchema } from "joi";
|
import type { NumberSchema, StringSchema, ObjectSchema } from "joi";
|
||||||
import type { Request } from "@hapi/hapi";
|
import type { Request } from "@hapi/hapi";
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@
|
||||||
"outDir": "build/main",
|
"outDir": "build/main",
|
||||||
"rootDir": "src",
|
"rootDir": "src",
|
||||||
"baseUrl": "./",
|
"baseUrl": "./",
|
||||||
|
"skipLibCheck": true,
|
||||||
"types": ["jest", "node"]
|
"types": ["jest", "node"]
|
||||||
},
|
},
|
||||||
"include": ["src/**/*.ts"],
|
"include": ["src/**/*.ts"],
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
// @ts-nocheck
|
||||||
import { writeFileSync } from "node:fs";
|
import { writeFileSync } from "node:fs";
|
||||||
import {
|
import {
|
||||||
getIntrospectionQuery,
|
getIntrospectionQuery,
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
/* eslint-disable unicorn/no-null,max-params */
|
/* eslint-disable unicorn/no-null,max-params */
|
||||||
|
// @ts-nocheck
|
||||||
import { createHash, randomBytes } from "node:crypto";
|
import { createHash, randomBytes } from "node:crypto";
|
||||||
import type { AdapterInstance } from "next-auth/adapters";
|
import type { AdapterInstance } from "next-auth/adapters";
|
||||||
import omit from "lodash/omit";
|
import omit from "lodash/omit";
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@
|
||||||
"outDir": "build/main",
|
"outDir": "build/main",
|
||||||
"rootDir": "src",
|
"rootDir": "src",
|
||||||
"baseUrl": "./",
|
"baseUrl": "./",
|
||||||
|
"skipLibCheck": true,
|
||||||
"types": ["jest", "node"]
|
"types": ["jest", "node"]
|
||||||
},
|
},
|
||||||
"include": ["src/**/*.ts"],
|
"include": ["src/**/*.ts"],
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue