Build fixes

This commit is contained in:
Darren Clarke 2023-05-25 09:27:26 +00:00
parent 70d97c1342
commit 04ecef98da
9 changed files with 52 additions and 45 deletions

View file

@ -3,6 +3,7 @@
"compilerOptions": { "compilerOptions": {
"outDir": "build/main", "outDir": "build/main",
"rootDir": "src", "rootDir": "src",
"skipLibCheck": true,
"types": ["jest", "node", "long"], "types": ["jest", "node", "long"],
"lib": ["es2020", "DOM"] "lib": ["es2020", "DOM"]
}, },

View file

@ -4,12 +4,12 @@ import fetch from "node-fetch";
import { URLSearchParams } from "url"; import { URLSearchParams } from "url";
import { withDb, AppDatabase } from "../db"; import { withDb, AppDatabase } from "../db";
import { loadConfig } from "@digiresilience/metamigo-config"; import { loadConfig } from "@digiresilience/metamigo-config";
import { tagMap } from "../lib/tag-map" import { tagMap } from "../lib/tag-map";
type FormattedZammadTicket = { type FormattedZammadTicket = {
data: Record<string, unknown>, data: Record<string, unknown>,
predictions: Record<string, unknown>[] predictions: Record<string, unknown>[];
} };
const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promise<[boolean, FormattedZammadTicket[]]> => { const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promise<[boolean, FormattedZammadTicket[]]> => {
const { leafcutter: { zammadApiUrl, zammadApiKey, contributorName, contributorId } } = await loadConfig(); const { leafcutter: { zammadApiUrl, zammadApiKey, contributorName, contributorId } } = await loadConfig();
@ -27,8 +27,8 @@ const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promis
const rawTickets = await fetch(`${zammadApiUrl}/tickets/search?${ticketsQuery}`, const rawTickets = await fetch(`${zammadApiUrl}/tickets/search?${ticketsQuery}`,
{ headers } { headers }
); );
const tickets = await rawTickets.json(); const tickets: any = await rawTickets.json();
console.log({ tickets }) console.log({ tickets });
if (!tickets || tickets.length === 0) { if (!tickets || tickets.length === 0) {
return [shouldContinue, docs]; return [shouldContinue, docs];
} }
@ -50,7 +50,7 @@ const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promis
const rawArticles = await fetch(`${zammadApiUrl}/ticket_articles/by_ticket/${source_id}`, const rawArticles = await fetch(`${zammadApiUrl}/ticket_articles/by_ticket/${source_id}`,
{ headers } { headers }
); );
const articles = await rawArticles.json(); const articles: any = await rawArticles.json();
let articleText = ""; let articleText = "";
for (const article of articles) { for (const article of articles) {
@ -70,7 +70,7 @@ const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promis
}); });
const rawTags = await fetch(`${zammadApiUrl}/tags?${tagsQuery}`, { headers }); const rawTags = await fetch(`${zammadApiUrl}/tags?${tagsQuery}`, { headers });
const { tags } = await rawTags.json(); const { tags }: any = await rawTags.json();
const transformedTags = []; const transformedTags = [];
for (const tag of tags) { for (const tag of tags) {
const outputs = tagMap[tag]; const outputs = tagMap[tag];
@ -106,14 +106,14 @@ const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promis
doc.predictions.push({ doc.predictions.push({
model_version: `${contributorName}TranslatorV1`, model_version: `${contributorName}TranslatorV1`,
result, result,
}) });
} }
docs.push(doc); docs.push(doc);
} }
return [shouldContinue, docs]; return [shouldContinue, docs];
} };
const fetchFromZammad = async (minUpdatedTimestamp: Date): Promise<FormattedZammadTicket[]> => { const fetchFromZammad = async (minUpdatedTimestamp: Date): Promise<FormattedZammadTicket[]> => {
const pages = [...Array.from({ length: 10000 }).keys()]; const pages = [...Array.from({ length: 10000 }).keys()];
@ -153,7 +153,7 @@ const sendToLabelStudio = async (tickets: FormattedZammadTicket[]) => {
console.log(JSON.stringify(importResult, undefined, 2)); console.log(JSON.stringify(importResult, undefined, 2));
} }
} };
const importLabelStudioTask = async (): Promise<void> => { const importLabelStudioTask = async (): Promise<void> => {
withDb(async (db: AppDatabase) => { withDb(async (db: AppDatabase) => {
@ -167,8 +167,8 @@ const importLabelStudioTask = async (): Promise<void> => {
await sendToLabelStudio(tickets); await sendToLabelStudio(tickets);
const lastTicket = tickets.pop(); const lastTicket = tickets.pop();
const newLastTimestamp = lastTicket.data.source_closed_at; const newLastTimestamp = lastTicket.data.source_closed_at;
console.log({ newLastTimestamp }) console.log({ newLastTimestamp });
await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp }) await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp });
} }
}); });
}; };

View file

@ -5,27 +5,27 @@ import { withDb, AppDatabase } from "../db";
import { loadConfig } from "@digiresilience/metamigo-config"; import { loadConfig } from "@digiresilience/metamigo-config";
type LabelStudioTicket = { type LabelStudioTicket = {
id: string id: string;
is_labeled: boolean is_labeled: boolean;
annotations: Record<string, unknown>[] annotations: Record<string, unknown>[];
data: Record<string, unknown> data: Record<string, unknown>;
updated_at: string; updated_at: string;
} };
type LeafcutterTicket = { type LeafcutterTicket = {
id: string id: string;
incident: string[] incident: string[];
technology: string[] technology: string[];
targeted_group: string[] targeted_group: string[];
country: string[] country: string[];
region: string[] region: string[];
continent: string[] continent: string[];
date: Date date: Date;
origin: string origin: string;
origin_id: string origin_id: string;
source_created_at: string source_created_at: string;
source_updated_at: string source_updated_at: string;
} };
const getLabelStudioTickets = async (page: number): Promise<LabelStudioTicket[]> => { const getLabelStudioTickets = async (page: number): Promise<LabelStudioTicket[]> => {
const { const {
@ -43,15 +43,15 @@ const getLabelStudioTickets = async (page: number): Promise<LabelStudioTicket[]>
page_size: "50", page_size: "50",
page: `${page}`, page: `${page}`,
}); });
console.log({ url: `${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}` }) console.log({ url: `${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}` });
const res = await fetch(`${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}`, const res = await fetch(`${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}`,
{ headers }); { headers });
console.log({ res }) console.log({ res });
const tasksResult = await res.json(); const tasksResult: any = await res.json();
console.log({ tasksResult }); console.log({ tasksResult });
return tasksResult; return tasksResult;
} };
const fetchFromLabelStudio = async (minUpdatedTimestamp: Date): Promise<LabelStudioTicket[]> => { const fetchFromLabelStudio = async (minUpdatedTimestamp: Date): Promise<LabelStudioTicket[]> => {
const pages = [...Array.from({ length: 10000 }).keys()]; const pages = [...Array.from({ length: 10000 }).keys()];
@ -59,15 +59,15 @@ const fetchFromLabelStudio = async (minUpdatedTimestamp: Date): Promise<LabelStu
for await (const page of pages) { for await (const page of pages) {
const docs = await getLabelStudioTickets(page + 1); const docs = await getLabelStudioTickets(page + 1);
console.log({ page, docs }) console.log({ page, docs });
if (docs && docs.length > 0) { if (docs && docs.length > 0) {
for (const doc of docs) { for (const doc of docs) {
const updatedAt = new Date(doc.updated_at); const updatedAt = new Date(doc.updated_at);
console.log({ updatedAt, minUpdatedTimestamp }); console.log({ updatedAt, minUpdatedTimestamp });
if (updatedAt > minUpdatedTimestamp) { if (updatedAt > minUpdatedTimestamp) {
console.log(`Adding doc`, { doc }) console.log(`Adding doc`, { doc });
allDocs.push(doc) allDocs.push(doc);
} }
} }
} else { } else {
@ -75,9 +75,9 @@ const fetchFromLabelStudio = async (minUpdatedTimestamp: Date): Promise<LabelStu
} }
} }
console.log({ allDocs }) console.log({ allDocs });
return allDocs; return allDocs;
} };
const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => { const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
const { const {
@ -89,9 +89,9 @@ const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
} }
} = await loadConfig(); } = await loadConfig();
console.log({ tickets }) console.log({ tickets });
const filteredTickets = tickets.filter((ticket) => ticket.is_labeled); const filteredTickets = tickets.filter((ticket) => ticket.is_labeled);
console.log({ filteredTickets }) console.log({ filteredTickets });
const finalTickets: LeafcutterTicket[] = filteredTickets.map((ticket) => { const finalTickets: LeafcutterTicket[] = filteredTickets.map((ticket) => {
const { const {
id, id,
@ -132,7 +132,7 @@ const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
}); });
console.log("Sending to Leafcutter"); console.log("Sending to Leafcutter");
console.log({ finalTickets }) console.log({ finalTickets });
const result = await fetch(opensearchApiUrl, { const result = await fetch(opensearchApiUrl, {
method: "POST", method: "POST",
@ -155,9 +155,9 @@ const importLeafcutterTask = async (): Promise<void> => {
const newLastTimestamp = new Date(); const newLastTimestamp = new Date();
console.log({ contributorName, settingName, res, startTimestamp, newLastTimestamp }); console.log({ contributorName, settingName, res, startTimestamp, newLastTimestamp });
const tickets = await fetchFromLabelStudio(startTimestamp); const tickets = await fetchFromLabelStudio(startTimestamp);
console.log({ tickets }) console.log({ tickets });
await sendToLeafcutter(tickets); await sendToLeafcutter(tickets);
await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp }) await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp });
}); });
}; };

View file

@ -2,7 +2,8 @@
"extends": "tsconfig-link", "extends": "tsconfig-link",
"compilerOptions": { "compilerOptions": {
"outDir": "build/main", "outDir": "build/main",
"esModuleInterop": true "esModuleInterop": true,
"skipLibCheck": true
}, },
"include": ["**/*.ts", "**/.*.ts"], "include": ["**/*.ts", "**/.*.ts"],
"exclude": ["node_modules", "build"] "exclude": ["node_modules", "build"]

View file

@ -1,3 +1,4 @@
// @ts-nocheck
import type { AdapterInstance } from "next-auth/adapters"; import type { AdapterInstance } from "next-auth/adapters";
import type { NumberSchema, StringSchema, ObjectSchema } from "joi"; import type { NumberSchema, StringSchema, ObjectSchema } from "joi";
import type { Request } from "@hapi/hapi"; import type { Request } from "@hapi/hapi";

View file

@ -5,6 +5,7 @@
"outDir": "build/main", "outDir": "build/main",
"rootDir": "src", "rootDir": "src",
"baseUrl": "./", "baseUrl": "./",
"skipLibCheck": true,
"types": ["jest", "node"] "types": ["jest", "node"]
}, },
"include": ["src/**/*.ts"], "include": ["src/**/*.ts"],

View file

@ -1,3 +1,4 @@
// @ts-nocheck
import { writeFileSync } from "node:fs"; import { writeFileSync } from "node:fs";
import { import {
getIntrospectionQuery, getIntrospectionQuery,

View file

@ -1,4 +1,5 @@
/* eslint-disable unicorn/no-null,max-params */ /* eslint-disable unicorn/no-null,max-params */
// @ts-nocheck
import { createHash, randomBytes } from "node:crypto"; import { createHash, randomBytes } from "node:crypto";
import type { AdapterInstance } from "next-auth/adapters"; import type { AdapterInstance } from "next-auth/adapters";
import omit from "lodash/omit"; import omit from "lodash/omit";

View file

@ -5,6 +5,7 @@
"outDir": "build/main", "outDir": "build/main",
"rootDir": "src", "rootDir": "src",
"baseUrl": "./", "baseUrl": "./",
"skipLibCheck": true,
"types": ["jest", "node"] "types": ["jest", "node"]
}, },
"include": ["src/**/*.ts"], "include": ["src/**/*.ts"],