Metamigo -> Bridge

This commit is contained in:
Darren Clarke 2024-04-21 09:44:30 +02:00
parent 242f3cf6b8
commit a445762a37
145 changed files with 396 additions and 16668 deletions

2
.gitignore vendored
View file

@ -19,7 +19,7 @@ docker-compose.yml
coverage
.pgpass
**/dist/**
.metamigo.local.json
.bridge.local.json
out/
signald-state/*
!./signald-state/.gitkeep

View file

@ -84,16 +84,16 @@ leafcutter-docker-release:
variables:
DOCKER_NS: ${CI_REGISTRY}/digiresilience/link/link-stack/leafcutter
metamigo-docker-build:
bridge-docker-build:
extends: .docker-build
variables:
DOCKER_NS: ${CI_REGISTRY}/digiresilience/link/link-stack/metamigo
DOCKERFILE_PATH: ./apps/metamigo-cli/Dockerfile
DOCKER_NS: ${CI_REGISTRY}/digiresilience/link/link-stack/bridge
DOCKERFILE_PATH: ./apps/bridge-cli/Dockerfile
metamigo-docker-release:
bridge-docker-release:
extends: .docker-release
variables:
DOCKER_NS: ${CI_REGISTRY}/digiresilience/link/link-stack/metamigo
DOCKER_NS: ${CI_REGISTRY}/digiresilience/link/link-stack/bridge
elasticsearch-docker-build:
extends: .docker-build

View file

@ -4,8 +4,8 @@
Local dev with docker-compose
* Create `link-stack/.env` from Bitwarden `.env for root of link-stack`
* Run local dev with docker-compose:
- Create `link-stack/.env` from Bitwarden `.env for root of link-stack`
- Run local dev with docker-compose:
```
git clone ...
cd link-stack
@ -14,23 +14,23 @@ Local dev with docker-compose
Or for local dev of a single app
* Create `link-stack/apps/link/.env.local` from Bitwarden `.env.local for link-stack/apps/link`
* Create `link-stack/apps/metamigo-frontend/.metamigo.local.json` from Bitwarden `.metamigo.local.json for link-stack/apps/metamigo/frontend`
* Build locally for development:
- Create `link-stack/apps/link/.env.local` from Bitwarden `.env.local for link-stack/apps/link`
- Create `link-stack/apps/bridge-frontend/.bridge.local.json` from Bitwarden `.bridge.local.json for link-stack/apps/bridge/frontend`
- Build locally for development:
```
npm install
make dev-metamigo # this starts the containers
make dev-bridge # this starts the containers
npm run migrate # this migrates the db
npm run dev:metamigo # this runs metamigo frontend and api
npm run dev:bridge # this runs bridge frontend and api
```
# TODO
- [ ] Delete old JWT config stuff
- [ ] Consolidate config
- [ ] Complete react-admin upgrade.. make all the metamigo-frontend stuff work
* https://marmelab.com/react-admin/Upgrade.html#no-more-prop-injection-in-page-components
- [ ] Get metamigo-worker working
- [ ] Complete react-admin upgrade.. make all the bridge-frontend stuff work
- https://marmelab.com/react-admin/Upgrade.html#no-more-prop-injection-in-page-components
- [ ] Get bridge-worker working
- [ ] Migrate off mui/styles
* https://mui.com/material-ui/migration/v5-style-changes/
* the codemods might help us?
- https://mui.com/material-ui/migration/v5-style-changes/
- the codemods might help us?

View file

@ -241,7 +241,7 @@ export const Sidebar: FC<SidebarProps> = ({ open, setOpen }) => {
fontFamily: poppins.style.fontFamily,
}}
>
Metamigo
CDR Bridge
</Typography>
</Grid>
)}

View file

Before

Width:  |  Height:  |  Size: 25 KiB

After

Width:  |  Height:  |  Size: 25 KiB

Before After
Before After

View file

@ -2,10 +2,12 @@ import type { Metadata } from "next";
import { InternalLayout } from "./_components/InternalLayout";
import { LicenseInfo } from "@mui/x-license";
LicenseInfo.setLicenseKey("7c9bf25d9e240f76e77cbf7d2ba58a23Tz02NjU4OCxFPTE3MTU4NjIzMzQ2ODgsUz1wcm8sTE09c3Vic2NyaXB0aW9uLEtWPTI=");
LicenseInfo.setLicenseKey(
"7c9bf25d9e240f76e77cbf7d2ba58a23Tz02NjU4OCxFPTE3MTU4NjIzMzQ2ODgsUz1wcm8sTE09c3Vic2NyaXB0aW9uLEtWPTI=",
);
export const metadata: Metadata = {
title: "Metamigo",
title: "CDR Bridge",
description: "",
};

View file

@ -1,5 +1,5 @@
{
"name": "metamigo-frontend",
"name": "bridge-frontend",
"version": "0.1.0",
"private": true,
"scripts": {
@ -9,7 +9,7 @@
"lint": "next lint"
},
"dependencies": {
"@auth/kysely-adapter": "^0.7.0",
"@auth/kysely-adapter": "^1.0.0",
"@emotion/cache": "^11.11.0",
"@emotion/react": "^11.11.4",
"@emotion/styled": "^11.11.5",
@ -17,14 +17,14 @@
"@mui/lab": "^5.0.0-alpha.170",
"@mui/material": "^5",
"@mui/material-nextjs": "^5.15.11",
"@mui/x-data-grid-pro": "^7.1.1",
"@mui/x-date-pickers-pro": "^7.1.1",
"@mui/x-license": "^7.1.1",
"@mui/x-data-grid-pro": "^7.3.0",
"@mui/x-date-pickers-pro": "^7.2.0",
"@mui/x-license": "^7.2.0",
"date-fns": "^3.6.0",
"kysely": "^0.26.1",
"material-ui-popup-state": "^5.1.0",
"mui-chips-input": "^2.1.4",
"next": "14.1.4",
"next": "14.2.2",
"next-auth": "^4.24.7",
"pg": "^8.11.5",
"react": "18.2.0",
@ -41,8 +41,8 @@
"@types/pg": "^8.11.5",
"@types/react": "^18",
"@types/react-dom": "^18",
"eslint": "^9",
"eslint-config-next": "14.1.4",
"eslint": "^8",
"eslint-config-next": "14.2.2",
"ts-config": "*",
"typescript": "^5"
}

View file

Before

Width:  |  Height:  |  Size: 4.6 KiB

After

Width:  |  Height:  |  Size: 4.6 KiB

Before After
Before After

View file

@ -0,0 +1,3 @@
{
"presets": ["@digiresilience/babel-preset-bridge"]
}

View file

@ -1,16 +1,16 @@
/* eslint-disable camelcase */
import { SavedVoiceProvider } from "@digiresilience/metamigo-db";
import { SavedVoiceProvider } from "@digiresilience/bridge-db";
import Twilio from "twilio";
import { CallInstance } from "twilio/lib/rest/api/v2010/account/call";
import { Zammad, getOrCreateUser } from "./zammad";
export const twilioClientFor = (
provider: SavedVoiceProvider
provider: SavedVoiceProvider,
): Twilio.Twilio => {
const { accountSid, apiKeySid, apiKeySecret } = provider.credentials;
if (!accountSid || !apiKeySid || !apiKeySecret)
throw new Error(
`twilio provider ${provider.name} does not have credentials`
`twilio provider ${provider.name} does not have credentials`,
);
return Twilio(apiKeySid, apiKeySecret, {
@ -20,7 +20,7 @@ export const twilioClientFor = (
export const createZammadTicket = async (
call: CallInstance,
mp3: Buffer
mp3: Buffer,
): Promise<void> => {
const title = `Call from ${call.fromFormatted} at ${call.startTime}`;
const body = `<ul>
@ -36,7 +36,7 @@ export const createZammadTicket = async (
{
token: "EviH_WL0p6YUlCoIER7noAZEAPsYA_fVU4FZCKdpq525Vmzzvl8d7dNuP_8d-Amb",
},
"https://demo.digiresilience.org"
"https://demo.digiresilience.org",
);
try {
const customer = await getOrCreateUser(zammad, call.fromFormatted);

View file

@ -1,12 +1,16 @@
import pgPromise from "pg-promise";
import * as pgMonitor from "pg-monitor";
import { dbInitOptions, IRepositories, AppDatabase } from "@digiresilience/metamigo-db";
import config from "@digiresilience/metamigo-config";
import {
dbInitOptions,
IRepositories,
AppDatabase,
} from "@digiresilience/bridge-db";
import config from "@digiresilience/bridge-config";
import type { IInitOptions } from "pg-promise";
export const initDiagnostics = (
logSql: boolean,
initOpts: IInitOptions<IRepositories>
initOpts: IInitOptions<IRepositories>,
): void => {
if (logSql) {
pgMonitor.attach(initOpts);
@ -44,4 +48,4 @@ export const withDb = <T>(f: (db: AppDatabase) => Promise<T>): Promise<T> => {
}
};
export type { AppDatabase } from "@digiresilience/metamigo-db";
export type { AppDatabase } from "@digiresilience/bridge-db";

View file

@ -1,6 +1,6 @@
import { defState } from "@digiresilience/montar";
import { configureLogger } from "@digiresilience/metamigo-common";
import config from "@digiresilience/metamigo-config";
import { configureLogger } from "@digiresilience/bridge-common";
import config from "@digiresilience/bridge-config";
export const logger = defState("workerLogger", {
start: async () => configureLogger(config),

View file

@ -1,5 +1,5 @@
{
"name": "metamigo-worker",
"name": "bridge-worker",
"version": "0.2.0",
"main": "build/main/index.js",
"type": "module",
@ -10,8 +10,8 @@
"html-to-text": "^9.0.5",
"node-fetch": "^3",
"pg-promise": "^11.6.0",
"remeda": "^1.57.2",
"twilio": "^5.0.3"
"remeda": "^1.60.1",
"twilio": "^5.0.4"
},
"devDependencies": {
"ts-config": "*",
@ -29,7 +29,7 @@
"prettier": "^3.2.5",
"ts-node": "^10.9.2",
"typedoc": "^0.25.13",
"typescript": "^5.4.4"
"typescript": "^5.4.5"
},
"nodemonConfig": {
"ignore": [

View file

@ -3,29 +3,35 @@ import { convert } from "html-to-text";
import fetch from "node-fetch";
import { URLSearchParams } from "url";
import { withDb, AppDatabase } from "../db";
import { loadConfig } from "@digiresilience/metamigo-config";
import { loadConfig } from "@digiresilience/bridge-config";
import { tagMap } from "../lib/tag-map";
type FormattedZammadTicket = {
data: Record<string, unknown>,
data: Record<string, unknown>;
predictions: Record<string, unknown>[];
};
const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promise<[boolean, FormattedZammadTicket[]]> => {
const { leafcutter: { zammadApiUrl, zammadApiKey, contributorName, contributorId } } = await loadConfig();
const getZammadTickets = async (
page: number,
minUpdatedTimestamp: Date,
): Promise<[boolean, FormattedZammadTicket[]]> => {
const {
leafcutter: { zammadApiUrl, zammadApiKey, contributorName, contributorId },
} = await loadConfig();
const headers = { Authorization: `Token ${zammadApiKey}` };
let shouldContinue = false;
const docs = [];
const ticketsQuery = new URLSearchParams({
"expand": "true",
"sort_by": "updated_at",
"order_by": "asc",
"query": "state.name: closed",
"per_page": "25",
"page": `${page}`,
expand: "true",
sort_by: "updated_at",
order_by: "asc",
query: "state.name: closed",
per_page: "25",
page: `${page}`,
});
const rawTickets = await fetch(`${zammadApiUrl}/tickets/search?${ticketsQuery}`,
{ headers }
const rawTickets = await fetch(
`${zammadApiUrl}/tickets/search?${ticketsQuery}`,
{ headers },
);
const tickets: any = await rawTickets.json();
console.log({ tickets });
@ -41,14 +47,25 @@ const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promis
shouldContinue = true;
if (source_closed_at <= minUpdatedTimestamp) {
console.log(`Skipping ticket`, { source_id, source_updated_at, source_closed_at, minUpdatedTimestamp });
console.log(`Skipping ticket`, {
source_id,
source_updated_at,
source_closed_at,
minUpdatedTimestamp,
});
continue;
}
console.log(`Processing ticket`, { source_id, source_updated_at, source_closed_at, minUpdatedTimestamp });
console.log(`Processing ticket`, {
source_id,
source_updated_at,
source_closed_at,
minUpdatedTimestamp,
});
const rawArticles = await fetch(`${zammadApiUrl}/ticket_articles/by_ticket/${source_id}`,
{ headers }
const rawArticles = await fetch(
`${zammadApiUrl}/ticket_articles/by_ticket/${source_id}`,
{ headers },
);
const articles: any = await rawArticles.json();
let articleText = "";
@ -69,7 +86,9 @@ const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promis
o_id: source_id,
});
const rawTags = await fetch(`${zammadApiUrl}/tags?${tagsQuery}`, { headers });
const rawTags = await fetch(`${zammadApiUrl}/tags?${tagsQuery}`, {
headers,
});
const { tags }: any = await rawTags.json();
const transformedTags = [];
for (const tag of tags) {
@ -88,7 +107,7 @@ const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promis
source_created_at,
source_updated_at,
},
predictions: []
predictions: [],
};
const result = transformedTags.map((tag) => {
@ -115,12 +134,17 @@ const getZammadTickets = async (page: number, minUpdatedTimestamp: Date): Promis
return [shouldContinue, docs];
};
const fetchFromZammad = async (minUpdatedTimestamp: Date): Promise<FormattedZammadTicket[]> => {
const fetchFromZammad = async (
minUpdatedTimestamp: Date,
): Promise<FormattedZammadTicket[]> => {
const pages = [...Array.from({ length: 10000 }).keys()];
const allTickets: FormattedZammadTicket[] = [];
for await (const page of pages) {
const [shouldContinue, tickets] = await getZammadTickets(page + 1, minUpdatedTimestamp);
const [shouldContinue, tickets] = await getZammadTickets(
page + 1,
minUpdatedTimestamp,
);
if (!shouldContinue) {
break;
@ -135,7 +159,9 @@ const fetchFromZammad = async (minUpdatedTimestamp: Date): Promise<FormattedZamm
};
const sendToLabelStudio = async (tickets: FormattedZammadTicket[]) => {
const { leafcutter: { labelStudioApiUrl, labelStudioApiKey } } = await loadConfig();
const {
leafcutter: { labelStudioApiUrl, labelStudioApiKey },
} = await loadConfig();
const headers = {
Authorization: `Token ${labelStudioApiKey}`,
@ -157,10 +183,14 @@ const sendToLabelStudio = async (tickets: FormattedZammadTicket[]) => {
const importLabelStudioTask = async (): Promise<void> => {
withDb(async (db: AppDatabase) => {
const { leafcutter: { contributorName } } = await loadConfig();
const {
leafcutter: { contributorName },
} = await loadConfig();
const settingName = `${contributorName}ImportLabelStudioTask`;
const res: any = await db.settings.findByName(settingName);
const startTimestamp = res?.value?.minUpdatedTimestamp ? new Date(res.value.minUpdatedTimestamp as string) : new Date("2023-03-01");
const startTimestamp = res?.value?.minUpdatedTimestamp
? new Date(res.value.minUpdatedTimestamp as string)
: new Date("2023-03-01");
const tickets = await fetchFromZammad(startTimestamp);
if (tickets.length > 0) {
@ -168,7 +198,9 @@ const importLabelStudioTask = async (): Promise<void> => {
const lastTicket = tickets.pop();
const newLastTimestamp = lastTicket.data.source_closed_at;
console.log({ newLastTimestamp });
await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp });
await db.settings.upsert(settingName, {
minUpdatedTimestamp: newLastTimestamp,
});
}
});
};

View file

@ -2,7 +2,7 @@
import fetch from "node-fetch";
import { URLSearchParams } from "url";
import { withDb, AppDatabase } from "../db";
import { loadConfig } from "@digiresilience/metamigo-config";
import { loadConfig } from "@digiresilience/bridge-config";
type LabelStudioTicket = {
id: string;
@ -27,12 +27,11 @@ type LeafcutterTicket = {
source_updated_at: string;
};
const getLabelStudioTickets = async (page: number): Promise<LabelStudioTicket[]> => {
const getLabelStudioTickets = async (
page: number,
): Promise<LabelStudioTicket[]> => {
const {
leafcutter: {
labelStudioApiUrl,
labelStudioApiKey,
}
leafcutter: { labelStudioApiUrl, labelStudioApiKey },
} = await loadConfig();
const headers = {
@ -44,8 +43,10 @@ const getLabelStudioTickets = async (page: number): Promise<LabelStudioTicket[]>
page: `${page}`,
});
console.log({ url: `${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}` });
const res = await fetch(`${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}`,
{ headers });
const res = await fetch(
`${labelStudioApiUrl}/projects/1/tasks?${ticketsQuery}`,
{ headers },
);
console.log({ res });
const tasksResult: any = await res.json();
console.log({ tasksResult });
@ -53,7 +54,9 @@ const getLabelStudioTickets = async (page: number): Promise<LabelStudioTicket[]>
return tasksResult;
};
const fetchFromLabelStudio = async (minUpdatedTimestamp: Date): Promise<LabelStudioTicket[]> => {
const fetchFromLabelStudio = async (
minUpdatedTimestamp: Date,
): Promise<LabelStudioTicket[]> => {
const pages = [...Array.from({ length: 10000 }).keys()];
const allDocs: LabelStudioTicket[] = [];
@ -85,8 +88,8 @@ const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
contributorId,
opensearchApiUrl,
opensearchUsername,
opensearchPassword
}
opensearchPassword,
},
} = await loadConfig();
console.log({ tickets });
@ -96,11 +99,7 @@ const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
const {
id,
annotations,
data: {
source_id,
source_created_at,
source_updated_at
}
data: { source_id, source_created_at, source_updated_at },
} = ticket;
const getTags = (tags: Record<string, any>[], name: string) =>
@ -127,7 +126,7 @@ const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
origin: contributorId,
origin_id: source_id as string,
source_created_at: source_created_at as string,
source_updated_at: source_updated_at as string
source_updated_at: source_updated_at as string,
};
});
@ -145,19 +144,30 @@ const sendToLeafcutter = async (tickets: LabelStudioTicket[]) => {
console.log({ result });
};
const importLeafcutterTask = async (): Promise<void> => {
withDb(async (db: AppDatabase) => {
const { leafcutter: { contributorName } } = await loadConfig();
const {
leafcutter: { contributorName },
} = await loadConfig();
const settingName = `${contributorName}ImportLeafcutterTask`;
const res: any = await db.settings.findByName(settingName);
const startTimestamp = res?.value?.minUpdatedTimestamp ? new Date(res.value.minUpdatedTimestamp as string) : new Date("2023-03-01");
const startTimestamp = res?.value?.minUpdatedTimestamp
? new Date(res.value.minUpdatedTimestamp as string)
: new Date("2023-03-01");
const newLastTimestamp = new Date();
console.log({ contributorName, settingName, res, startTimestamp, newLastTimestamp });
console.log({
contributorName,
settingName,
res,
startTimestamp,
newLastTimestamp,
});
const tickets = await fetchFromLabelStudio(startTimestamp);
console.log({ tickets });
await sendToLeafcutter(tickets);
await db.settings.upsert(settingName, { minUpdatedTimestamp: newLastTimestamp });
await db.settings.upsert(settingName, {
minUpdatedTimestamp: newLastTimestamp,
});
});
};

View file

@ -1,5 +1,5 @@
import Twilio from "twilio";
import config from "@digiresilience/metamigo-config";
import config from "@digiresilience/bridge-config";
import { withDb, AppDatabase } from "../db";
interface VoiceLineDeleteTaskOptions {
@ -9,7 +9,7 @@ interface VoiceLineDeleteTaskOptions {
}
const voiceLineDeleteTask = async (
payload: VoiceLineDeleteTaskOptions
payload: VoiceLineDeleteTaskOptions,
): Promise<void> =>
withDb(async (db: AppDatabase) => {
const { voiceLineId, providerId, providerLineSid } = payload;
@ -19,7 +19,7 @@ const voiceLineDeleteTask = async (
const { accountSid, apiKeySid, apiKeySecret } = provider.credentials;
if (!accountSid || !apiKeySid || !apiKeySecret)
throw new Error(
`twilio provider ${provider.name} does not have credentials`
`twilio provider ${provider.name} does not have credentials`,
);
const client = Twilio(apiKeySid, apiKeySecret, {

View file

@ -1,5 +1,5 @@
import Twilio from "twilio";
import config from "@digiresilience/metamigo-config";
import config from "@digiresilience/bridge-config";
import { withDb, AppDatabase } from "../db";
interface VoiceLineUpdateTaskOptions {
@ -7,7 +7,7 @@ interface VoiceLineUpdateTaskOptions {
}
const voiceLineUpdateTask = async (
payload: VoiceLineUpdateTaskOptions
payload: VoiceLineUpdateTaskOptions,
): Promise<void> =>
withDb(async (db: AppDatabase) => {
const { voiceLineId } = payload;
@ -22,7 +22,7 @@ const voiceLineUpdateTask = async (
const { accountSid, apiKeySid, apiKeySecret } = provider.credentials;
if (!accountSid || !apiKeySid || !apiKeySecret)
throw new Error(
`twilio provider ${provider.name} does not have credentials`
`twilio provider ${provider.name} does not have credentials`,
);
const client = Twilio(apiKeySid, apiKeySecret, {

View file

@ -1,6 +1,6 @@
import * as Worker from "graphile-worker";
import { defState } from "@digiresilience/montar";
import config from "@digiresilience/metamigo-config";
import config from "@digiresilience/bridge-config";
const startWorkerUtils = async (): Promise<Worker.WorkerUtils> => {
const workerUtils = await Worker.makeWorkerUtils({

View file

@ -20,15 +20,15 @@
"@mui/icons-material": "^5",
"@mui/lab": "^5.0.0-alpha.170",
"@mui/material": "^5",
"@mui/x-data-grid-pro": "^7.1.1",
"@mui/x-date-pickers-pro": "^7.1.1",
"@opensearch-project/opensearch": "^2.6.0",
"@mui/x-data-grid-pro": "^7.3.0",
"@mui/x-date-pickers-pro": "^7.2.0",
"@opensearch-project/opensearch": "^2.7.0",
"cryptr": "^6.3.0",
"date-fns": "^3.6.0",
"http-proxy-middleware": "^3.0.0",
"leafcutter-ui": "*",
"material-ui-popup-state": "^5.1.0",
"next": "14.1.4",
"next": "14.2.2",
"next-auth": "^4.24.7",
"next-http-proxy-middleware": "^1.2.6",
"opensearch-common": "*",
@ -47,17 +47,17 @@
},
"devDependencies": {
"@babel/core": "^7.24.4",
"@types/node": "^20.12.6",
"@types/react": "18.2.75",
"@types/node": "^20.12.7",
"@types/react": "18.2.79",
"@types/uuid": "^9.0.8",
"babel-loader": "^9.1.3",
"eslint": "^9.0.0",
"eslint-config-next": "^14.1.4",
"eslint": "^8.0.0",
"eslint-config-next": "^14.2.2",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-jsx-a11y": "^6.8.0",
"eslint-plugin-prettier": "^5.1.3",
"eslint-plugin-react": "^7.34.1",
"typescript": "5.4.4"
"typescript": "5.4.5"
}
}

View file

@ -53,7 +53,8 @@ const MenuItem = ({
}: any) => {
const { roboto } = fonts;
return (<Link href={href} target={target}>
return (
<Link href={href} target={target}>
<ListItemButton
sx={{
p: 0,
@ -160,7 +161,7 @@ const MenuItem = ({
</ListItemButton>
</Link>
);
}
};
interface SidebarProps {
open: boolean;
@ -577,13 +578,13 @@ export const Sidebar: FC<SidebarProps> = ({ open, setOpen }) => {
selected={pathname.endsWith("/admin/zammad")}
open={open}
/>
{false && roles.includes("metamigo") && (
{false && roles.includes("bridge") && (
<MenuItem
name="Metamigo"
href="/admin/metamigo"
name="Bridge"
href="/admin/bridge"
Icon={FeaturedPlayListIcon}
iconSize={0}
selected={pathname.endsWith("/admin/metamigo")}
selected={pathname.endsWith("/admin/bridge")}
open={open}
/>
)}

View file

@ -4,10 +4,10 @@ const nextConfig = {
experimental: {
missingSuspenseWithCSRBailout: false,
},
transpilePackages: ["leafcutter-ui", "metamigo-ui", "opensearch-common", "ui"],
transpilePackages: ["leafcutter-ui", "bridge-ui", "opensearch-common", "ui"],
publicRuntimeConfig: {
linkURL: process.env.LINK_URL ?? "http://localhost:3000",
metamigoURL: process.env.METAMIGO_URL ?? "http://localhost:8002",
bridgeURL: process.env.BRIDGE_URL ?? "http://localhost:8002",
labelStudioURL: process.env.LABEL_STUDIO_URL ?? "http://localhost:8006",
muiLicenseKey: process.env.MUI_LICENSE_KEY ?? "",
},

View file

@ -18,15 +18,15 @@
"@mui/icons-material": "^5",
"@mui/lab": "^5.0.0-alpha.170",
"@mui/material": "^5",
"@mui/x-data-grid-pro": "^7.1.1",
"@mui/x-date-pickers-pro": "^7.1.1",
"@mui/x-data-grid-pro": "^7.3.0",
"@mui/x-date-pickers-pro": "^7.2.0",
"date-fns": "^3.6.0",
"graphql-request": "^6.1.0",
"leafcutter-ui": "*",
"material-ui-popup-state": "^5.1.0",
"metamigo-ui": "*",
"bridge-ui": "*",
"mui-chips-input": "^2.1.4",
"next": "14.1.4",
"next": "14.2.2",
"next-auth": "^4.24.7",
"opensearch-common": "*",
"react": "18.2.0",
@ -42,17 +42,17 @@
},
"devDependencies": {
"@babel/core": "^7.24.4",
"@types/node": "^20.12.6",
"@types/react": "18.2.75",
"@types/node": "^20.12.7",
"@types/react": "18.2.79",
"@types/uuid": "^9.0.8",
"babel-loader": "^9.1.3",
"eslint": "^9.0.0",
"eslint-config-next": "^14.1.4",
"eslint": "^8.0.0",
"eslint-config-next": "^14.2.2",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-jsx-a11y": "^6.8.0",
"eslint-plugin-prettier": "^5.1.3",
"eslint-plugin-react": "^7.34.1",
"typescript": "5.4.4"
"typescript": "5.4.5"
}
}

View file

@ -1,2 +0,0 @@
@digiresilience:registry=https://gitlab.com/api/v4/packages/npm/
@guardianproject-ops:registry=https://gitlab.com/api/v4/packages/npm/

View file

@ -1,5 +0,0 @@
{
"presets": [
"@digiresilience/babel-preset-metamigo"
]
}

View file

@ -1,7 +1,7 @@
#!/bin/bash
set -e
echo "Creating the Metamigo database and the roles"
echo "Creating the Bridge database and the roles"
# We're using 'template1' because we know it should exist. We should not actually change this database.
psql -Xv ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname template1 <<EOF
CREATE ROLE ${DATABASE_OWNER} WITH LOGIN PASSWORD '${DATABASE_PASSWORD}';
@ -32,4 +32,3 @@ CREATE ROLE app_admin WITH IN ROLE app_user;
GRANT app_anonymous TO ${DATABASE_AUTHENTICATOR};
GRANT app_admin TO ${DATABASE_AUTHENTICATOR};
EOF

View file

@ -1,4 +1,4 @@
echo "Creating Metamigo admin user"
echo "Creating Bridge admin user"
psql -Xv ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$DATABASE_NAME" <<EOF
INSERT INTO app_public.users(email, name, user_role, is_active, created_by)
VALUES('$GITLAB_EMAIL_ADDRESS', 'Admin', 'admin'::app_public.role_type, true, '')

49
docker/compose/bridge.yml Normal file
View file

@ -0,0 +1,49 @@
version: "3.4"
x-global-vars: &common-global-variables
TZ: Etc/UTC
x-bridge-vars: &common-bridge-variables
DATABASE_HOST: "bridge-postgresql"
DATABASE_NAME: "bridge"
DATABASE_ROOT_OWNER: "root"
DATABASE_ROOT_PASSWORD: ${BRIDGE_DATABASE_ROOT_PASSWORD}
DATABASE_OWNER: "bridge"
DATABASE_PASSWORD: ${BRIDGE_DATABASE_PASSWORD}
DATABASE_VISITOR: "app_visitor"
DATABASE_AUTHENTICATOR: "app_graphile_auth"
DATABASE_AUTHENTICATOR_PASSWORD: ${BRIDGE_DATABASE_AUTHENTICATOR_PASSWORD}
DATABASE_URL: "postgresql://bridge:${BRIDGE_DATABASE_PASSWORD}@bridge-postgresql/bridge"
WORKER_DATABASE_URL: "postgresql://bridge:${BRIDGE_DATABASE_PASSWORD}@bridge-postgresql/bridge"
SHADOW_DATABASE_URL: "postgresql://bridge:${BRIDGE_DATABASE_PASSWORD}@bridge-postgresql/bridge_shadow"
ROOT_DATABASE_URL: "postgresql://bridge:${BRIDGE_DATABASE_PASSWORD}@bridge-postgresql/template1"
APP_ROOT_DATABASE_URL: "postgresql://root:${BRIDGE_DATABASE_ROOT_PASSWORD}@bridge-postgresql/bridge"
DATABASE_AUTH_URL: "postgresql://app_graphile_auth:${BRIDGE_DATABASE_AUTHENTICATOR_PASSWORD}@bridge-postgresql/bridge"
CORS_ALLOWED_ORIGINS: "https://bridge-api,${BRIDGE_DOMAIN},http://localhost:3000,http://127.0.0.1:3000"
FRONTEND_URL: ${BRIDGE_DOMAIN}
API_URL: "http://bridge-api:3001"
NEXTAUTH_URL: ${BRIDGE_DOMAIN}
NEXTAUTH_SECRET: ${NEXTAUTH_SECRET}
NEXTAUTH_AUDIENCE: ${NEXTAUTH_AUDIENCE}
NEXTAUTH_SIGNING_KEY_B64: ${NEXTAUTH_SIGNING_KEY_B64}
NEXTAUTH_ENCRYPTION_KEY_B64: ${NEXTAUTH_ENCRYPTION_KEY_B64}
GITLAB_EMAIL_ADDRESS: ${GITLAB_EMAIL_ADDRESS}
GITLAB_ID: ${GITLAB_ID}
GITLAB_SECRET: ${GITLAB_SECRET}
SIGNALD_ENABLED: "true"
SIGNALD_SOCKET: /signald/signald.sock
services:
bridge-frontend:
build: ../../apps/bridge-frontend
container_name: bridge-frontend
restart: ${RESTART}
ports:
- 8006:3000
environment: *common-bridge-variables
bridge-worker:
build: ../../apps/bridge-worker
container_name: bridge-worker
restart: ${RESTART}
environment: *common-bridge-variables

View file

@ -17,7 +17,7 @@ services:
ZAMMAD_VIRUAL_HOST: ${ZAMMAD_VIRTUAL_HOST}
LINK_URL: http://localhost:3000
LEAFCUTTER_URL: https://lc.digiresilience.org
METAMIGO_URL: http://metamigo-frontend:3000
BRIDGE_URL: http://bridge-frontend:3000
ZAMMAD_URL: http://zammad-nginx:8080
NEXTAUTH_URL: ${LINK_URL}
NEXTAUTH_SECRET: ${NEXTAUTH_SECRET}

View file

@ -1,49 +0,0 @@
version: "3.4"
x-global-vars: &common-global-variables
TZ: Etc/UTC
x-metamigo-vars: &common-metamigo-variables
DATABASE_HOST: "metamigo-postgresql"
DATABASE_NAME: "metamigo"
DATABASE_ROOT_OWNER: "root"
DATABASE_ROOT_PASSWORD: ${METAMIGO_DATABASE_ROOT_PASSWORD}
DATABASE_OWNER: "metamigo"
DATABASE_PASSWORD: ${METAMIGO_DATABASE_PASSWORD}
DATABASE_VISITOR: "app_visitor"
DATABASE_AUTHENTICATOR: "app_graphile_auth"
DATABASE_AUTHENTICATOR_PASSWORD: ${METAMIGO_DATABASE_AUTHENTICATOR_PASSWORD}
DATABASE_URL: "postgresql://metamigo:${METAMIGO_DATABASE_PASSWORD}@metamigo-postgresql/metamigo"
WORKER_DATABASE_URL: "postgresql://metamigo:${METAMIGO_DATABASE_PASSWORD}@metamigo-postgresql/metamigo"
SHADOW_DATABASE_URL: "postgresql://metamigo:${METAMIGO_DATABASE_PASSWORD}@metamigo-postgresql/metamigo_shadow"
ROOT_DATABASE_URL: "postgresql://metamigo:${METAMIGO_DATABASE_PASSWORD}@metamigo-postgresql/template1"
APP_ROOT_DATABASE_URL: "postgresql://root:${METAMIGO_DATABASE_ROOT_PASSWORD}@metamigo-postgresql/metamigo"
DATABASE_AUTH_URL: "postgresql://app_graphile_auth:${METAMIGO_DATABASE_AUTHENTICATOR_PASSWORD}@metamigo-postgresql/metamigo"
CORS_ALLOWED_ORIGINS: "https://metamigo-api,${METAMIGO_DOMAIN},http://localhost:3000,http://127.0.0.1:3000"
FRONTEND_URL: ${METAMIGO_DOMAIN}
API_URL: "http://metamigo-api:3001"
NEXTAUTH_URL: ${METAMIGO_DOMAIN}
NEXTAUTH_SECRET: ${NEXTAUTH_SECRET}
NEXTAUTH_AUDIENCE: ${NEXTAUTH_AUDIENCE}
NEXTAUTH_SIGNING_KEY_B64: ${NEXTAUTH_SIGNING_KEY_B64}
NEXTAUTH_ENCRYPTION_KEY_B64: ${NEXTAUTH_ENCRYPTION_KEY_B64}
GITLAB_EMAIL_ADDRESS: ${GITLAB_EMAIL_ADDRESS}
GITLAB_ID: ${GITLAB_ID}
GITLAB_SECRET: ${GITLAB_SECRET}
SIGNALD_ENABLED: "true"
SIGNALD_SOCKET: /signald/signald.sock
services:
metamigo-frontend:
build: ../../apps/metamigo-frontend
container_name: metamigo-frontend
restart: ${RESTART}
ports:
- 8006:3000
environment: *common-metamigo-variables
metamigo-worker:
build: ../../apps/metamigo-worker
container_name: metamigo-worker
restart: ${RESTART}
environment: *common-metamigo-variables

View file

@ -17,26 +17,26 @@ x-zammad-vars: &common-zammad-variables
ELASTICSEARCH_SSL_VERIFY: false # this doesn't set es_ssl_verify as expected, but ideally it would
ELASTICSEARCH_SCHEMA: "https"
x-metamigo-vars: &common-metamigo-variables
x-bridge-vars: &common-bridge-variables
DATABASE_HOST: "postgresql"
DATABASE_NAME: "metamigo"
DATABASE_NAME: "bridge"
DATABASE_ROOT_OWNER: "root"
DATABASE_ROOT_PASSWORD: ${METAMIGO_DATABASE_ROOT_PASSWORD}
DATABASE_OWNER: "metamigo"
DATABASE_PASSWORD: ${METAMIGO_DATABASE_PASSWORD}
DATABASE_ROOT_PASSWORD: ${BRIDGE_DATABASE_ROOT_PASSWORD}
DATABASE_OWNER: "bridge"
DATABASE_PASSWORD: ${BRIDGE_DATABASE_PASSWORD}
DATABASE_VISITOR: "app_visitor"
DATABASE_AUTHENTICATOR: "app_graphile_auth"
DATABASE_AUTHENTICATOR_PASSWORD: ${METAMIGO_DATABASE_AUTHENTICATOR_PASSWORD}
DATABASE_URL: "postgresql://metamigo:${METAMIGO_DATABASE_PASSWORD}@metamigo-postgresql/metamigo"
WORKER_DATABASE_URL: "postgresql://metamigo:${METAMIGO_DATABASE_PASSWORD}@metamigo-postgresql/metamigo"
SHADOW_DATABASE_URL: "postgresql://metamigo:${METAMIGO_DATABASE_PASSWORD}@metamigo-postgresql/metamigo_shadow"
ROOT_DATABASE_URL: "postgresql://metamigo:${METAMIGO_DATABASE_PASSWORD}@metamigo-postgresql/template1"
APP_ROOT_DATABASE_URL: "postgresql://root:${METAMIGO_DATABASE_ROOT_PASSWORD}@metamigo-postgresql/metamigo"
DATABASE_AUTH_URL: "postgresql://app_graphile_auth:${METAMIGO_DATABASE_AUTHENTICATOR_PASSWORD}@metamigo-postgresql/metamigo"
CORS_ALLOWED_ORIGINS: "https://metamigo-api,${METAMIGO_DOMAIN},http://localhost:3000,http://127.0.0.1:3000"
FRONTEND_URL: ${METAMIGO_DOMAIN}
API_URL: "http://metamigo-api:3001"
NEXTAUTH_URL: ${METAMIGO_DOMAIN}
DATABASE_AUTHENTICATOR_PASSWORD: ${BRIDGE_DATABASE_AUTHENTICATOR_PASSWORD}
DATABASE_URL: "postgresql://bridge:${BRIDGE_DATABASE_PASSWORD}@bridge-postgresql/bridge"
WORKER_DATABASE_URL: "postgresql://bridge:${BRIDGE_DATABASE_PASSWORD}@bridge-postgresql/bridge"
SHADOW_DATABASE_URL: "postgresql://bridge:${BRIDGE_DATABASE_PASSWORD}@bridge-postgresql/bridge_shadow"
ROOT_DATABASE_URL: "postgresql://bridge:${BRIDGE_DATABASE_PASSWORD}@bridge-postgresql/template1"
APP_ROOT_DATABASE_URL: "postgresql://root:${BRIDGE_DATABASE_ROOT_PASSWORD}@bridge-postgresql/bridge"
DATABASE_AUTH_URL: "postgresql://app_graphile_auth:${BRIDGE_DATABASE_AUTHENTICATOR_PASSWORD}@bridge-postgresql/bridge"
CORS_ALLOWED_ORIGINS: "https://bridge-api,${BRIDGE_DOMAIN},http://localhost:3000,http://127.0.0.1:3000"
FRONTEND_URL: ${BRIDGE_DOMAIN}
API_URL: "http://bridge-api:3001"
NEXTAUTH_URL: ${BRIDGE_DOMAIN}
NEXTAUTH_SECRET: ${NEXTAUTH_SECRET}
NEXTAUTH_AUDIENCE: ${NEXTAUTH_AUDIENCE}
NEXTAUTH_SIGNING_KEY_B64: ${NEXTAUTH_SIGNING_KEY_B64}
@ -55,7 +55,7 @@ services:
[
*common-global-variables,
*common-zammad-variables,
*common-metamigo-variables,
*common-bridge-variables,
]
POSTGRES_USER: zammad
POSTGRES_PASSWORD: ${ZAMMAD_DATABASE_PASSWORD}
@ -68,13 +68,13 @@ services:
- postgresql-data:/var/lib/postgresql/data
# volumes:
# - metamigo-data:/var/lib/postgresql/data
# - ./scripts/bootstrap-metamigo.sh:/docker-entrypoint-initdb.d/bootstrap-metamigo.sh
# - bridge-data:/var/lib/postgresql/data
# - ./scripts/bootstrap-bridge.sh:/docker-entrypoint-initdb.d/bootstrap-bridge.sh
#environment:
# <<: *common-metamigo-variables
# POSTGRES_PASSWORD: ${METAMIGO_DATABASE_ROOT_PASSWORD}
# <<: *common-bridge-variables
# POSTGRES_PASSWORD: ${BRIDGE_DATABASE_ROOT_PASSWORD}
# POSTGRES_USER: "root"
# POSTGRES_DB: "metamigo"
# POSTGRES_DB: "bridge"
volumes:
postgresql-data:

View file

@ -4,14 +4,14 @@ const app = process.argv[2];
const command = process.argv[3];
const files = {
all: ["zammad", "postgresql", "metamigo", "opensearch", "leafcutter", "link"],
all: ["zammad", "postgresql", "bridge", "opensearch", "leafcutter", "link"],
linkDev: ["zammad", "postgresql", "opensearch"],
link: ["zammad", "postgresql", "opensearch", "link"],
leafcutterDev: ["opensearch"],
leafcutter: ["opensearch", "leafcutter"],
opensearch: ["opensearch"],
metamigoDev: ["zammad", "postgresql"],
metamigo: ["zammad", "postgresql", "metamigo"],
bridgeDev: ["zammad", "postgresql"],
bridge: ["zammad", "postgresql", "bridge"],
zammad: ["zammad", "postgresql", "opensearch"],
};

View file

@ -1,7 +1,7 @@
# frozen_string_literal: true
# uninstall
package_names = %w[Hardening Leafcutter Metamigo]
package_names = %w[Hardening Leafcutter Bridge]
package_names.each do |name|
puts "Attempting to uninstall #{name} package..."

16316
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -9,7 +9,7 @@
"fmt": "turbo run fmt",
"upgrade:setup": "npm i -g npm-check-updates",
"upgrade:check": "ncu && ncu -ws",
"upgrade:all": "ncu -u && ncu -ws -u && npm i",
"upgrade:all": "ncu -u -x eslint && ncu -ws -u -x eslint && npm i",
"clean": "rm -f package-lock.json && rm -rf node_modules && rm -rf apps/*/node_modules && rm -rf packages/*/node_modules && rm -rf apps/*/.next && rm -rf packages/*/.turbo && rm -rf apps/*/.turbo && rm -rf docker/zammad/addons/*",
"docker:all:up": "node docker/scripts/docker.js all up",
"docker:all:down": "node docker/scripts/docker.js all down",
@ -30,11 +30,11 @@
"docker:zammad:up": "node docker/scripts/docker.js zammad up",
"docker:zammad:down": "node docker/scripts/docker.js zammad down",
"docker:zammad:build": "node docker/scripts/docker.js zammad build",
"docker:metamigo:dev:up": "node docker/scripts/docker.js metamigoDev up",
"docker:metamigo:dev:down": "node docker/scripts/docker.js metamigoDev down",
"docker:metamigo:up": "node docker/scripts/docker.js metamigo up",
"docker:metamigo:down": "node docker/scripts/docker.js metamigo down",
"docker:metamigo:build": "node docker/scripts/docker.js metamigo build"
"docker:bridge:dev:up": "node docker/scripts/docker.js bridgeDev up",
"docker:bridge:dev:down": "node docker/scripts/docker.js bridgeDev down",
"docker:bridge:up": "node docker/scripts/docker.js bridge up",
"docker:bridge:down": "node docker/scripts/docker.js bridge down",
"docker:bridge:build": "node docker/scripts/docker.js bridge build"
},
"workspaces": [
"apps/*",

View file

@ -1,5 +1,5 @@
{
"name": "metamigo-ui",
"name": "bridge-ui",
"version": "1.0.0",
"description": "",
"main": "index.js",

View file

@ -9,12 +9,12 @@
"fmt": "prettier \"profile/**/*.js\" --write"
},
"dependencies": {
"@rushstack/eslint-patch": "^1.10.1",
"@typescript-eslint/eslint-plugin": "^7.6.0",
"@typescript-eslint/parser": "^7.6.0",
"@rushstack/eslint-patch": "^1.10.2",
"@typescript-eslint/eslint-plugin": "^7.7.0",
"@typescript-eslint/parser": "^7.7.0",
"eslint-config-prettier": "^9.1.0",
"eslint-config-xo-space": "^0.35.0",
"eslint-plugin-cypress": "^2.15.1",
"eslint-plugin-cypress": "^2.15.2",
"eslint-plugin-eslint-comments": "^3.2.0",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-jest": "^28.2.0",
@ -24,12 +24,12 @@
"@babel/eslint-parser": "7.24.1"
},
"peerDependencies": {
"eslint": "^7.32.0",
"eslint": "^8",
"typescript": "^4.9.5"
},
"devDependencies": {
"eslint": "^9.0.0",
"eslint": "^8",
"jest": "^29.7.0",
"typescript": "^5.4.4"
"typescript": "^5.4.5"
}
}

View file

@ -12,12 +12,12 @@
"@mui/icons-material": "^5",
"@mui/lab": "^5.0.0-alpha.170",
"@mui/material": "^5",
"@mui/x-data-grid-pro": "^7.1.1",
"@mui/x-date-pickers-pro": "^7.1.1",
"@mui/x-data-grid-pro": "^7.3.0",
"@mui/x-date-pickers-pro": "^7.2.0",
"opensearch-common": "*",
"date-fns": "^3.6.0",
"material-ui-popup-state": "^5.1.0",
"next": "14.1.4",
"next": "14.2.2",
"react": "18.2.0",
"react-cookie": "^7.1.4",
"react-cookie-consent": "^9.0.0",
@ -30,18 +30,18 @@
},
"devDependencies": {
"@babel/core": "^7.24.4",
"@types/node": "^20.12.6",
"@types/react": "18.2.75",
"@types/node": "^20.12.7",
"@types/react": "18.2.79",
"@types/uuid": "^9.0.8",
"babel-loader": "^9.1.3",
"eslint": "^9.0.0",
"eslint-config-next": "^14.1.4",
"eslint": "^8.0.0",
"eslint-config-next": "^14.2.2",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-jsx-a11y": "^6.8.0",
"eslint-plugin-prettier": "^5.1.3",
"eslint-plugin-react": "^7.34.1",
"file-loader": "^6.2.0",
"typescript": "5.4.4"
"typescript": "5.4.5"
}
}

View file

@ -5,23 +5,23 @@
"build": "tsc -p tsconfig.json"
},
"dependencies": {
"@opensearch-project/opensearch": "^2.6.0",
"@opensearch-project/opensearch": "^2.7.0",
"uuid": "^9.0.1"
},
"devDependencies": {
"@babel/core": "^7.24.4",
"@types/node": "^20.12.6",
"@types/react": "18.2.75",
"@types/node": "^20.12.7",
"@types/react": "18.2.79",
"@types/uuid": "^9.0.8",
"babel-loader": "^9.1.3",
"eslint": "^9.0.0",
"eslint-config-next": "^14.1.4",
"eslint": "^8.0.0",
"eslint-config-next": "^14.2.2",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-jsx-a11y": "^6.8.0",
"eslint-plugin-prettier": "^5.1.3",
"eslint-plugin-react": "^7.34.1",
"file-loader": "^6.2.0",
"typescript": "5.4.4"
"typescript": "5.4.5"
}
}

View file

@ -241,7 +241,7 @@ export const Sidebar: FC<SidebarProps> = ({ open, setOpen }) => {
fontFamily: poppins.style.fontFamily,
}}
>
Metamigo
CDR Bridge
</Typography>
</Grid>
)}

View file

@ -11,15 +11,15 @@
"@mui/icons-material": "^5",
"@mui/lab": "^5.0.0-alpha.170",
"@mui/material": "^5",
"@mui/x-data-grid-pro": "^7.1.1",
"@mui/x-date-pickers-pro": "^7.1.1",
"next": "14.1.4",
"@mui/x-data-grid-pro": "^7.3.0",
"@mui/x-date-pickers-pro": "^7.2.0",
"next": "14.2.2",
"react": "18.2.0",
"react-dom": "18.2.0"
},
"devDependencies": {
"@types/node": "^20.12.6",
"@types/react": "18.2.75",
"typescript": "^5.4.4"
"@types/node": "^20.12.7",
"@types/react": "18.2.79",
"typescript": "^5.4.5"
}
}

Some files were not shown because too many files have changed in this diff Show more