Bridge worker updates

This commit is contained in:
Darren Clarke 2024-04-21 16:59:50 +02:00
parent a445762a37
commit f93c4ad317
33 changed files with 17584 additions and 161 deletions

View file

@ -1,10 +1,11 @@
import * as Worker from "graphile-worker";
import { parseCronItems } from "graphile-worker";
import { initPgp } from "./db.js";
import logger from "./logger.js";
import workerUtils from "./utils.js";
import { assertFfmpegAvailable } from "./lib/media-convert.js";
import { run, Runner, RunnerOptions } from "graphile-worker";
// import { parseCronItems } from "graphile-worker";
// import { initPgp } from "./db.js";
// import logger from "./logger.js";
// import workerUtils from "./utils.js";
// import { assertFfmpegAvailable } from "./lib/media-convert.js";
/*
const logFactory = (scope: any) => (level: any, message: any, meta: any) => {
const pinoLevel = level === "warning" ? "warn" : level;
const childLogger = logger.child({ scope });
@ -12,40 +13,28 @@ const logFactory = (scope: any) => (level: any, message: any, meta: any) => {
else childLogger[pinoLevel](message);
};
export const configWorker = async (): Promise<Worker.RunnerOptions> => {
const {
connection,
concurrency,
pollInterval,
leafcutter: {
enabled: leafcutterEnabled
}
} = config.worker;
logger.info({ concurrency, pollInterval }, "Starting worker");
*/
const getConfig = (): RunnerOptions => {
// logger.info({ concurrency, pollInterval }, "Starting worker");
return {
concurrency,
pollInterval,
logger: new Worker.Logger(logFactory),
connectionString: connection,
concurrency: 10,
pollInterval: 1000,
// logger: new Worker.Logger(logFactory),
connectionString: process.env.DATABASE_URL,
// eslint-disable-next-line unicorn/prefer-module
taskDirectory: `${__dirname}/tasks`,
parsedCronItems: parseCronItems(leafcutterEnabled ?
[{ task: "import-label-studio", pattern: "*/15 * * * *" },
{ task: "import-leafcutter", pattern: "*/17 * * * *" }] :
[])
// parsedCronItems: parseCronItems(
// leafcutterEnabled
// ? [
// { task: "import-label-studio", pattern: "*/15 * * * *" },
// { task: "import-leafcutter", pattern: "*/17 * * * *" },
// ]
// : [],
// )
};
};
export const startWorker = async (): Promise<Worker.Runner> => {
// ensure ffmpeg is installed and working
await assertFfmpegAvailable();
logger.info("ffmpeg found");
await workerUtils.migrate();
logger.info("worker database migrated");
initPgp();
export const startWorker = async (): Promise<Runner> => {
const workerConfig = await configWorker();
const worker = await Worker.run(workerConfig);
return worker;
@ -55,9 +44,15 @@ export const stopWorker = async (): Promise<void> => {
await worker.stop();
};
const worker = defState("worker", {
start: startWorker,
stop: stopWorker,
});
const main = async () => {
await worker.start();
process.on("SIGTERM", async () => {
await worker.stop();
process.exit(0);
});
};
export default worker;
main().catch((err) => {
console.error(err);
process.exit(1);
});