feat: add tasks page (#692)

This commit is contained in:
Manuel
2024-07-01 18:57:40 +02:00
committed by GitHub
parent 663eb0bf5b
commit 08d571ad74
43 changed files with 668 additions and 174 deletions

View File

@@ -21,6 +21,9 @@
"dependencies": {
"@homarr/auth": "workspace:^0.1.0",
"@homarr/common": "workspace:^0.1.0",
"@homarr/cron-jobs": "workspace:^0.1.0",
"@homarr/cron-job-runner": "workspace:^0.1.0",
"@homarr/cron-job-status": "workspace:^0.1.0",
"@homarr/db": "workspace:^0.1.0",
"@homarr/definitions": "workspace:^0.1.0",
"@homarr/integrations": "workspace:^0.1.0",

View File

@@ -1,5 +1,6 @@
import { appRouter as innerAppRouter } from "./router/app";
import { boardRouter } from "./router/board";
import { cronJobsRouter } from "./router/cron-jobs";
import { dockerRouter } from "./router/docker/docker-router";
import { groupRouter } from "./router/group";
import { homeRouter } from "./router/home";
@@ -27,6 +28,7 @@ export const appRouter = createTRPCRouter({
home: homeRouter,
docker: dockerRouter,
serverSettings: serverSettingsRouter,
cronJobs: cronJobsRouter,
});
// export type definition of API

View File

@@ -0,0 +1,34 @@
import { observable } from "@trpc/server/observable";
import { jobNameSchema, triggerCronJobAsync } from "@homarr/cron-job-runner";
import type { TaskStatus } from "@homarr/cron-job-status";
import { createCronJobStatusChannel } from "@homarr/cron-job-status";
import { jobGroup } from "@homarr/cron-jobs";
import { logger } from "@homarr/log";
import { createTRPCRouter, publicProcedure } from "../trpc";
export const cronJobsRouter = createTRPCRouter({
triggerJob: publicProcedure.input(jobNameSchema).mutation(async ({ input }) => {
await triggerCronJobAsync(input);
}),
getJobs: publicProcedure.query(() => {
const registry = jobGroup.getJobRegistry();
return [...registry.values()].map((job) => ({
name: job.name,
expression: job.cronExpression,
}));
}),
subscribeToStatusUpdates: publicProcedure.subscription(() => {
return observable<TaskStatus>((emit) => {
for (const job of jobGroup.getJobRegistry().values()) {
const channel = createCronJobStatusChannel(job.name);
channel.subscribe((data) => {
emit.next(data);
});
}
logger.info("A tRPC client has connected to the cron job status updates procedure");
});
}),
});

View File

@@ -0,0 +1,9 @@
import baseConfig from "@homarr/eslint-config/base";
/** @type {import('typescript-eslint').Config} */
export default [
{
ignores: [],
},
...baseConfig,
];

View File

@@ -0,0 +1 @@
export * from "./src";

View File

@@ -0,0 +1,36 @@
{
"name": "@homarr/cron-job-runner",
"private": true,
"version": "0.1.0",
"type": "module",
"exports": {
".": "./index.ts"
},
"typesVersions": {
"*": {
"*": [
"src/*"
]
}
},
"license": "MIT",
"scripts": {
"clean": "rm -rf .turbo node_modules",
"lint": "eslint",
"format": "prettier --check . --ignore-path ../../.gitignore",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@homarr/cron-jobs": "workspace:^0.1.0",
"@homarr/log": "workspace:^0.1.0",
"@homarr/redis": "workspace:^0.1.0"
},
"devDependencies": {
"@homarr/eslint-config": "workspace:^0.2.0",
"@homarr/prettier-config": "workspace:^0.1.0",
"@homarr/tsconfig": "workspace:^0.1.0",
"eslint": "^9.6.0",
"typescript": "^5.5.2"
},
"prettier": "@homarr/prettier-config"
}

View File

@@ -0,0 +1,27 @@
import type { JobGroupKeys } from "@homarr/cron-jobs";
import { jobGroup } from "@homarr/cron-jobs";
import { createSubPubChannel } from "../../redis/src/lib/channel";
import { zodEnumFromArray } from "../../validation/src/enums";
const cronJobRunnerChannel = createSubPubChannel<JobGroupKeys>("cron-job-runner", { persist: false });
/**
* Registers the cron job runner to listen to the Redis PubSub channel.
*/
export const registerCronJobRunner = () => {
cronJobRunnerChannel.subscribe((jobName) => {
jobGroup.runManually(jobName);
});
};
/**
* Triggers a cron job to run immediately.
* This works over the Redis PubSub channel.
* @param jobName name of the job to be triggered
*/
export const triggerCronJobAsync = async (jobName: JobGroupKeys) => {
await cronJobRunnerChannel.publishAsync(jobName);
};
export const jobNameSchema = zodEnumFromArray(jobGroup.getKeys());

View File

@@ -0,0 +1,8 @@
{
"extends": "@homarr/tsconfig/base.json",
"compilerOptions": {
"tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
},
"include": ["*.ts", "src"],
"exclude": ["node_modules"]
}

View File

@@ -0,0 +1,9 @@
import baseConfig from "@homarr/eslint-config/base";
/** @type {import('typescript-eslint').Config} */
export default [
{
ignores: [],
},
...baseConfig,
];

View File

@@ -0,0 +1 @@
export * from "./src";

View File

@@ -0,0 +1,35 @@
{
"name": "@homarr/cron-job-status",
"private": true,
"version": "0.1.0",
"type": "module",
"exports": {
".": "./index.ts",
"./publisher": "./src/publisher.ts"
},
"typesVersions": {
"*": {
"*": [
"src/*"
]
}
},
"license": "MIT",
"scripts": {
"clean": "rm -rf .turbo node_modules",
"lint": "eslint",
"format": "prettier --check . --ignore-path ../../.gitignore",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@homarr/redis": "workspace:^0.1.0"
},
"devDependencies": {
"@homarr/eslint-config": "workspace:^0.2.0",
"@homarr/prettier-config": "workspace:^0.1.0",
"@homarr/tsconfig": "workspace:^0.1.0",
"eslint": "^9.6.0",
"typescript": "^5.5.2"
},
"prettier": "@homarr/prettier-config"
}

View File

@@ -0,0 +1,10 @@
import { createSubPubChannel } from "../../redis/src/lib/channel";
export interface TaskStatus {
name: string;
status: "running" | "idle";
lastExecutionTimestamp: string;
lastExecutionStatus: "success" | "error" | null;
}
export const createCronJobStatusChannel = (name: string) => createSubPubChannel<TaskStatus>(`cron-job-status:${name}`);

View File

@@ -0,0 +1,38 @@
import { createCronJobStatusChannel } from ".";
export const beforeCallbackAsync = async (name: string) => {
const channel = createCronJobStatusChannel(name);
const previous = await channel.getLastDataAsync();
await channel.publishAsync({
name,
lastExecutionStatus: previous?.lastExecutionStatus ?? null,
lastExecutionTimestamp: new Date().toISOString(),
status: "running",
});
};
export const onCallbackSuccessAsync = async (name: string) => {
const channel = createCronJobStatusChannel(name);
const previous = await channel.getLastDataAsync();
await channel.publishAsync({
name,
lastExecutionStatus: "success",
lastExecutionTimestamp: previous?.lastExecutionTimestamp ?? new Date().toISOString(),
status: "idle",
});
};
export const onCallbackErrorAsync = async (name: string, _error: unknown) => {
const channel = createCronJobStatusChannel(name);
const previous = await channel.getLastDataAsync();
await channel.publishAsync({
name,
lastExecutionStatus: "error",
lastExecutionTimestamp: previous?.lastExecutionTimestamp ?? new Date().toISOString(),
status: "idle",
});
};

View File

@@ -0,0 +1,8 @@
{
"extends": "@homarr/tsconfig/base.json",
"compilerOptions": {
"tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
},
"include": ["*.ts", "src"],
"exclude": ["node_modules"]
}

View File

@@ -1,4 +1,4 @@
import { objectEntries } from "@homarr/common";
import { objectEntries, objectKeys } from "@homarr/common";
import type { JobCallback } from "./creator";
import type { Logger } from "./logger";
@@ -43,6 +43,13 @@ export const createJobGroupCreator = <TAllowedNames extends string = string>(
job.scheduledTask.start();
}
},
runManually: (name: keyof TJobs) => {
const job = jobRegistry.get(name as string);
if (!job) return;
options.logger.logInfo(`Running schedule cron job ${job.name} manually.`);
job.scheduledTask.now();
},
stop: (name: keyof TJobs) => {
const job = jobRegistry.get(name as string);
if (!job) return;
@@ -59,6 +66,9 @@ export const createJobGroupCreator = <TAllowedNames extends string = string>(
getJobRegistry() {
return jobRegistry as Map<TAllowedNames, ReturnType<JobCallback<TAllowedNames, TAllowedNames>>>;
},
getKeys() {
return objectKeys(jobs);
},
};
};
};

View File

@@ -0,0 +1,9 @@
import baseConfig from "@homarr/eslint-config/base";
/** @type {import('typescript-eslint').Config} */
export default [
{
ignores: [],
},
...baseConfig,
];

View File

@@ -0,0 +1 @@
export * from "./src";

View File

@@ -0,0 +1,45 @@
{
"name": "@homarr/cron-jobs",
"private": true,
"version": "0.1.0",
"type": "module",
"exports": {
".": "./index.ts"
},
"typesVersions": {
"*": {
"*": [
"src/*"
]
}
},
"license": "MIT",
"scripts": {
"clean": "rm -rf .turbo node_modules",
"lint": "eslint",
"format": "prettier --check . --ignore-path ../../.gitignore",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@homarr/log": "workspace:^0.1.0",
"@homarr/cron-job-status": "workspace:^0.1.0",
"@homarr/cron-jobs-core": "workspace:^0.1.0",
"@homarr/analytics": "workspace:^0.1.0",
"@homarr/db": "workspace:^0.1.0",
"@homarr/server-settings": "workspace:^0.1.0",
"@homarr/common": "workspace:^0.1.0",
"@homarr/icons": "workspace:^0.1.0",
"@homarr/integrations": "workspace:^0.1.0",
"@homarr/redis": "workspace:^0.1.0",
"@homarr/ping": "workspace:^0.1.0",
"@homarr/translation": "workspace:^0.1.0"
},
"devDependencies": {
"@homarr/eslint-config": "workspace:^0.2.0",
"@homarr/prettier-config": "workspace:^0.1.0",
"@homarr/tsconfig": "workspace:^0.1.0",
"eslint": "^9.6.0",
"typescript": "^5.5.2"
},
"prettier": "@homarr/prettier-config"
}

View File

@@ -0,0 +1,14 @@
import { analyticsJob } from "./jobs/analytics";
import { iconsUpdaterJob } from "./jobs/icons-updater";
import { smartHomeEntityStateJob } from "./jobs/integrations/home-assistant";
import { pingJob } from "./jobs/ping";
import { createCronJobGroup } from "./lib";
export const jobGroup = createCronJobGroup({
analytics: analyticsJob,
iconsUpdater: iconsUpdaterJob,
ping: pingJob,
smartHomeEntityState: smartHomeEntityStateJob,
});
export type JobGroupKeys = ReturnType<(typeof jobGroup)["getKeys"]>[number];

View File

@@ -0,0 +1,29 @@
import SuperJSON from "superjson";
import { sendServerAnalyticsAsync } from "@homarr/analytics";
import { EVERY_WEEK } from "@homarr/cron-jobs-core/expressions";
import { db, eq } from "@homarr/db";
import { serverSettings } from "@homarr/db/schema/sqlite";
import type { defaultServerSettings } from "@homarr/server-settings";
import { createCronJob } from "../lib";
export const analyticsJob = createCronJob("analytics", EVERY_WEEK, {
runOnStart: true,
}).withCallback(async () => {
const analyticSetting = await db.query.serverSettings.findFirst({
where: eq(serverSettings.settingKey, "analytics"),
});
if (!analyticSetting) {
return;
}
const value = SuperJSON.parse<(typeof defaultServerSettings)["analytics"]>(analyticSetting.value);
if (!value.enableGeneral) {
return;
}
await sendServerAnalyticsAsync();
});

View File

@@ -0,0 +1,97 @@
import { Stopwatch } from "@homarr/common";
import { EVERY_WEEK } from "@homarr/cron-jobs-core/expressions";
import type { InferInsertModel } from "@homarr/db";
import { db, inArray } from "@homarr/db";
import { createId } from "@homarr/db/client";
import { iconRepositories, icons } from "@homarr/db/schema/sqlite";
import { fetchIconsAsync } from "@homarr/icons";
import { logger } from "@homarr/log";
import { createCronJob } from "../lib";
export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, {
runOnStart: true,
}).withCallback(async () => {
logger.info("Updating icon repository cache...");
const stopWatch = new Stopwatch();
const repositoryIconGroups = await fetchIconsAsync();
const countIcons = repositoryIconGroups
.map((group) => group.icons.length)
.reduce((partialSum, arrayLength) => partialSum + arrayLength, 0);
logger.info(
`Successfully fetched ${countIcons} icons from ${repositoryIconGroups.length} repositories within ${stopWatch.getElapsedInHumanWords()}`,
);
const databaseIconGroups = await db.query.iconRepositories.findMany({
with: {
icons: true,
},
});
const skippedChecksums: string[] = [];
let countDeleted = 0;
let countInserted = 0;
logger.info("Updating icons in database...");
stopWatch.reset();
const newIconRepositories: InferInsertModel<typeof iconRepositories>[] = [];
const newIcons: InferInsertModel<typeof icons>[] = [];
for (const repositoryIconGroup of repositoryIconGroups) {
if (!repositoryIconGroup.success) {
continue;
}
const repositoryInDb = databaseIconGroups.find((dbIconGroup) => dbIconGroup.slug === repositoryIconGroup.slug);
const repositoryIconGroupId: string = repositoryInDb?.id ?? createId();
if (!repositoryInDb?.id) {
newIconRepositories.push({
id: repositoryIconGroupId,
slug: repositoryIconGroup.slug,
});
}
for (const icon of repositoryIconGroup.icons) {
if (databaseIconGroups.flatMap((group) => group.icons).some((dbIcon) => dbIcon.checksum === icon.checksum)) {
skippedChecksums.push(icon.checksum);
continue;
}
newIcons.push({
id: createId(),
checksum: icon.checksum,
name: icon.fileNameWithExtension,
url: icon.imageUrl.href,
iconRepositoryId: repositoryIconGroupId,
});
countInserted++;
}
}
const deadIcons = databaseIconGroups
.flatMap((group) => group.icons)
.filter((icon) => !skippedChecksums.includes(icon.checksum));
await db.transaction(async (transaction) => {
if (newIconRepositories.length >= 1) {
await transaction.insert(iconRepositories).values(newIconRepositories);
}
if (newIcons.length >= 1) {
await transaction.insert(icons).values(newIcons);
}
if (deadIcons.length >= 1) {
await transaction.delete(icons).where(
inArray(
icons.checksum,
deadIcons.map((icon) => icon.checksum),
),
);
}
countDeleted += deadIcons.length;
});
logger.info(`Updated database within ${stopWatch.getElapsedInHumanWords()} (-${countDeleted}, +${countInserted})`);
});

View File

@@ -0,0 +1,65 @@
import SuperJSON from "superjson";
import { decryptSecret } from "@homarr/common";
import { EVERY_MINUTE } from "@homarr/cron-jobs-core/expressions";
import { db, eq } from "@homarr/db";
import { items } from "@homarr/db/schema/sqlite";
import { HomeAssistantIntegration } from "@homarr/integrations";
import { logger } from "@homarr/log";
import { homeAssistantEntityState } from "@homarr/redis";
// This import is done that way to avoid circular dependencies.
import type { WidgetComponentProps } from "../../../../widgets";
import { createCronJob } from "../../lib";
export const smartHomeEntityStateJob = createCronJob("smartHomeEntityState", EVERY_MINUTE).withCallback(async () => {
const itemsForIntegration = await db.query.items.findMany({
where: eq(items.kind, "smartHome-entityState"),
with: {
integrations: {
with: {
integration: {
with: {
secrets: {
columns: {
kind: true,
value: true,
},
},
},
},
},
},
},
});
for (const itemForIntegration of itemsForIntegration) {
const integration = itemForIntegration.integrations[0]?.integration;
if (!integration) {
continue;
}
const options = SuperJSON.parse<WidgetComponentProps<"smartHome-entityState">["options"]>(
itemForIntegration.options,
);
const homeAssistant = new HomeAssistantIntegration({
...integration,
decryptedSecrets: integration.secrets.map((secret) => ({
...secret,
value: decryptSecret(secret.value),
})),
});
const state = await homeAssistant.getEntityStateAsync(options.entityId);
if (!state.success) {
logger.error("Unable to fetch data from Home Assistant");
continue;
}
await homeAssistantEntityState.publishAsync({
entityId: options.entityId,
state: state.data.state,
});
}
});

View File

@@ -0,0 +1,25 @@
import { EVERY_MINUTE } from "@homarr/cron-jobs-core/expressions";
import { logger } from "@homarr/log";
import { sendPingRequestAsync } from "@homarr/ping";
import { pingChannel, pingUrlChannel } from "@homarr/redis";
import { createCronJob } from "../lib";
export const pingJob = createCronJob("ping", EVERY_MINUTE).withCallback(async () => {
const urls = await pingUrlChannel.getAllAsync();
for (const url of new Set(urls)) {
const pingResult = await sendPingRequestAsync(url);
if ("statusCode" in pingResult) {
logger.debug(`executed ping for url ${url} with status code ${pingResult.statusCode}`);
} else {
logger.error(`Executing ping for url ${url} failed with error: ${pingResult.error}`);
}
await pingChannel.publishAsync({
url,
...pingResult,
});
}
});

View File

@@ -0,0 +1,28 @@
import { beforeCallbackAsync, onCallbackErrorAsync, onCallbackSuccessAsync } from "@homarr/cron-job-status/publisher";
import { createCronJobFunctions } from "@homarr/cron-jobs-core";
import type { Logger } from "@homarr/cron-jobs-core/logger";
import { logger } from "@homarr/log";
import type { TranslationObject } from "@homarr/translation";
class WinstonCronJobLogger implements Logger {
logDebug(message: string) {
logger.debug(message);
}
logInfo(message: string) {
logger.info(message);
}
logError(error: unknown) {
logger.error(error);
}
}
export const { createCronJob, createCronJobGroup } = createCronJobFunctions<
keyof TranslationObject["management"]["page"]["tool"]["tasks"]["job"]
>({
logger: new WinstonCronJobLogger(),
beforeCallback: beforeCallbackAsync,
onCallbackSuccess: onCallbackSuccessAsync,
onCallbackError: onCallbackErrorAsync,
});

View File

@@ -0,0 +1,8 @@
{
"extends": "@homarr/tsconfig/base.json",
"compilerOptions": {
"tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
},
"include": ["*.ts", "src"],
"exclude": ["node_modules"]
}

View File

@@ -11,6 +11,6 @@ export const integrationCreatorByKind = (kind: IntegrationKind, integration: Int
case "homeAssistant":
return new HomeAssistantIntegration(integration);
default:
throw new Error(`Unknown integration kind ${kind}`);
throw new Error(`Unknown integration kind ${kind}. Did you forget to add it to the integration creator?`);
}
};

View File

@@ -14,7 +14,7 @@ const lastDataClient = createRedisConnection();
* @param name name of the channel
* @returns pub/sub channel object
*/
export const createSubPubChannel = <TData>(name: string) => {
export const createSubPubChannel = <TData>(name: string, { persist }: { persist: boolean } = { persist: true }) => {
const lastChannelName = `pubSub:last:${name}`;
const channelName = `pubSub:${name}`;
return {
@@ -23,11 +23,13 @@ export const createSubPubChannel = <TData>(name: string) => {
* @param callback callback function to be called when new data is published
*/
subscribe: (callback: (data: TData) => void) => {
void lastDataClient.get(lastChannelName).then((data) => {
if (data) {
callback(superjson.parse(data));
}
});
if (persist) {
void lastDataClient.get(lastChannelName).then((data) => {
if (data) {
callback(superjson.parse(data));
}
});
}
void subscriber.subscribe(channelName, (err) => {
if (!err) {
return;
@@ -45,9 +47,15 @@ export const createSubPubChannel = <TData>(name: string) => {
* @param data data to be published
*/
publishAsync: async (data: TData) => {
await lastDataClient.set(lastChannelName, superjson.stringify(data));
if (persist) {
await lastDataClient.set(lastChannelName, superjson.stringify(data));
}
await publisher.publish(channelName, superjson.stringify(data));
},
getLastDataAsync: async () => {
const data = await lastDataClient.get(lastChannelName);
return data ? superjson.parse<TData>(data) : null;
},
};
};

View File

@@ -1252,6 +1252,7 @@ export default {
items: {
docker: "Docker",
logs: "Logs",
tasks: "Tasks",
},
},
settings: "Settings",
@@ -1451,6 +1452,30 @@ export default {
},
},
},
tool: {
tasks: {
title: "Tasks",
status: {
idle: "Idle",
running: "Running",
error: "Error",
},
job: {
iconsUpdater: {
label: "Icons Updater",
},
analytics: {
label: "Analytics",
},
smartHomeEntityState: {
label: "Smart Home Entity State",
},
ping: {
label: "Pings",
},
},
},
},
about: {
version: "Version {version}",
text: "Homarr is a community driven open source project that is being maintained by volunteers. Thanks to these people, Homarr has been a growing project since 2021. Our team is working completely remote from many different countries on Homarr in their leisure time for no compensation.",