refactor: add request handlers for centralized cached requests (#1504)

* feat: add object base64 hash method

* chore: add script to add package

* feat: add request-handler package

* wip: add request handlers for all jobs and widget api procedures

* wip: remove errors shown in logs, add missing decryption for secrets in cached-request-job-handler

* wip: highly improve request handler, add request handlers for calendar, media-server, indexer-manager and more, add support for multiple inputs from job handler creator

* refactor: move media-server requests to request-handler, add invalidation logic for dns-hole and media requests

* refactor: remove unused integration item middleware

* feat: add invalidation to switch entity action of smart-home

* fix: lint issues

* chore: use integration-kind-by-category instead of union for request-handlers

* fix: build not working for tasks and websocket

* refactor: add more logs

* refactor: readd timestamp logic for diconnect status

* fix: lint and typecheck issue

* chore: address pull request feedback

packages/cron-jobs/src/index.ts

@@ -6,7 +6,7 @@ import { healthMonitoringJob } from "./jobs/integrations/health-monitoring";
 import { smartHomeEntityStateJob } from "./jobs/integrations/home-assistant";
 import { indexerManagerJob } from "./jobs/integrations/indexer-manager";
 import { mediaOrganizerJob } from "./jobs/integrations/media-organizer";
-import { mediaRequestsJob } from "./jobs/integrations/media-requests";
+import { mediaRequestListJob, mediaRequestStatsJob } from "./jobs/integrations/media-requests";
 import { mediaServerJob } from "./jobs/integrations/media-server";
 import { pingJob } from "./jobs/ping";
 import type { RssFeed } from "./jobs/rss-feeds";
@@ -23,7 +23,8 @@ export const jobGroup = createCronJobGroup({
   mediaOrganizer: mediaOrganizerJob,
   downloads: downloadsJob,
   dnsHole: dnsHoleJob,
-  mediaRequests: mediaRequestsJob,
+  mediaRequestStats: mediaRequestStatsJob,
+  mediaRequestList: mediaRequestListJob,
   rssFeeds: rssFeedsJob,
   indexerManager: indexerManagerJob,
   healthMonitoring: healthMonitoringJob,

packages/cron-jobs/src/jobs/integrations/dns-hole.ts

@@ -1,28 +1,15 @@
 import { EVERY_5_SECONDS } from "@homarr/cron-jobs-core/expressions";
-import { db } from "@homarr/db";
-import { getItemsWithIntegrationsAsync } from "@homarr/db/queries";
-import { integrationCreatorFromSecrets } from "@homarr/integrations";
-import type { DnsHoleSummary } from "@homarr/integrations/types";
-import { logger } from "@homarr/log";
-import { createItemAndIntegrationChannel } from "@homarr/redis";
+import { dnsHoleRequestHandler } from "@homarr/request-handler/dns-hole";
+import { createRequestIntegrationJobHandler } from "@homarr/request-handler/lib/cached-request-integration-job-handler";
 
 import { createCronJob } from "../../lib";
 
-export const dnsHoleJob = createCronJob("dnsHole", EVERY_5_SECONDS).withCallback(async () => {
-  const itemsForIntegration = await getItemsWithIntegrationsAsync(db, {
-    kinds: ["dnsHoleSummary", "dnsHoleControls"],
-  });
-
-  for (const itemForIntegration of itemsForIntegration) {
-    for (const { integration } of itemForIntegration.integrations) {
-      const integrationInstance = integrationCreatorFromSecrets(integration);
-      await integrationInstance
-        .getSummaryAsync()
-        .then(async (data) => {
-          const channel = createItemAndIntegrationChannel<DnsHoleSummary>(itemForIntegration.kind, integration.id);
-          await channel.publishAndUpdateLastStateAsync(data);
-        })
-        .catch((error) => logger.error(`Could not retrieve data for ${integration.name}: "${error}"`));
-    }
-  }
-});
+export const dnsHoleJob = createCronJob("dnsHole", EVERY_5_SECONDS).withCallback(
+  createRequestIntegrationJobHandler(dnsHoleRequestHandler.handler, {
+    widgetKinds: ["dnsHoleSummary", "dnsHoleControls"],
+    getInput: {
+      dnsHoleSummary: () => ({}),
+      dnsHoleControls: () => ({}),
+    },
+  }),
+);

packages/cron-jobs/src/jobs/integrations/downloads.ts

@@ -1,27 +1,14 @@
 import { EVERY_5_SECONDS } from "@homarr/cron-jobs-core/expressions";
-import { db } from "@homarr/db";
-import { getItemsWithIntegrationsAsync } from "@homarr/db/queries";
-import type { DownloadClientJobsAndStatus } from "@homarr/integrations";
-import { integrationCreatorFromSecrets } from "@homarr/integrations";
-import { createItemAndIntegrationChannel } from "@homarr/redis";
+import { downloadClientRequestHandler } from "@homarr/request-handler/downloads";
+import { createRequestIntegrationJobHandler } from "@homarr/request-handler/lib/cached-request-integration-job-handler";
 
 import { createCronJob } from "../../lib";
 
-export const downloadsJob = createCronJob("downloads", EVERY_5_SECONDS).withCallback(async () => {
-  const itemsForIntegration = await getItemsWithIntegrationsAsync(db, {
-    kinds: ["downloads"],
-  });
-
-  for (const itemForIntegration of itemsForIntegration) {
-    for (const { integration } of itemForIntegration.integrations) {
-      const integrationInstance = integrationCreatorFromSecrets(integration);
-      await integrationInstance
-        .getClientJobsAndStatusAsync()
-        .then(async (data) => {
-          const channel = createItemAndIntegrationChannel<DownloadClientJobsAndStatus>("downloads", integration.id);
-          await channel.publishAndUpdateLastStateAsync(data);
-        })
-        .catch((error) => console.error(`Could not retrieve data for ${integration.name}: "${error}"`));
-    }
-  }
-});
+export const downloadsJob = createCronJob("downloads", EVERY_5_SECONDS).withCallback(
+  createRequestIntegrationJobHandler(downloadClientRequestHandler.handler, {
+    widgetKinds: ["downloads"],
+    getInput: {
+      downloads: () => ({}),
+    },
+  }),
+);

packages/cron-jobs/src/jobs/integrations/health-monitoring.ts

@@ -1,22 +1,14 @@
 import { EVERY_5_SECONDS } from "@homarr/cron-jobs-core/expressions";
-import { db } from "@homarr/db";
-import { getItemsWithIntegrationsAsync } from "@homarr/db/queries";
-import { integrationCreatorFromSecrets } from "@homarr/integrations";
-import { createItemAndIntegrationChannel } from "@homarr/redis";
+import { systemInfoRequestHandler } from "@homarr/request-handler/health-monitoring";
+import { createRequestIntegrationJobHandler } from "@homarr/request-handler/lib/cached-request-integration-job-handler";
 
 import { createCronJob } from "../../lib";
 
-export const healthMonitoringJob = createCronJob("healthMonitoring", EVERY_5_SECONDS).withCallback(async () => {
-  const itemsForIntegration = await getItemsWithIntegrationsAsync(db, {
-    kinds: ["healthMonitoring"],
-  });
-
-  for (const itemForIntegration of itemsForIntegration) {
-    for (const integration of itemForIntegration.integrations) {
-      const openmediavault = integrationCreatorFromSecrets(integration.integration);
-      const healthInfo = await openmediavault.getSystemInfoAsync();
-      const channel = createItemAndIntegrationChannel("healthMonitoring", integration.integrationId);
-      await channel.publishAndUpdateLastStateAsync(healthInfo);
-    }
-  }
-});
+export const healthMonitoringJob = createCronJob("healthMonitoring", EVERY_5_SECONDS).withCallback(
+  createRequestIntegrationJobHandler(systemInfoRequestHandler.handler, {
+    widgetKinds: ["healthMonitoring"],
+    getInput: {
+      healthMonitoring: () => ({}),
+    },
+  }),
+);

packages/cron-jobs/src/jobs/integrations/home-assistant.ts

@@ -1,42 +1,16 @@
-import SuperJSON from "superjson";
-
 import { EVERY_MINUTE } from "@homarr/cron-jobs-core/expressions";
-import { db } from "@homarr/db";
-import { getItemsWithIntegrationsAsync } from "@homarr/db/queries";
-import { integrationCreatorFromSecrets } from "@homarr/integrations";
-import { logger } from "@homarr/log";
-import { homeAssistantEntityState } from "@homarr/redis";
+import { createRequestIntegrationJobHandler } from "@homarr/request-handler/lib/cached-request-integration-job-handler";
+import { smartHomeEntityStateRequestHandler } from "@homarr/request-handler/smart-home-entity-state";
 
-// This import is done that way to avoid circular dependencies.
-import type { WidgetComponentProps } from "../../../../widgets";
 import { createCronJob } from "../../lib";
 
-export const smartHomeEntityStateJob = createCronJob("smartHomeEntityState", EVERY_MINUTE).withCallback(async () => {
-  const itemsForIntegration = await getItemsWithIntegrationsAsync(db, {
-    kinds: ["smartHome-entityState"],
-  });
-
-  for (const itemForIntegration of itemsForIntegration) {
-    const integration = itemForIntegration.integrations[0]?.integration;
-    if (!integration) {
-      continue;
-    }
-
-    const options = SuperJSON.parse<WidgetComponentProps<"smartHome-entityState">["options"]>(
-      itemForIntegration.options,
-    );
-
-    const homeAssistant = integrationCreatorFromSecrets(integration);
-    const state = await homeAssistant.getEntityStateAsync(options.entityId);
-
-    if (!state.success) {
-      logger.error("Unable to fetch data from Home Assistant");
-      continue;
-    }
-
-    await homeAssistantEntityState.publishAsync({
-      entityId: options.entityId,
-      state: state.data.state,
-    });
-  }
-});
+export const smartHomeEntityStateJob = createCronJob("smartHomeEntityState", EVERY_MINUTE).withCallback(
+  createRequestIntegrationJobHandler(smartHomeEntityStateRequestHandler.handler, {
+    widgetKinds: ["smartHome-entityState"],
+    getInput: {
+      "smartHome-entityState": (options) => ({
+        entityId: options.entityId,
+      }),
+    },
+  }),
+);

packages/cron-jobs/src/jobs/integrations/indexer-manager.ts

@@ -1,19 +1,14 @@
-import { EVERY_MINUTE } from "@homarr/cron-jobs-core/expressions";
-import { db } from "@homarr/db";
-import { getItemsWithIntegrationsAsync } from "@homarr/db/queries";
-import { integrationCreatorFromSecrets } from "@homarr/integrations";
+import { EVERY_5_MINUTES } from "@homarr/cron-jobs-core/expressions";
+import { indexerManagerRequestHandler } from "@homarr/request-handler/indexer-manager";
+import { createRequestIntegrationJobHandler } from "@homarr/request-handler/lib/cached-request-integration-job-handler";
 
 import { createCronJob } from "../../lib";
 
-export const indexerManagerJob = createCronJob("indexerManager", EVERY_MINUTE).withCallback(async () => {
-  const itemsForIntegration = await getItemsWithIntegrationsAsync(db, {
-    kinds: ["indexerManager"],
-  });
-
-  for (const itemForIntegration of itemsForIntegration) {
-    for (const { integration } of itemForIntegration.integrations) {
-      const integrationInstance = integrationCreatorFromSecrets(integration);
-      await integrationInstance.getIndexersAsync();
-    }
-  }
-});
+export const indexerManagerJob = createCronJob("indexerManager", EVERY_5_MINUTES).withCallback(
+  createRequestIntegrationJobHandler(indexerManagerRequestHandler.handler, {
+    widgetKinds: ["indexerManager"],
+    getInput: {
+      indexerManager: () => ({}),
+    },
+  }),
+);

packages/cron-jobs/src/jobs/integrations/media-organizer.ts

@@ -1,36 +1,35 @@
 import dayjs from "dayjs";
-import SuperJSON from "superjson";
 
 import { EVERY_MINUTE } from "@homarr/cron-jobs-core/expressions";
-import { db } from "@homarr/db";
-import { getItemsWithIntegrationsAsync } from "@homarr/db/queries";
-import { integrationCreatorFromSecrets } from "@homarr/integrations";
-import type { CalendarEvent } from "@homarr/integrations/types";
-import { createItemAndIntegrationChannel } from "@homarr/redis";
+import { calendarMonthRequestHandler } from "@homarr/request-handler/calendar";
+import { createRequestIntegrationJobHandler } from "@homarr/request-handler/lib/cached-request-integration-job-handler";
 
-// This import is done that way to avoid circular dependencies.
-import type { WidgetComponentProps } from "../../../../widgets";
 import { createCronJob } from "../../lib";
 
-export const mediaOrganizerJob = createCronJob("mediaOrganizer", EVERY_MINUTE).withCallback(async () => {
-  const itemsForIntegration = await getItemsWithIntegrationsAsync(db, {
-    kinds: ["calendar"],
-  });
+export const mediaOrganizerJob = createCronJob("mediaOrganizer", EVERY_MINUTE).withCallback(
+  createRequestIntegrationJobHandler(calendarMonthRequestHandler.handler, {
+    widgetKinds: ["calendar"],
+    getInput: {
+      // Request handler will run for all specified months
+      calendar: (options) => {
+        const inputs = [];
 
-  for (const itemForIntegration of itemsForIntegration) {
-    for (const { integration } of itemForIntegration.integrations) {
-      const options = SuperJSON.parse<WidgetComponentProps<"calendar">["options"]>(itemForIntegration.options);
+        const startOffset = -Number(options.filterPastMonths);
+        const endOffset = Number(options.filterFutureMonths);
 
-      const start = dayjs().subtract(Number(options.filterPastMonths), "months").toDate();
-      const end = dayjs().add(Number(options.filterFutureMonths), "months").toDate();
+        for (let offsetMonths = startOffset; offsetMonths <= endOffset; offsetMonths++) {
+          const year = dayjs().subtract(offsetMonths, "months").year();
+          const month = dayjs().subtract(offsetMonths, "months").month();
 
-      //Asserting the integration kind until all of them get implemented
-      const integrationInstance = integrationCreatorFromSecrets(integration);
+          inputs.push({
+            year,
+            month,
+            releaseType: options.releaseType,
+          });
+        }
 
-      const events = await integrationInstance.getCalendarEventsAsync(start, end);
-
-      const cache = createItemAndIntegrationChannel<CalendarEvent[]>("calendar", integration.id);
-      await cache.setAsync(events);
-    }
-  }
-});
+        return inputs;
+      },
+    },
+  }),
+);

packages/cron-jobs/src/jobs/integrations/media-requests.ts

@@ -1,42 +1,24 @@
-import { EVERY_5_SECONDS } from "@homarr/cron-jobs-core/expressions";
-import { db } from "@homarr/db";
-import { getItemsWithIntegrationsAsync } from "@homarr/db/queries";
-import type { MediaRequestList, MediaRequestStats } from "@homarr/integrations";
-import { integrationCreatorFromSecrets } from "@homarr/integrations";
-import { createItemAndIntegrationChannel } from "@homarr/redis";
+import { EVERY_MINUTE } from "@homarr/cron-jobs-core/expressions";
+import { createRequestIntegrationJobHandler } from "@homarr/request-handler/lib/cached-request-integration-job-handler";
+import { mediaRequestListRequestHandler } from "@homarr/request-handler/media-request-list";
+import { mediaRequestStatsRequestHandler } from "@homarr/request-handler/media-request-stats";
 
 import { createCronJob } from "../../lib";
 
-export const mediaRequestsJob = createCronJob("mediaRequests", EVERY_5_SECONDS).withCallback(async () => {
-  const itemsForIntegration = await getItemsWithIntegrationsAsync(db, {
-    kinds: ["mediaRequests-requestList", "mediaRequests-requestStats"],
-  });
+export const mediaRequestStatsJob = createCronJob("mediaRequestStats", EVERY_MINUTE).withCallback(
+  createRequestIntegrationJobHandler(mediaRequestStatsRequestHandler.handler, {
+    widgetKinds: ["mediaRequests-requestStats"],
+    getInput: {
+      "mediaRequests-requestStats": () => ({}),
+    },
+  }),
+);
 
-  for (const itemForIntegration of itemsForIntegration) {
-    for (const { integration } of itemForIntegration.integrations) {
-      const requestsIntegration = integrationCreatorFromSecrets(integration);
-
-      const mediaRequests = await requestsIntegration.getRequestsAsync();
-      const requestsStats = await requestsIntegration.getStatsAsync();
-      const requestsUsers = await requestsIntegration.getUsersAsync();
-      const requestListChannel = createItemAndIntegrationChannel<MediaRequestList>(
-        "mediaRequests-requestList",
-        integration.id,
-      );
-      await requestListChannel.publishAndUpdateLastStateAsync({
-        integration: { id: integration.id },
-        medias: mediaRequests,
-      });
-
-      const requestStatsChannel = createItemAndIntegrationChannel<MediaRequestStats>(
-        "mediaRequests-requestStats",
-        integration.id,
-      );
-      await requestStatsChannel.publishAndUpdateLastStateAsync({
-        integration: { kind: integration.kind, name: integration.name },
-        stats: requestsStats,
-        users: requestsUsers,
-      });
-    }
-  }
-});
+export const mediaRequestListJob = createCronJob("mediaRequestList", EVERY_MINUTE).withCallback(
+  createRequestIntegrationJobHandler(mediaRequestListRequestHandler.handler, {
+    widgetKinds: ["mediaRequests-requestList"],
+    getInput: {
+      "mediaRequests-requestList": () => ({}),
+    },
+  }),
+);

packages/cron-jobs/src/jobs/integrations/media-server.ts

@@ -1,22 +1,14 @@
 import { EVERY_5_SECONDS } from "@homarr/cron-jobs-core/expressions";
-import { db } from "@homarr/db";
-import { getItemsWithIntegrationsAsync } from "@homarr/db/queries";
-import { integrationCreatorFromSecrets } from "@homarr/integrations";
-import { createItemAndIntegrationChannel } from "@homarr/redis";
+import { createRequestIntegrationJobHandler } from "@homarr/request-handler/lib/cached-request-integration-job-handler";
+import { mediaServerRequestHandler } from "@homarr/request-handler/media-server";
 
 import { createCronJob } from "../../lib";
 
-export const mediaServerJob = createCronJob("mediaServer", EVERY_5_SECONDS).withCallback(async () => {
-  const itemsForIntegration = await getItemsWithIntegrationsAsync(db, {
-    kinds: ["mediaServer"],
-  });
-
-  for (const itemForIntegration of itemsForIntegration) {
-    for (const { integration } of itemForIntegration.integrations) {
-      const integrationInstance = integrationCreatorFromSecrets(integration);
-      const streamSessions = await integrationInstance.getCurrentSessionsAsync();
-      const channel = createItemAndIntegrationChannel("mediaServer", integration.id);
-      await channel.publishAndUpdateLastStateAsync(streamSessions);
-    }
-  }
-});
+export const mediaServerJob = createCronJob("mediaServer", EVERY_5_SECONDS).withCallback(
+  createRequestIntegrationJobHandler(mediaServerRequestHandler.handler, {
+    widgetKinds: ["mediaServer"],
+    getInput: {
+      mediaServer: () => ({}),
+    },
+  }),
+);