feat: add real time logger page (#276)

* feat: add real time logger

* feat: add subscription for logging

* feat: use timestamp and level in xterm, migrate to new xterm package

* feat: improve design on log page

* fit: remove xterm fit addon

* fix: dispose terminal correctly

* style: format code

* refactor: add jsdoc for redis-transport

* fix: redis connection not possible sometimes

* feat: make terminal full size

* fix: deepsource issues

* fix: lint issue

---------

Co-authored-by: Meier Lukas <meierschlumpf@gmail.com>

apps/nextjs/package.json

@@ -44,8 +44,11 @@
     "@trpc/next": "next",
     "@trpc/react-query": "next",
     "@trpc/server": "next",
+    "@xterm/addon-canvas": "^0.6.0",
+    "@xterm/xterm": "^5.4.0",
     "chroma-js": "^2.4.2",
     "dayjs": "^1.11.10",
+    "dotenv": "^16.4.5",
     "jotai": "^2.7.2",
     "next": "^14.1.4",
     "postcss-preset-mantine": "^1.13.0",
@@ -54,7 +57,7 @@
     "sass": "^1.72.0",
     "superjson": "2.2.1",
     "use-deep-compare-effect": "^1.8.1",
-    "dotenv": "^16.4.5"
+    "xterm-addon-fit": "^0.8.0"
   },
   "devDependencies": {
     "@homarr/eslint-config": "workspace:^0.2.0",
@@ -66,6 +69,7 @@
     "@types/chroma-js": "2.4.4",
     "dotenv-cli": "^7.4.1",
     "concurrently": "^8.2.2",
+    "dotenv-cli": "^7.4.1",
     "eslint": "^8.57.0",
     "prettier": "^3.2.5",
     "tsx": "^4.7.1",

apps/nextjs/src/app/[locale]/boards/_client.tsx

@@ -9,6 +9,7 @@ import { Box, LoadingOverlay, Stack } from "@homarr/ui";
 import { BoardCategorySection } from "~/components/board/sections/category-section";
 import { BoardEmptySection } from "~/components/board/sections/empty-section";
 import { BoardBackgroundVideo } from "~/components/layout/background";
+import { fullHeightWithoutHeaderAndFooter } from "~/constants";
 import { useIsBoardReady, useRequiredBoard } from "./_context";
 
 let boardName: string | null = null;
@@ -58,7 +59,7 @@ export const ClientBoard = () => {
         visible={!isReady}
         transitionProps={{ duration: 500 }}
         loaderProps={{ size: "lg" }}
-        h="calc(100dvh - var(--app-shell-header-offset, 0px) - var(--app-shell-padding) - var(--app-shell-footer-offset, 0px) - var(--app-shell-padding))"
+        h={fullHeightWithoutHeaderAndFooter}
       />
       <Stack
         ref={ref}

apps/nextjs/src/app/[locale]/manage/layout.tsx

@@ -11,6 +11,7 @@ import {
   IconHome,
   IconInfoSmall,
   IconLayoutDashboard,
+  IconLogs,
   IconMailForward,
   IconQuestionMark,
   IconTool,
@@ -61,6 +62,11 @@ export default async function ManageLayout({ children }: PropsWithChildren) {
           icon: IconBrandDocker,
           href: "/manage/tools/docker",
         },
+        {
+          label: t("items.tools.items.logs"),
+          icon: IconLogs,
+          href: "/manage/tools/logs",
+        },
       ],
     },
     {

apps/nextjs/src/app/[locale]/manage/tools/logs/page.tsx

@@ -0,0 +1,34 @@
+import { getScopedI18n } from "@homarr/translation/server";
+import { Box } from "@homarr/ui";
+
+import "@xterm/xterm/css/xterm.css";
+
+import dynamic from "next/dynamic";
+
+import { fullHeightWithoutHeaderAndFooter } from "~/constants";
+
+export async function generateMetadata() {
+  const t = await getScopedI18n("management");
+  const metaTitle = `${t("metaTitle")} • Homarr`;
+
+  return {
+    title: metaTitle,
+  };
+}
+
+const ClientSideTerminalComponent = dynamic(() => import("./terminal"), {
+  ssr: false,
+});
+
+export default function LogsManagementPage() {
+  return (
+    <Box
+      style={{ borderRadius: 6 }}
+      h={fullHeightWithoutHeaderAndFooter}
+      p="md"
+      bg="black"
+    >
+      <ClientSideTerminalComponent />
+    </Box>
+  );
+}

apps/nextjs/src/app/[locale]/manage/tools/logs/terminal.module.css

@@ -0,0 +1,3 @@
+.outerTerminal > div {
+  height: 100%;
+}

apps/nextjs/src/app/[locale]/manage/tools/logs/terminal.tsx

@@ -0,0 +1,66 @@
+"use client";
+
+import { useEffect, useRef } from "react";
+import { CanvasAddon } from "@xterm/addon-canvas";
+import { Terminal } from "@xterm/xterm";
+import { FitAddon } from "xterm-addon-fit";
+
+import { clientApi } from "@homarr/api/client";
+import { Box } from "@homarr/ui";
+
+import classes from "./terminal.module.css";
+
+export default function TerminalComponent() {
+  const ref = useRef<HTMLDivElement>(null);
+
+  const terminalRef = useRef<Terminal>();
+  clientApi.log.subscribe.useSubscription(undefined, {
+    onData(data) {
+      terminalRef.current?.writeln(
+        `${data.timestamp} ${data.level} ${data.message}`,
+      );
+      terminalRef.current?.refresh(0, terminalRef.current.rows - 1);
+    },
+    onError(err) {
+      // This makes sense as logging might cause an infinite loop
+      alert(err);
+    },
+  });
+
+  useEffect(() => {
+    if (!ref.current) {
+      return () => undefined;
+    }
+
+    const canvasAddon = new CanvasAddon();
+
+    terminalRef.current = new Terminal({
+      cursorBlink: false,
+      disableStdin: true,
+      convertEol: true,
+    });
+    terminalRef.current.open(ref.current);
+    terminalRef.current.loadAddon(canvasAddon);
+
+    // This is a hack to make sure the terminal is rendered before we try to fit it
+    // You can blame @Meierschlumpf for this
+    setTimeout(() => {
+      const fitAddon = new FitAddon();
+      terminalRef.current?.loadAddon(fitAddon);
+      fitAddon.fit();
+    });
+
+    return () => {
+      terminalRef.current?.dispose();
+      canvasAddon.dispose();
+    };
+  }, []);
+  return (
+    <Box
+      ref={ref}
+      id="terminal"
+      className={classes.outerTerminal}
+      h="100%"
+    ></Box>
+  );
+}

apps/nextjs/src/constants.ts

@@ -0,0 +1,2 @@
+export const fullHeightWithoutHeaderAndFooter =
+  "calc(100dvh - var(--app-shell-header-offset, 0px) - var(--app-shell-padding) - var(--app-shell-footer-offset, 0px) - var(--app-shell-padding))";

packages/api/src/root.ts

@@ -1,6 +1,7 @@
 import { appRouter as innerAppRouter } from "./router/app";
 import { boardRouter } from "./router/board";
 import { integrationRouter } from "./router/integration";
+import { logRouter } from "./router/log";
 import { userRouter } from "./router/user";
 import { createTRPCRouter } from "./trpc";
 
@@ -9,6 +10,7 @@ export const appRouter = createTRPCRouter({
   integration: integrationRouter,
   board: boardRouter,
   app: innerAppRouter,
+  log: logRouter,
 });
 
 // export type definition of API

packages/api/src/router/log.ts

@@ -0,0 +1,18 @@
+import { observable } from "@trpc/server/observable";
+
+import { logger } from "@homarr/log";
+import type { LoggerMessage } from "@homarr/redis";
+import { loggingChannel } from "@homarr/redis";
+
+import { createTRPCRouter, publicProcedure } from "../trpc";
+
+export const logRouter = createTRPCRouter({
+  subscribe: publicProcedure.subscription(() => {
+    return observable<LoggerMessage>((emit) => {
+      loggingChannel.subscribe((data) => {
+        emit.next(data);
+      });
+      logger.info("A tRPC client has connected to the logging procedure");
+    });
+  }),
+});

packages/log/package.json

@@ -24,6 +24,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "ioredis": "5.3.2",
+    "superjson": "2.2.1",
     "winston": "3.13.0"
   },
   "devDependencies": {

packages/log/src/index.mjs

@@ -1,5 +1,7 @@
 import winston, { format, transports } from "winston";
 
+import { RedisTransport } from "./redis-transport.mjs";
+
 const logMessageFormat = format.printf(({ level, message, timestamp }) => {
   return `${timestamp} ${level}: ${message}`;
 });
@@ -10,7 +12,7 @@ const logger = winston.createLogger({
     format.timestamp(),
     logMessageFormat,
   ),
-  transports: [new transports.Console()],
+  transports: [new transports.Console(), new RedisTransport()],
 });
 
 export { logger };

packages/log/src/redis-transport.mjs

@@ -0,0 +1,44 @@
+import { Redis } from "ioredis";
+import superjson from "superjson";
+import Transport from "winston-transport";
+
+//
+// Inherit from `winston-transport` so you can take advantage
+// of the base functionality and `.exceptions.handle()`.
+//
+export class RedisTransport extends Transport {
+  /** @type {Redis} */
+  redis;
+
+  /**
+   * Log the info to the Redis channel
+   * @param {{ message: string; timestamp: string; level: string; }} info
+   * @param {() => void} callback
+   */
+  log(info, callback) {
+    setImmediate(() => {
+      this.emit("logged", info);
+    });
+
+    if (!this.redis) {
+      // Is only initialized here because it did not work when initialized in the constructor or outside the class
+      this.redis = new Redis();
+    }
+
+    this.redis
+      .publish(
+        "logging",
+        superjson.stringify({
+          message: info.message,
+          timestamp: info.timestamp,
+          level: info.level,
+        }),
+      )
+      .then(() => {
+        callback();
+      })
+      .catch(() => {
+        // Ignore errors
+      });
+  }
+}

packages/redis/src/index.ts

@@ -36,4 +36,12 @@ const createChannel = <TData>(name: string) => {
   };
 };
 
+export interface LoggerMessage {
+  message: string;
+  level: string;
+  timestamp: string;
+}
+
+export const loggingChannel = createChannel<LoggerMessage>("logging");
+
 export const exampleChannel = createChannel<{ message: string }>("example");

packages/translation/src/lang/en.ts

@@ -620,6 +620,7 @@ export default {
           label: "Tools",
           items: {
             docker: "Docker",
+            logs: "Logs",
           },
         },
         help: {

pnpm-lock.yaml

@@ -228,6 +228,12 @@ importers:
       '@trpc/server':
         specifier: next
         version: 11.0.0-next-beta.289
+      '@xterm/addon-canvas':
+        specifier: ^0.6.0
+        version: 0.6.0(@xterm/xterm@5.4.0)
+      '@xterm/xterm':
+        specifier: ^5.4.0
+        version: 5.4.0
       chroma-js:
         specifier: ^2.4.2
         version: 2.4.2
@@ -261,6 +267,9 @@ importers:
       use-deep-compare-effect:
         specifier: ^1.8.1
         version: 1.8.1(react@18.2.0)
+      xterm-addon-fit:
+        specifier: ^0.8.0
+        version: 0.8.0(xterm@5.3.0)
     devDependencies:
       '@homarr/eslint-config':
         specifier: workspace:^0.2.0
@@ -534,6 +543,12 @@ importers:
 
   packages/log:
     dependencies:
+      ioredis:
+        specifier: 5.3.2
+        version: 5.3.2
+      superjson:
+        specifier: 2.2.1
+        version: 2.2.1
       winston:
         specifier: 3.13.0
         version: 3.13.0
@@ -4117,6 +4132,18 @@ packages:
       '@xtuc/long': 4.2.2
     dev: true
 
+  /@xterm/addon-canvas@0.6.0(@xterm/xterm@5.4.0):
+    resolution: {integrity: sha512-+nj2x595vItxfuAFxzXp46Izrh4EnEyS0Z60hX1iy6OFliP5OQu8Wu7n59m7m1vT6Q4nIWoN1WiH+VLAk4D9jQ==}
+    peerDependencies:
+      '@xterm/xterm': ^5.0.0
+    dependencies:
+      '@xterm/xterm': 5.4.0
+    dev: false
+
+  /@xterm/xterm@5.4.0:
+    resolution: {integrity: sha512-GlyzcZZ7LJjhFevthHtikhiDIl8lnTSgol6eTM4aoSNLcuXu3OEhnbqdCVIjtIil3jjabf3gDtb1S8FGahsuEw==}
+    dev: false
+
   /@xtuc/ieee754@1.2.0:
     resolution: {integrity: sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==}
     dev: true
@@ -10751,6 +10778,18 @@ packages:
     resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}
     engines: {node: '>=0.4'}
 
+  /xterm-addon-fit@0.8.0(xterm@5.3.0):
+    resolution: {integrity: sha512-yj3Np7XlvxxhYF/EJ7p3KHaMt6OdwQ+HDu573Vx1lRXsVxOcnVJs51RgjZOouIZOczTsskaS+CpXspK81/DLqw==}
+    peerDependencies:
+      xterm: ^5.0.0
+    dependencies:
+      xterm: 5.3.0
+    dev: false
+
+  /xterm@5.3.0:
+    resolution: {integrity: sha512-8QqjlekLUFTrU6x7xck1MsPzPA571K5zNqWm0M0oroYEWVOptZ0+ubQSkQ3uxIEhcIHRujJy6emDWX4A7qyFzg==}
+    dev: false
+
   /y18n@5.0.8:
     resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==}
     engines: {node: '>=10'}