refactor(http): move to core package (#4690)

packages/core/src/infrastructure/certificates/hostnames/index.ts

@@ -1,3 +1,5 @@
+import type { InferSelectModel } from "drizzle-orm";
+
 import { createDb } from "../../db";
 import { schema } from "./db/schema";
 
@@ -6,3 +8,5 @@ const db = createDb(schema);
 export const getTrustedCertificateHostnamesAsync = async () => {
   return await db.query.trustedCertificateHostnames.findMany();
 };
+
+export type TrustedCertificateHostname = InferSelectModel<typeof schema.trustedCertificateHostnames>;

packages/core/src/infrastructure/http/http-agent.ts

@@ -0,0 +1,72 @@
+import type { Dispatcher } from "undici";
+import { Agent } from "undici";
+
+import type { ILogger } from "@homarr/core/infrastructure/logs";
+import { createLogger } from "@homarr/core/infrastructure/logs";
+
+// The below import statement initializes dns-caching
+import "@homarr/core/infrastructure/dns/init";
+
+interface HttpAgentOptions extends Agent.Options {
+  logger?: ILogger;
+}
+
+export class UndiciHttpAgent extends Agent {
+  private logger: ILogger;
+
+  constructor(props?: HttpAgentOptions) {
+    super(props);
+
+    this.logger = props?.logger ?? createLogger({ module: "httpAgent" });
+  }
+
+  dispatch(options: Dispatcher.DispatchOptions, handler: Dispatcher.DispatchHandler): boolean {
+    this.logRequestDispatch(options);
+    return super.dispatch(options, handler);
+  }
+
+  private logRequestDispatch(options: Dispatcher.DispatchOptions) {
+    const path = this.redactPathParams(options.path);
+    let url = new URL(`${options.origin as string}${path}`);
+    url = this.redactSearchParams(url);
+
+    this.logger.debug(
+      `Dispatching request ${url.toString().replaceAll("=&", "&")} (${Object.keys(options.headers ?? {}).length} headers)`,
+    );
+  }
+
+  /**
+   * Redact path parameters that are longer than 32 characters
+   * This is to prevent sensitive data from being logged
+   * @param path path of the request
+   * @returns redacted path
+   */
+  private redactPathParams(path: string): string {
+    return path
+      .split("/")
+      .map((segment) => (segment.length >= 32 && !segment.startsWith("?") ? "REDACTED" : segment))
+      .join("/");
+  }
+
+  /**
+   * Redact sensitive search parameters from the URL.
+   * It allows certain patterns to remain unredacted.
+   * Like small numbers, booleans, short strings, dates, and date-times.
+   * Some integrations use query parameters for auth.
+   * @param url URL object of the request
+   * @returns redacted URL object
+   */
+  private redactSearchParams(url: URL): URL {
+    url.searchParams.forEach((value, key) => {
+      if (value === "") return; // Skip empty values
+      if (/^-?\d{1,12}$/.test(value)) return; // Skip small numbers
+      if (value === "true" || value === "false") return; // Skip boolean values
+      if (/^[a-zA-Z]{1,12}$/.test(value)) return; // Skip short strings
+      if (/^\d{4}-\d{2}-\d{2}$/.test(value)) return; // Skip dates
+      if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(value)) return; // Skip date times
+
+      url.searchParams.set(key, "REDACTED");
+    });
+    return url;
+  }
+}

packages/core/src/infrastructure/http/index.ts

@@ -0,0 +1,8 @@
+export { UndiciHttpAgent } from "./http-agent";
+export {
+  createAxiosCertificateInstanceAsync,
+  createCertificateAgentAsync,
+  createCustomCheckServerIdentity,
+  createHttpsAgentAsync,
+  fetchWithTrustedCertificatesAsync,
+} from "./request";

packages/core/src/infrastructure/http/request.ts

@@ -0,0 +1,82 @@
+import type { AgentOptions } from "node:https";
+import { Agent as HttpsAgent } from "node:https";
+import { checkServerIdentity } from "node:tls";
+import axios from "axios";
+import type { RequestInfo, RequestInit, Response } from "undici";
+import { fetch } from "undici";
+
+import {
+  getAllTrustedCertificatesAsync,
+  getTrustedCertificateHostnamesAsync,
+} from "@homarr/core/infrastructure/certificates";
+import { UndiciHttpAgent } from "@homarr/core/infrastructure/http";
+
+import type { TrustedCertificateHostname } from "../certificates/hostnames";
+import { withTimeoutAsync } from "./timeout";
+
+export const createCustomCheckServerIdentity = (
+  trustedHostnames: TrustedCertificateHostname[],
+): typeof checkServerIdentity => {
+  return (hostname, peerCertificate) => {
+    const matchingTrustedHostnames = trustedHostnames.filter(
+      (cert) => cert.thumbprint === peerCertificate.fingerprint256,
+    );
+
+    // We trust the certificate if we have a matching hostname
+    if (matchingTrustedHostnames.some((cert) => cert.hostname === hostname)) return undefined;
+
+    return checkServerIdentity(hostname, peerCertificate);
+  };
+};
+
+export const createCertificateAgentAsync = async (override?: {
+  ca: string | string[];
+  checkServerIdentity: typeof checkServerIdentity;
+}) => {
+  return new UndiciHttpAgent({
+    connect: override ?? {
+      ca: await getAllTrustedCertificatesAsync(),
+      checkServerIdentity: createCustomCheckServerIdentity(await getTrustedCertificateHostnamesAsync()),
+    },
+  });
+};
+
+export const createHttpsAgentAsync = async (override?: Pick<AgentOptions, "ca" | "checkServerIdentity">) => {
+  return new HttpsAgent(
+    override ?? {
+      ca: await getAllTrustedCertificatesAsync(),
+      checkServerIdentity: createCustomCheckServerIdentity(await getTrustedCertificateHostnamesAsync()),
+    },
+  );
+};
+
+export const createAxiosCertificateInstanceAsync = async (
+  override?: Pick<AgentOptions, "ca" | "checkServerIdentity">,
+) => {
+  return axios.create({
+    httpsAgent: await createHttpsAgentAsync(override),
+  });
+};
+
+export const fetchWithTrustedCertificatesAsync = async (
+  url: RequestInfo,
+  options?: RequestInit & { timeout?: number },
+): Promise<Response> => {
+  const agent = await createCertificateAgentAsync(undefined);
+  if (options?.timeout) {
+    return await withTimeoutAsync(
+      async (signal) =>
+        fetch(url, {
+          ...options,
+          signal,
+          dispatcher: agent,
+        }),
+      options.timeout,
+    );
+  }
+
+  return fetch(url, {
+    ...options,
+    dispatcher: agent,
+  });
+};

packages/core/src/infrastructure/http/timeout.ts

@@ -0,0 +1,19 @@
+import type { Response as UndiciResponse } from "undici";
+
+// https://stackoverflow.com/questions/46946380/fetch-api-request-timeout
+export const withTimeoutAsync = async <TResponse extends Response | UndiciResponse>(
+  callback: (signal: AbortSignal) => Promise<TResponse>,
+  timeout = 10000,
+) => {
+  const controller = new AbortController();
+
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+
+  return await callback(controller.signal).finally(() => {
+    clearTimeout(timeoutId);
+  });
+};
+
+export const fetchWithTimeoutAsync = async (...[url, requestInit]: Parameters<typeof fetch>) => {
+  return await withTimeoutAsync((signal) => fetch(url, { ...requestInit, signal }));
+};

packages/core/src/infrastructure/logs/index.ts

@@ -15,4 +15,12 @@ interface DefaultMetadata {
 }
 
 export const createLogger = (metadata: DefaultMetadata & Record<string, unknown>) => logger.child(metadata);
-export type Logger = winston.Logger;
+
+type LogMethod = ((message: string, metadata?: Record<string, unknown>) => void) | ((error: unknown) => void);
+
+export interface ILogger {
+  debug: LogMethod;
+  info: LogMethod;
+  warn: LogMethod;
+  error: LogMethod;
+}

packages/core/src/test/infrastructure/http/http-agent.spec.ts

@@ -0,0 +1,103 @@
+import type { Dispatcher } from "undici";
+import { describe, expect, test, vi } from "vitest";
+
+import { UndiciHttpAgent } from "@homarr/core/infrastructure/http";
+
+import { TestLogger } from "../logs";
+
+vi.mock("undici", () => {
+  return {
+    Agent: class Agent {
+      dispatch(_options: Dispatcher.DispatchOptions, _handler: Dispatcher.DispatchHandler): boolean {
+        return true;
+      }
+    },
+    setGlobalDispatcher: () => undefined,
+  };
+});
+
+const REDACTED = "REDACTED";
+
+describe("UndiciHttpAgent should log all requests", () => {
+  test("should log all requests", () => {
+    // Arrange
+    const logger = new TestLogger();
+    const agent = new UndiciHttpAgent({ logger });
+
+    // Act
+    agent.dispatch({ origin: "https://homarr.dev", path: "/", method: "GET" }, {});
+
+    // Assert
+    expect(logger.messages).toContainEqual({
+      level: "debug",
+      message: "Dispatching request https://homarr.dev/ (0 headers)",
+    });
+  });
+
+  test("should show amount of headers", () => {
+    // Arrange
+    const logger = new TestLogger();
+    const agent = new UndiciHttpAgent({ logger });
+
+    // Act
+    agent.dispatch(
+      {
+        origin: "https://homarr.dev",
+        path: "/",
+        method: "GET",
+        headers: {
+          "Content-Type": "text/html",
+          "User-Agent": "Mozilla/5.0",
+        },
+      },
+      {},
+    );
+
+    // Assert
+    expect(logger.messages.at(-1)?.message).toContain("(2 headers)");
+  });
+
+  test.each([
+    ["/?hex=a3815e8ada2ef9a31", `/?hex=${REDACTED}`],
+    ["/?uuid=f7c3f65e-c511-4f90-ba9a-3fd31418bd49", `/?uuid=${REDACTED}`],
+    ["/?password=complexPassword123", `/?password=${REDACTED}`],
+    [
+      // JWT for John Doe
+      "/?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c",
+      `/?jwt=${REDACTED}`,
+    ],
+    ["/?one=a1&two=b2&three=c3", `/?one=${REDACTED}&two=${REDACTED}&three=${REDACTED}`],
+    ["/?numberWith13Chars=1234567890123", `/?numberWith13Chars=${REDACTED}`],
+    [`/?stringWith13Chars=${"a".repeat(13)}`, `/?stringWith13Chars=${REDACTED}`],
+    [`/${"a".repeat(32)}/?param=123`, `/${REDACTED}/?param=123`],
+  ])("should redact sensitive data in url https://homarr.dev%s", (path, expected) => {
+    // Arrange
+    const logger = new TestLogger();
+    const agent = new UndiciHttpAgent({ logger });
+
+    // Act
+    agent.dispatch({ origin: "https://homarr.dev", path, method: "GET" }, {});
+
+    // Assert
+    expect(logger.messages.at(-1)?.message).toContain(` https://homarr.dev${expected} `);
+  });
+  test.each([
+    ["empty", "/?empty"],
+    ["numbers with max 12 chars", "/?number=123456789012"],
+    ["true", "/?true=true"],
+    ["false", "/?false=false"],
+    ["strings with max 12 chars", `/?short=${"a".repeat(12)}`],
+    ["dates", "/?date=2022-01-01"],
+    ["date times", "/?datetime=2022-01-01T00:00:00.000Z"],
+  ])("should not redact values that are %s", (_reason, path) => {
+    // Arrange
+    const logger = new TestLogger();
+    const agent = new UndiciHttpAgent({ logger });
+
+    // Act
+    agent.dispatch({ origin: "https://homarr.dev", path, method: "GET" }, {});
+
+    // Assert
+    expect(logger.messages.at(-1)?.message).toContain(` https://homarr.dev${path} `);
+  });
+});

packages/core/src/test/infrastructure/logs/index.ts

@@ -0,0 +1,49 @@
+import type { ILogger } from "@homarr/core/infrastructure/logs";
+import type { LogLevel } from "@homarr/core/infrastructure/logs/constants";
+
+interface LogMessage {
+  level: LogLevel;
+  message: string;
+  meta?: Record<string, unknown>;
+}
+
+interface LogError {
+  level: LogLevel;
+  error: unknown;
+}
+
+type LogEntry = LogMessage | LogError;
+
+export class TestLogger implements ILogger {
+  public entries: LogEntry[] = [];
+  public get messages(): LogMessage[] {
+    return this.entries.filter((entry) => "message" in entry);
+  }
+  public get errors(): LogError[] {
+    return this.entries.filter((entry) => "error" in entry);
+  }
+
+  private log(level: LogLevel, param1: unknown, param2?: Record<string, unknown>): void {
+    if (typeof param1 === "string") {
+      this.entries.push({ level, message: param1, meta: param2 });
+    } else {
+      this.entries.push({ level, error: param1 });
+    }
+  }
+
+  debug(param1: unknown, param2?: Record<string, unknown>): void {
+    this.log("debug", param1, param2);
+  }
+
+  info(param1: unknown, param2?: Record<string, unknown>): void {
+    this.log("info", param1, param2);
+  }
+
+  warn(param1: unknown, param2?: Record<string, unknown>): void {
+    this.log("warn", param1, param2);
+  }
+
+  error(param1: unknown, param2?: Record<string, unknown>): void {
+    this.log("error", param1, param2);
+  }
+}