refactor(http): move to core package (#4690)

packages/core/src/infrastructure/certificates/hostnames/index.ts

@@ -1,3 +1,5 @@
+import type { InferSelectModel } from "drizzle-orm";
+
 import { createDb } from "../../db";
 import { schema } from "./db/schema";
 
@@ -6,3 +8,5 @@ const db = createDb(schema);
 export const getTrustedCertificateHostnamesAsync = async () => {
   return await db.query.trustedCertificateHostnames.findMany();
 };
+
+export type TrustedCertificateHostname = InferSelectModel<typeof schema.trustedCertificateHostnames>;

packages/core/src/infrastructure/http/http-agent.ts

@@ -0,0 +1,72 @@
+import type { Dispatcher } from "undici";
+import { Agent } from "undici";
+
+import type { ILogger } from "@homarr/core/infrastructure/logs";
+import { createLogger } from "@homarr/core/infrastructure/logs";
+
+// The below import statement initializes dns-caching
+import "@homarr/core/infrastructure/dns/init";
+
+interface HttpAgentOptions extends Agent.Options {
+  logger?: ILogger;
+}
+
+export class UndiciHttpAgent extends Agent {
+  private logger: ILogger;
+
+  constructor(props?: HttpAgentOptions) {
+    super(props);
+
+    this.logger = props?.logger ?? createLogger({ module: "httpAgent" });
+  }
+
+  dispatch(options: Dispatcher.DispatchOptions, handler: Dispatcher.DispatchHandler): boolean {
+    this.logRequestDispatch(options);
+    return super.dispatch(options, handler);
+  }
+
+  private logRequestDispatch(options: Dispatcher.DispatchOptions) {
+    const path = this.redactPathParams(options.path);
+    let url = new URL(`${options.origin as string}${path}`);
+    url = this.redactSearchParams(url);
+
+    this.logger.debug(
+      `Dispatching request ${url.toString().replaceAll("=&", "&")} (${Object.keys(options.headers ?? {}).length} headers)`,
+    );
+  }
+
+  /**
+   * Redact path parameters that are longer than 32 characters
+   * This is to prevent sensitive data from being logged
+   * @param path path of the request
+   * @returns redacted path
+   */
+  private redactPathParams(path: string): string {
+    return path
+      .split("/")
+      .map((segment) => (segment.length >= 32 && !segment.startsWith("?") ? "REDACTED" : segment))
+      .join("/");
+  }
+
+  /**
+   * Redact sensitive search parameters from the URL.
+   * It allows certain patterns to remain unredacted.
+   * Like small numbers, booleans, short strings, dates, and date-times.
+   * Some integrations use query parameters for auth.
+   * @param url URL object of the request
+   * @returns redacted URL object
+   */
+  private redactSearchParams(url: URL): URL {
+    url.searchParams.forEach((value, key) => {
+      if (value === "") return; // Skip empty values
+      if (/^-?\d{1,12}$/.test(value)) return; // Skip small numbers
+      if (value === "true" || value === "false") return; // Skip boolean values
+      if (/^[a-zA-Z]{1,12}$/.test(value)) return; // Skip short strings
+      if (/^\d{4}-\d{2}-\d{2}$/.test(value)) return; // Skip dates
+      if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(value)) return; // Skip date times
+
+      url.searchParams.set(key, "REDACTED");
+    });
+    return url;
+  }
+}

packages/core/src/infrastructure/http/index.ts

@@ -0,0 +1,8 @@
+export { UndiciHttpAgent } from "./http-agent";
+export {
+  createAxiosCertificateInstanceAsync,
+  createCertificateAgentAsync,
+  createCustomCheckServerIdentity,
+  createHttpsAgentAsync,
+  fetchWithTrustedCertificatesAsync,
+} from "./request";

packages/core/src/infrastructure/http/request.ts

@@ -0,0 +1,82 @@
+import type { AgentOptions } from "node:https";
+import { Agent as HttpsAgent } from "node:https";
+import { checkServerIdentity } from "node:tls";
+import axios from "axios";
+import type { RequestInfo, RequestInit, Response } from "undici";
+import { fetch } from "undici";
+
+import {
+  getAllTrustedCertificatesAsync,
+  getTrustedCertificateHostnamesAsync,
+} from "@homarr/core/infrastructure/certificates";
+import { UndiciHttpAgent } from "@homarr/core/infrastructure/http";
+
+import type { TrustedCertificateHostname } from "../certificates/hostnames";
+import { withTimeoutAsync } from "./timeout";
+
+export const createCustomCheckServerIdentity = (
+  trustedHostnames: TrustedCertificateHostname[],
+): typeof checkServerIdentity => {
+  return (hostname, peerCertificate) => {
+    const matchingTrustedHostnames = trustedHostnames.filter(
+      (cert) => cert.thumbprint === peerCertificate.fingerprint256,
+    );
+
+    // We trust the certificate if we have a matching hostname
+    if (matchingTrustedHostnames.some((cert) => cert.hostname === hostname)) return undefined;
+
+    return checkServerIdentity(hostname, peerCertificate);
+  };
+};
+
+export const createCertificateAgentAsync = async (override?: {
+  ca: string | string[];
+  checkServerIdentity: typeof checkServerIdentity;
+}) => {
+  return new UndiciHttpAgent({
+    connect: override ?? {
+      ca: await getAllTrustedCertificatesAsync(),
+      checkServerIdentity: createCustomCheckServerIdentity(await getTrustedCertificateHostnamesAsync()),
+    },
+  });
+};
+
+export const createHttpsAgentAsync = async (override?: Pick<AgentOptions, "ca" | "checkServerIdentity">) => {
+  return new HttpsAgent(
+    override ?? {
+      ca: await getAllTrustedCertificatesAsync(),
+      checkServerIdentity: createCustomCheckServerIdentity(await getTrustedCertificateHostnamesAsync()),
+    },
+  );
+};
+
+export const createAxiosCertificateInstanceAsync = async (
+  override?: Pick<AgentOptions, "ca" | "checkServerIdentity">,
+) => {
+  return axios.create({
+    httpsAgent: await createHttpsAgentAsync(override),
+  });
+};
+
+export const fetchWithTrustedCertificatesAsync = async (
+  url: RequestInfo,
+  options?: RequestInit & { timeout?: number },
+): Promise<Response> => {
+  const agent = await createCertificateAgentAsync(undefined);
+  if (options?.timeout) {
+    return await withTimeoutAsync(
+      async (signal) =>
+        fetch(url, {
+          ...options,
+          signal,
+          dispatcher: agent,
+        }),
+      options.timeout,
+    );
+  }
+
+  return fetch(url, {
+    ...options,
+    dispatcher: agent,
+  });
+};

packages/core/src/infrastructure/http/timeout.ts

@@ -0,0 +1,19 @@
+import type { Response as UndiciResponse } from "undici";
+
+// https://stackoverflow.com/questions/46946380/fetch-api-request-timeout
+export const withTimeoutAsync = async <TResponse extends Response | UndiciResponse>(
+  callback: (signal: AbortSignal) => Promise<TResponse>,
+  timeout = 10000,
+) => {
+  const controller = new AbortController();
+
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+
+  return await callback(controller.signal).finally(() => {
+    clearTimeout(timeoutId);
+  });
+};
+
+export const fetchWithTimeoutAsync = async (...[url, requestInit]: Parameters<typeof fetch>) => {
+  return await withTimeoutAsync((signal) => fetch(url, { ...requestInit, signal }));
+};

packages/core/src/infrastructure/logs/index.ts

@@ -15,4 +15,12 @@ interface DefaultMetadata {
 }
 
 export const createLogger = (metadata: DefaultMetadata & Record<string, unknown>) => logger.child(metadata);
-export type Logger = winston.Logger;
+
+type LogMethod = ((message: string, metadata?: Record<string, unknown>) => void) | ((error: unknown) => void);
+
+export interface ILogger {
+  debug: LogMethod;
+  info: LogMethod;
+  warn: LogMethod;
+  error: LogMethod;
+}