feat: add onboarding with oldmarr import (#1606)

packages/old-import/src/import/collections/board-collection.ts

@@ -0,0 +1,78 @@
+import { createId } from "@homarr/db";
+import { logger } from "@homarr/log";
+
+import { fixSectionIssues } from "../../fix-section-issues";
+import { mapBoard } from "../../mappers/map-board";
+import { moveWidgetsAndAppsIfMerge } from "../../move-widgets-and-apps-merge";
+import { prepareItems } from "../../prepare/prepare-items";
+import type { prepareMultipleImports } from "../../prepare/prepare-multiple";
+import { prepareSections } from "../../prepare/prepare-sections";
+import type { InitialOldmarrImportSettings } from "../../settings";
+import { createDbInsertCollection } from "./common";
+
+export const createBoardInsertCollection = (
+  { preparedApps, preparedBoards }: Omit<ReturnType<typeof prepareMultipleImports>, "preparedIntegrations">,
+  settings: InitialOldmarrImportSettings,
+) => {
+  const insertCollection = createDbInsertCollection(["apps", "boards", "sections", "items"]);
+  logger.info("Preparing boards for insert collection");
+
+  const appsMap = new Map(
+    preparedApps.flatMap(({ ids, ...app }) => {
+      const id = app.existingId ?? createId();
+      return ids.map((oldId) => [oldId, { id, ...app }] as const);
+    }),
+  );
+
+  for (const app of appsMap.values()) {
+    // Skip duplicate apps
+    if (insertCollection.apps.some((appEntry) => appEntry.id === app.id)) {
+      continue;
+    }
+    // Skip apps that already exist in the database
+    if (app.existingId) {
+      continue;
+    }
+
+    insertCollection.apps.push(app);
+  }
+
+  if (settings.onlyImportApps) {
+    logger.info(
+      `Skipping boards and sections import due to onlyImportApps setting appCount=${insertCollection.apps.length}`,
+    );
+    return insertCollection;
+  }
+  logger.debug(`Added apps to board insert collection count=${insertCollection.apps.length}`);
+
+  preparedBoards.forEach((board) => {
+    const { wrappers, categories, wrapperIdsToMerge } = fixSectionIssues(board.config);
+    const { apps, widgets } = moveWidgetsAndAppsIfMerge(board.config, wrapperIdsToMerge, {
+      ...settings,
+      screenSize: board.size,
+      name: board.name,
+    });
+
+    logger.debug(`Fixed issues with sections and item positions fileName=${board.name}`);
+
+    const mappedBoard = mapBoard(board);
+    logger.debug(`Mapped board fileName=${board.name} boardId=${mappedBoard.id}`);
+    insertCollection.boards.push(mappedBoard);
+    const preparedSections = prepareSections(mappedBoard.id, { wrappers, categories });
+
+    for (const section of preparedSections.values()) {
+      insertCollection.sections.push(section);
+    }
+    logger.debug(`Added sections to board insert collection count=${insertCollection.sections.length}`);
+
+    const preparedItems = prepareItems({ apps, widgets }, board.size, appsMap, preparedSections);
+    preparedItems.forEach((item) => insertCollection.items.push(item));
+    logger.debug(`Added items to board insert collection count=${insertCollection.items.length}`);
+  });
+
+  logger.info(
+    `Board collection prepared boardCount=${insertCollection.boards.length} sectionCount=${insertCollection.sections.length} itemCount=${insertCollection.items.length} appCount=${insertCollection.apps.length}`,
+  );
+
+  return insertCollection;
+};

packages/old-import/src/import/collections/common.ts

@@ -0,0 +1,33 @@
+import { objectEntries } from "@homarr/common";
+import type { Database, InferInsertModel } from "@homarr/db";
+import { schema } from "@homarr/db";
+
+type TableKey = {
+  [K in keyof typeof schema]: (typeof schema)[K] extends { _: { brand: "Table" } } ? K : never;
+}[keyof typeof schema];
+
+export const createDbInsertCollection = <TTableKey extends TableKey>(tablesInInsertOrder: TTableKey[]) => {
+  const context = tablesInInsertOrder.reduce(
+    (acc, key) => {
+      acc[key] = [];
+      return acc;
+    },
+    {} as { [K in TTableKey]: InferInsertModel<(typeof schema)[K]>[] },
+  );
+
+  return {
+    ...context,
+    insertAll: (db: Database) => {
+      db.transaction((transaction) => {
+        for (const [key, values] of objectEntries(context)) {
+          if (values.length >= 1) {
+            transaction
+              .insert(schema[key])
+              .values(values as never)
+              .run();
+          }
+        }
+      });
+    },
+  };
+};

packages/old-import/src/import/collections/integration-collection.ts

@@ -0,0 +1,47 @@
+import { encryptSecret } from "@homarr/common/server";
+import { logger } from "@homarr/log";
+
+import { mapAndDecryptIntegrations } from "../../mappers/map-integration";
+import type { PreparedIntegration } from "../../prepare/prepare-integrations";
+import { createDbInsertCollection } from "./common";
+
+export const createIntegrationInsertCollection = (
+  preparedIntegrations: PreparedIntegration[],
+  encryptionToken: string | null,
+) => {
+  const insertCollection = createDbInsertCollection(["integrations", "integrationSecrets"]);
+  logger.info(`Preparing integrations for insert collection count=${preparedIntegrations.length}`);
+
+  if (encryptionToken === null) {
+    logger.debug("Skipping integration decryption due to missing token");
+    return insertCollection;
+  }
+
+  const preparedIntegrationsDecrypted = mapAndDecryptIntegrations(preparedIntegrations, encryptionToken);
+
+  preparedIntegrationsDecrypted.forEach((integration) => {
+    insertCollection.integrations.push({
+      id: integration.id,
+      kind: integration.kind,
+      name: integration.name,
+      url: integration.url,
+    });
+
+    integration.secrets
+      .filter((secret) => secret.value !== null)
+      .forEach((secret) => {
+        insertCollection.integrationSecrets.push({
+          integrationId: integration.id,
+          kind: secret.field,
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          value: encryptSecret(secret.value!),
+        });
+      });
+  });
+
+  logger.info(
+    `Added integrations and secrets to insert collection integrationCount=${insertCollection.integrations.length} secretCount=${insertCollection.integrationSecrets.length}`,
+  );
+
+  return insertCollection;
+};

packages/old-import/src/import/collections/user-collection.ts

@@ -0,0 +1,53 @@
+import { createId } from "@homarr/db";
+import { credentialsAdminGroup } from "@homarr/definitions";
+import { logger } from "@homarr/log";
+
+import { mapAndDecryptUsers } from "../../mappers/map-user";
+import type { OldmarrImportUser } from "../../user-schema";
+import { createDbInsertCollection } from "./common";
+
+export const createUserInsertCollection = (importUsers: OldmarrImportUser[], encryptionToken: string | null) => {
+  const insertCollection = createDbInsertCollection(["users", "groups", "groupMembers", "groupPermissions"]);
+
+  logger.info(`Preparing users for insert collection count=${importUsers.length}`);
+
+  if (encryptionToken === null) {
+    logger.debug("Skipping user decryption due to missing token");
+    return insertCollection;
+  }
+
+  const preparedUsers = mapAndDecryptUsers(importUsers, encryptionToken);
+  preparedUsers.forEach((user) => insertCollection.users.push(user));
+  logger.debug(`Added users to insert collection count=${insertCollection.users.length}`);
+
+  if (!preparedUsers.some((user) => user.isAdmin)) {
+    logger.warn("No admin users found, skipping admin group creation");
+    return insertCollection;
+  }
+
+  const adminGroupId = createId();
+  insertCollection.groups.push({
+    id: adminGroupId,
+    name: credentialsAdminGroup,
+  });
+
+  insertCollection.groupPermissions.push({
+    groupId: adminGroupId,
+    permission: "admin",
+  });
+
+  const admins = preparedUsers.filter((user) => user.isAdmin);
+
+  admins.forEach((user) => {
+    insertCollection.groupMembers.push({
+      groupId: adminGroupId,
+      userId: user.id,
+    });
+  });
+
+  logger.info(
+    `Added admin group and permissions to insert collection adminGroupId=${adminGroupId} adminUsersCount=${admins.length}`,
+  );
+
+  return insertCollection;
+};

packages/old-import/src/import/import-initial-oldmarr.ts

@@ -0,0 +1,45 @@
+import type { z } from "zod";
+
+import { Stopwatch } from "@homarr/common";
+import type { Database } from "@homarr/db";
+import { logger } from "@homarr/log";
+
+import { analyseOldmarrImportAsync } from "../analyse/analyse-oldmarr-import";
+import { prepareMultipleImports } from "../prepare/prepare-multiple";
+import { createBoardInsertCollection } from "./collections/board-collection";
+import { createIntegrationInsertCollection } from "./collections/integration-collection";
+import { createUserInsertCollection } from "./collections/user-collection";
+import type { importInitialOldmarrInputSchema } from "./input";
+import { ensureValidTokenOrThrow } from "./validate-token";
+
+export const importInitialOldmarrAsync = async (
+  db: Database,
+  input: z.infer<typeof importInitialOldmarrInputSchema>,
+) => {
+  const stopwatch = new Stopwatch();
+  const { checksum, configs, users: importUsers } = await analyseOldmarrImportAsync(input.file);
+  ensureValidTokenOrThrow(checksum, input.token);
+
+  const { preparedApps, preparedBoards, preparedIntegrations } = prepareMultipleImports(
+    configs,
+    input.settings,
+    input.boardSelections,
+  );
+
+  logger.info("Preparing import data in insert collections for database");
+
+  const boardInsertCollection = createBoardInsertCollection({ preparedApps, preparedBoards }, input.settings);
+  const userInsertCollection = createUserInsertCollection(importUsers, input.token);
+  const integrationInsertCollection = createIntegrationInsertCollection(preparedIntegrations, input.token);
+
+  logger.info("Inserting import data to database");
+
+  // Due to a limitation with better-sqlite it's only possible to use it synchronously
+  db.transaction((transaction) => {
+    boardInsertCollection.insertAll(transaction);
+    userInsertCollection.insertAll(transaction);
+    integrationInsertCollection.insertAll(transaction);
+  });
+
+  logger.info(`Import successful (in ${stopwatch.getElapsedInHumanWords()})`);
+};

packages/old-import/src/import/import-single-oldmarr.ts

@@ -0,0 +1,36 @@
+import { inArray } from "@homarr/db";
+import type { Database } from "@homarr/db";
+import { apps } from "@homarr/db/schema/sqlite";
+import type { OldmarrConfig } from "@homarr/old-schema";
+
+import { doAppsMatch } from "../prepare/prepare-apps";
+import { prepareSingleImport } from "../prepare/prepare-single";
+import type { OldmarrImportConfiguration } from "../settings";
+import { createBoardInsertCollection } from "./collections/board-collection";
+
+export const importSingleOldmarrConfigAsync = async (
+  db: Database,
+  config: OldmarrConfig,
+  settings: OldmarrImportConfiguration,
+) => {
+  const { preparedApps, preparedBoards } = prepareSingleImport(config, settings);
+  const existingApps = await db.query.apps.findMany({
+    where: inArray(
+      apps.href,
+      preparedApps.map((app) => app.href).filter((href) => href !== null),
+    ),
+  });
+
+  preparedApps.forEach((app) => {
+    const existingApp = existingApps.find((existingApp) => doAppsMatch(existingApp, app));
+    if (existingApp) {
+      app.existingId = existingApp.id;
+    }
+    return app;
+  });
+
+  const boardInsertCollection = createBoardInsertCollection({ preparedApps, preparedBoards }, settings);
+
+  // Due to a limitation with better-sqlite it's only possible to use it synchronously
+  boardInsertCollection.insertAll(db);
+};

packages/old-import/src/import/index.ts

@@ -0,0 +1,3 @@
+export { importInitialOldmarrAsync } from "./import-initial-oldmarr";
+export * from "./input";
+export { ensureValidTokenOrThrow } from "./validate-token";

packages/old-import/src/import/input.ts

@@ -0,0 +1,24 @@
+import SuperJSON from "superjson";
+import { z } from "zod";
+import { zfd } from "zod-form-data";
+
+import { initialOldmarrImportSettings } from "../settings";
+
+const boardSelectionMapSchema = z.map(
+  z.string(),
+  z.object({
+    sm: z.boolean().nullable(),
+    md: z.boolean().nullable(),
+    lg: z.boolean().nullable(),
+  }),
+);
+
+export const importInitialOldmarrInputSchema = zfd.formData({
+  file: zfd.file(),
+  settings: zfd.json(initialOldmarrImportSettings),
+  boardSelections: zfd.text().transform((value) => {
+    const map = boardSelectionMapSchema.parse(SuperJSON.parse(value));
+    return map;
+  }),
+  token: zfd.text().nullable(),
+});

packages/old-import/src/import/validate-token.ts

@@ -0,0 +1,18 @@
+import { decryptSecretWithKey } from "@homarr/common/server";
+
+export const ensureValidTokenOrThrow = (checksum: string | undefined, encryptionToken: string | null) => {
+  if (!encryptionToken || !checksum) return;
+
+  const [first, second] = checksum.split("\n");
+  if (!first || !second) throw new Error("Malformed checksum");
+
+  const key = Buffer.from(encryptionToken, "hex");
+  let decrypted: string;
+  try {
+    decrypted = decryptSecretWithKey(second as `${string}.${string}`, key);
+  } catch {
+    throw new Error("Invalid checksum");
+  }
+  const isValid = decrypted === first;
+  if (!isValid) throw new Error("Invalid checksum");
+};