perf: improve performance of icon updater from 10s to 300ms (#502)

apps/tasks/src/jobs/icons-updater.ts

@@ -1,5 +1,6 @@
 import { Stopwatch } from "@homarr/common";
-import { db, eq } from "@homarr/db";
+import type { InferInsertModel } from "@homarr/db";
+import { db, inArray } from "@homarr/db";
 import { createId } from "@homarr/db/client";
 import { iconRepositories, icons } from "@homarr/db/schema/sqlite";
 import { fetchIconsAsync } from "@homarr/icons";
@@ -34,52 +35,67 @@ export const iconsUpdaterJob = createCronJob(EVERY_WEEK, {
   logger.info("Updating icons in database...");
   stopWatch.reset();
 
-  await db.transaction(async (transaction) => {
+  const newIconRepositories: InferInsertModel<typeof iconRepositories>[] = [];
-    for (const repositoryIconGroup of repositoryIconGroups) {
+  const newIcons: InferInsertModel<typeof icons>[] = [];
-      if (!repositoryIconGroup.success) {
+
+  for (const repositoryIconGroup of repositoryIconGroups) {
+    if (!repositoryIconGroup.success) {
+      continue;
+    }
+
+    const repositoryInDb = databaseIconGroups.find(
+      (dbIconGroup) => dbIconGroup.slug === repositoryIconGroup.slug,
+    );
+    const repositoryIconGroupId: string = repositoryInDb?.id ?? createId();
+    if (!repositoryInDb?.id) {
+      newIconRepositories.push({
+        id: repositoryIconGroupId,
+        slug: repositoryIconGroup.slug,
+      });
+    }
+
+    for (const icon of repositoryIconGroup.icons) {
+      if (
+        databaseIconGroups
+          .flatMap((group) => group.icons)
+          .some((dbIcon) => dbIcon.checksum === icon.checksum)
+      ) {
+        skippedChecksums.push(icon.checksum);
         continue;
       }
 
-      const repositoryInDb = databaseIconGroups.find(
+      newIcons.push({
-        (dbIconGroup) => dbIconGroup.slug === repositoryIconGroup.slug,
+        id: createId(),
-      );
+        checksum: icon.checksum,
-      const repositoryIconGroupId: string = repositoryInDb?.id ?? createId();
+        name: icon.fileNameWithExtension,
-      if (!repositoryInDb?.id) {
+        url: icon.imageUrl.href,
-        await transaction.insert(iconRepositories).values({
+        iconRepositoryId: repositoryIconGroupId,
-          id: repositoryIconGroupId,
+      });
-          slug: repositoryIconGroup.slug,
+      countInserted++;
-        });
+    }
-      }
+  }
 
-      for (const icon of repositoryIconGroup.icons) {
+  const deadIcons = databaseIconGroups
-        if (
+    .flatMap((group) => group.icons)
-          databaseIconGroups
+    .filter((icon) => !skippedChecksums.includes(icon.checksum));
-            .flatMap((group) => group.icons)
-            .some((dbIcon) => dbIcon.checksum === icon.checksum)
-        ) {
-          skippedChecksums.push(icon.checksum);
-          continue;
-        }
 
-        await transaction.insert(icons).values({
+  await db.transaction(async (transaction) => {
-          id: createId(),
+    if (newIconRepositories.length >= 1) {
-          checksum: icon.checksum,
+      await transaction.insert(iconRepositories).values(newIconRepositories);
-          name: icon.fileNameWithExtension,
-          url: icon.imageUrl.href,
-          iconRepositoryId: repositoryIconGroupId,
-        });
-        countInserted++;
-      }
     }
 
-    const deadIcons = databaseIconGroups
+    if (newIcons.length >= 1) {
-      .flatMap((group) => group.icons)
+      await transaction.insert(icons).values(newIcons);
-      .filter((icon) => !skippedChecksums.includes(icon.checksum));
-
-    for (const icon of deadIcons) {
-      await transaction.delete(icons).where(eq(icons.checksum, icon.checksum));
-      countDeleted++;
     }
+    await transaction.delete(icons).where(
+      deadIcons.length >= 1
+        ? inArray(
+            icons.checksum,
+            deadIcons.map((icon) => icon.checksum),
+          )
+        : undefined,
+    );
+    countDeleted += deadIcons.length;
   });
 
   logger.info(