perf: improve performance of icon updater from 10s to 300ms (#502)

This commit is contained in:
Meier Lukas
2024-05-16 17:06:03 +02:00
committed by GitHub
parent f329e2a393
commit 333dc571fa

View File

@@ -1,5 +1,6 @@
import { Stopwatch } from "@homarr/common"; import { Stopwatch } from "@homarr/common";
import { db, eq } from "@homarr/db"; import type { InferInsertModel } from "@homarr/db";
import { db, inArray } from "@homarr/db";
import { createId } from "@homarr/db/client"; import { createId } from "@homarr/db/client";
import { iconRepositories, icons } from "@homarr/db/schema/sqlite"; import { iconRepositories, icons } from "@homarr/db/schema/sqlite";
import { fetchIconsAsync } from "@homarr/icons"; import { fetchIconsAsync } from "@homarr/icons";
@@ -34,7 +35,9 @@ export const iconsUpdaterJob = createCronJob(EVERY_WEEK, {
logger.info("Updating icons in database..."); logger.info("Updating icons in database...");
stopWatch.reset(); stopWatch.reset();
await db.transaction(async (transaction) => { const newIconRepositories: InferInsertModel<typeof iconRepositories>[] = [];
const newIcons: InferInsertModel<typeof icons>[] = [];
for (const repositoryIconGroup of repositoryIconGroups) { for (const repositoryIconGroup of repositoryIconGroups) {
if (!repositoryIconGroup.success) { if (!repositoryIconGroup.success) {
continue; continue;
@@ -45,7 +48,7 @@ export const iconsUpdaterJob = createCronJob(EVERY_WEEK, {
); );
const repositoryIconGroupId: string = repositoryInDb?.id ?? createId(); const repositoryIconGroupId: string = repositoryInDb?.id ?? createId();
if (!repositoryInDb?.id) { if (!repositoryInDb?.id) {
await transaction.insert(iconRepositories).values({ newIconRepositories.push({
id: repositoryIconGroupId, id: repositoryIconGroupId,
slug: repositoryIconGroup.slug, slug: repositoryIconGroup.slug,
}); });
@@ -61,7 +64,7 @@ export const iconsUpdaterJob = createCronJob(EVERY_WEEK, {
continue; continue;
} }
await transaction.insert(icons).values({ newIcons.push({
id: createId(), id: createId(),
checksum: icon.checksum, checksum: icon.checksum,
name: icon.fileNameWithExtension, name: icon.fileNameWithExtension,
@@ -76,10 +79,23 @@ export const iconsUpdaterJob = createCronJob(EVERY_WEEK, {
.flatMap((group) => group.icons) .flatMap((group) => group.icons)
.filter((icon) => !skippedChecksums.includes(icon.checksum)); .filter((icon) => !skippedChecksums.includes(icon.checksum));
for (const icon of deadIcons) { await db.transaction(async (transaction) => {
await transaction.delete(icons).where(eq(icons.checksum, icon.checksum)); if (newIconRepositories.length >= 1) {
countDeleted++; await transaction.insert(iconRepositories).values(newIconRepositories);
} }
if (newIcons.length >= 1) {
await transaction.insert(icons).values(newIcons);
}
await transaction.delete(icons).where(
deadIcons.length >= 1
? inArray(
icons.checksum,
deadIcons.map((icon) => icon.checksum),
)
: undefined,
);
countDeleted += deadIcons.length;
}); });
logger.info( logger.info(