jazzymc/homarr
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat: add job on start option for icons updater (#442)

Browse Source
This commit is contained in:
Meier Lukas
2024-05-05 22:20:38 +02:00
committed by GitHub
parent 4b80b16b53
commit 7b31684c84
2 changed files with 79 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
apps/tasks/src/jobs/icons-updater.ts
View File

@@ -8,83 +8,81 @@ import { logger } from "@homarr/log";
import { EVERY_WEEK } from "~/lib/cron-job/constants"; import { EVERY_WEEK } from "~/lib/cron-job/constants";
import { createCronJob } from "~/lib/cron-job/creator"; import { createCronJob } from "~/lib/cron-job/creator";
export const iconsUpdaterJob = createCronJob(EVERY_WEEK).withCallback( export const iconsUpdaterJob = createCronJob(EVERY_WEEK, {
async () => { runOnStart: true,
logger.info(`Updating icon repository cache...`); }).withCallback(async () => {
const stopWatch = new Stopwatch(); logger.info("Updating icon repository cache...");
const repositoryIconGroups = await fetchIconsAsync(); const stopWatch = new Stopwatch();
const countIcons = repositoryIconGroups const repositoryIconGroups = await fetchIconsAsync();
.map((group) => group.icons.length) const countIcons = repositoryIconGroups
.reduce((partialSum, arrayLength) => partialSum + arrayLength, 0); .map((group) => group.icons.length)
logger.info( .reduce((partialSum, arrayLength) => partialSum + arrayLength, 0);
`Successfully fetched ${countIcons} icons from ${repositoryIconGroups.length} repositories within ${stopWatch.getElapsedInHumanWords()}`, logger.info(
); `Successfully fetched ${countIcons} icons from ${repositoryIconGroups.length} repositories within ${stopWatch.getElapsedInHumanWords()}`,
);
const databaseIconGroups = await db.query.iconRepositories.findMany({ const databaseIconGroups = await db.query.iconRepositories.findMany({
with: { with: {
icons: true, icons: true,
}, },
}); });
const skippedChecksums: string[] = []; const skippedChecksums: string[] = [];
let countDeleted = 0; let countDeleted = 0;
let countInserted = 0; let countInserted = 0;
logger.info(`Updating icons in database...`); logger.info("Updating icons in database...");
stopWatch.reset(); stopWatch.reset();
await db.transaction(async (transaction) => { await db.transaction(async (transaction) => {
for (const repositoryIconGroup of repositoryIconGroups) { for (const repositoryIconGroup of repositoryIconGroups) {
if (!repositoryIconGroup.success) { if (!repositoryIconGroup.success) {
continue;
}
const repositoryInDb = databaseIconGroups.find(
(dbIconGroup) => dbIconGroup.slug === repositoryIconGroup.slug,
);
const repositoryIconGroupId: string = repositoryInDb?.id ?? createId();
if (!repositoryInDb?.id) {
await transaction.insert(iconRepositories).values({
id: repositoryIconGroupId,
slug: repositoryIconGroup.slug,
});
}
for (const icon of repositoryIconGroup.icons) {
if (
databaseIconGroups
.flatMap((group) => group.icons)
.some((dbIcon) => dbIcon.checksum === icon.checksum)
) {
skippedChecksums.push(icon.checksum);
continue; continue;
} }
const repositoryInDb = databaseIconGroups.find( await transaction.insert(icons).values({
(dbIconGroup) => dbIconGroup.slug === repositoryIconGroup.slug, id: createId(),
); checksum: icon.checksum,
const repositoryIconGroupId: string = repositoryInDb?.id ?? createId(); name: icon.fileNameWithExtension,
if (!repositoryInDb?.id) { url: icon.imageUrl.href,
await transaction.insert(iconRepositories).values({ iconRepositoryId: repositoryIconGroupId,
id: repositoryIconGroupId, });
slug: repositoryIconGroup.slug, countInserted++;
});
}
for (const icon of repositoryIconGroup.icons) {
if (
databaseIconGroups
.flatMap((group) => group.icons)
.some((dbIcon) => dbIcon.checksum == icon.checksum)
) {
skippedChecksums.push(icon.checksum);
continue;
}
await transaction.insert(icons).values({
id: createId(),
checksum: icon.checksum,
name: icon.fileNameWithExtension,
url: icon.imageUrl.href,
iconRepositoryId: repositoryIconGroupId,
});
countInserted++;
}
} }
}
const deadIcons = databaseIconGroups const deadIcons = databaseIconGroups
.flatMap((group) => group.icons) .flatMap((group) => group.icons)
.filter((icon) => !skippedChecksums.includes(icon.checksum)); .filter((icon) => !skippedChecksums.includes(icon.checksum));
for (const icon of deadIcons) { for (const icon of deadIcons) {
await transaction await transaction.delete(icons).where(eq(icons.checksum, icon.checksum));
.delete(icons) countDeleted++;
.where(eq(icons.checksum, icon.checksum)); }
countDeleted++; });
}
});
logger.info( logger.info(
`Updated database within ${stopWatch.getElapsedInHumanWords()} (-${countDeleted}, +${countInserted})`, `Updated database within ${stopWatch.getElapsedInHumanWords()} (-${countDeleted}, +${countInserted})`,
); );
}, });
);

13
apps/tasks/src/lib/cron-job/creator.ts
View File

@@ -2,9 +2,20 @@ import cron from "node-cron";
import type { MaybePromise } from "@homarr/common/types"; import type { MaybePromise } from "@homarr/common/types";
export const createCronJob = (cronExpression: string) => { interface CreateCronJobOptions {
runOnStart?: boolean;
}
export const createCronJob = (
cronExpression: string,
options: CreateCronJobOptions = { runOnStart: false },
) => {
return { return {
withCallback: (callback: () => MaybePromise<void>) => { withCallback: (callback: () => MaybePromise<void>) => {
if (options.runOnStart) {
void callback();
}
const task = cron.schedule(cronExpression, () => void callback(), { const task = cron.schedule(cronExpression, () => void callback(), {
scheduled: false, scheduled: false,
}); });