config: migrate from nestjs to tasks and websocket server in docker (#316)

* feat: make tasks script run in docker

* feat: make websocket server work in docker

* fix: format issue

* fix: broken lockfile

* fix: non matching typescript versions

Dockerfile

@@ -1,14 +1,16 @@
 FROM node:20-alpine AS base
 
 FROM base AS builder
-RUN apk add --no-cache libc6-compat redis
+RUN apk add --no-cache libc6-compat
 RUN apk update
 # Set working directory
 WORKDIR /app
 COPY . .
 RUN npm i -g turbo
 RUN turbo prune @homarr/nextjs --docker --out-dir ./next-out
-RUN turbo prune @homarr/nest --docker --out-dir ./nest-out
+RUN turbo prune @homarr/tasks --docker --out-dir ./tasks-out
+RUN turbo prune @homarr/websocket --docker --out-dir ./websocket-out
+RUN turbo prune @homarr/db --docker --out-dir ./migration-out
 
 # Add lockfile and package.json's of isolated subworkspace
 FROM base AS installer
@@ -19,35 +21,35 @@ WORKDIR /app
 # First install the dependencies (as they change less often)
 COPY .gitignore .gitignore
 
-COPY --from=builder /app/nest-out/json/ .
+COPY --from=builder /app/tasks-out/json/ .
-COPY --from=builder /app/nest-out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=builder /app/tasks-out/pnpm-lock.yaml ./pnpm-lock.yaml
-RUN corepack enable pnpm && pnpm install --prod
+RUN corepack enable pnpm && pnpm install
-RUN rm -rf node_modules/next
-RUN rm -rf node_modules/typescript
-RUN rm -rf node_modules/@babel
-RUN rm -rf node_modules/esbuild
-RUN rm -rf node_modules/@esbuild
-RUN rm -rf node_modules/@typescript-eslint
-RUN rm -rf node_modules/prettier
-RUN rm -rf node_modules/webpack
-RUN rm -rf node_modules/eslint
-RUN rm -rf node_modules/@swc
-RUN mv node_modules ./temp_node_modules
 
+COPY --from=builder /app/websocket-out/json/ .
+COPY --from=builder /app/websocket-out/pnpm-lock.yaml ./pnpm-lock.yaml
+RUN corepack enable pnpm && pnpm install
+
+COPY --from=builder /app/migration-out/json/ .
+COPY --from=builder /app/migration-out/pnpm-lock.yaml ./pnpm-lock.yaml
+RUN corepack enable pnpm && pnpm install
 
 COPY --from=builder /app/next-out/json/ .
 COPY --from=builder /app/next-out/pnpm-lock.yaml ./pnpm-lock.yaml
 RUN corepack enable pnpm && pnpm install
 
 # Build the project
-COPY --from=builder /app/nest-out/full/ .
+COPY --from=builder /app/tasks-out/full/ .
+COPY --from=builder /app/websocket-out/full/ .
 COPY --from=builder /app/next-out/full/ .
+COPY --from=builder /app/migration-out/full/ .
 ARG SKIP_ENV_VALIDATION=true
 RUN corepack enable pnpm && pnpm turbo run build
 
 FROM base AS runner
 WORKDIR /app
 
+RUN apk add --no-cache redis
+
 # Don't run production as root
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
@@ -56,8 +58,12 @@ USER nextjs
 COPY --from=installer /app/apps/nextjs/next.config.mjs .
 COPY --from=installer /app/apps/nextjs/package.json .
 
-COPY --from=installer /app/temp_node_modules ./node_modules
+COPY --from=installer --chown=nextjs:nodejs /app/apps/tasks/tasks.cjs ./apps/tasks/tasks.cjs
-COPY --from=installer --chown=nextjs:nodejs /app/apps/nestjs/dist ./apps/nestjs/dist
+COPY --from=installer --chown=nextjs:nodejs /app/apps/websocket/wssServer.cjs ./apps/websocket/wssServer.cjs
+COPY --from=installer --chown=nextjs:nodejs /app/node_modules/better-sqlite3/build/Release/better_sqlite3.node /app/build/better_sqlite3.node
+
+COPY --from=installer --chown=nextjs:nodejs /app/packages/db/migrations ./db/migrations
+COPY --from=installer --chown=nextjs:nodejs /app/packages/db/migrate.cjs ./db/migrate.cjs
 
 # Automatically leverage output traces to reduce image size
 # https://nextjs.org/docs/advanced-features/output-file-tracing
@@ -65,8 +71,6 @@ COPY --from=installer --chown=nextjs:nodejs /app/apps/nextjs/.next/standalone ./
 COPY --from=installer --chown=nextjs:nodejs /app/apps/nextjs/.next/static ./apps/nextjs/.next/static
 COPY --from=installer --chown=nextjs:nodejs /app/apps/nextjs/public ./apps/nextjs/public
 COPY --chown=nextjs:nodejs scripts/run.sh ./run.sh
-COPY --chown=nextjs:nodejs packages/db/migrations ./db/migrations
-COPY --chown=nextjs:nodejs packages/db/migrate.mjs ./db/migrate.mjs
 
 ENV DB_URL='/app/db/db.sqlite'
 

apps/tasks/package.json

@@ -11,6 +11,7 @@
   "type": "module",
   "scripts": {
     "dev": "pnpm with-env tsx ./src/main.ts",
+    "build": "esbuild src/main.ts --bundle --platform=node --outfile=tasks.cjs",
     "clean": "rm -rf .turbo node_modules",
     "lint": "eslint .",
     "format": "prettier --check . --ignore-path ../../.gitignore",
@@ -20,10 +21,10 @@
   "dependencies": {
     "@homarr/common": "workspace:^0.1.0",
     "@homarr/db": "workspace:^0.1.0",
-    "@homarr/redis": "workspace:^0.1.0",
     "@homarr/definitions": "workspace:^0.1.0",
-    "@homarr/validation": "workspace:^0.1.0",
     "@homarr/log": "workspace:^",
+    "@homarr/redis": "workspace:^0.1.0",
+    "@homarr/validation": "workspace:^0.1.0",
     "dotenv": "^16.4.5",
     "node-cron": "^3.0.3"
   },

apps/websocket/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@homarr/websocket",
+  "version": "0.1.0",
+  "private": true,
+  "main": "./src/main.ts",
+  "types": "./src/main.ts",
+  "license": "MIT",
+  "type": "module",
+  "scripts": {
+    "dev": "pnpm with-env tsx ./src/main.ts",
+    "build": "esbuild src/main.ts --bundle --platform=node --outfile=wssServer.cjs --external:bcrypt --loader:.html=text",
+    "clean": "rm -rf .turbo node_modules",
+    "lint": "eslint .",
+    "format": "prettier --check . --ignore-path ../../.gitignore",
+    "typecheck": "tsc --noEmit",
+    "with-env": "dotenv -e ../../.env --"
+  },
+  "dependencies": {
+    "@homarr/api": "workspace:^0.1.0",
+    "@homarr/auth": "workspace:^0.1.0",
+    "@homarr/common": "workspace:^0.1.0",
+    "@homarr/db": "workspace:^0.1.0",
+    "@homarr/definitions": "workspace:^0.1.0",
+    "@homarr/log": "workspace:^",
+    "@homarr/redis": "workspace:^0.1.0",
+    "@homarr/validation": "workspace:^0.1.0",
+    "ws": "^8.16.0",
+    "dotenv": "^16.4.5"
+  },
+  "devDependencies": {
+    "@homarr/eslint-config": "workspace:^0.2.0",
+    "@homarr/prettier-config": "workspace:^0.1.0",
+    "@homarr/tsconfig": "workspace:^0.1.0",
+    "@types/ws": "^8.5.10",
+    "eslint": "^8.57.0",
+    "prettier": "^3.2.5",
+    "typescript": "^5.4.4"
+  },
+  "eslintConfig": {
+    "root": true,
+    "extends": [
+      "@homarr/eslint-config/base"
+    ]
+  },
+  "prettier": "@homarr/prettier-config"
+}

apps/websocket/src/main.ts

@@ -1,14 +1,12 @@
 import { applyWSSHandler } from "@trpc/server/adapters/ws";
 import { WebSocketServer } from "ws";
 
+import { appRouter, createTRPCContext } from "@homarr/api/websocket";
 import { getSessionFromToken, sessionTokenCookieName } from "@homarr/auth";
 import { parseCookies } from "@homarr/common";
 import { db } from "@homarr/db";
 import { logger } from "@homarr/log";
 
-import { appRouter } from "./root";
-import { createTRPCContext } from "./trpc";
-
 const wss = new WebSocketServer({
   port: 3001,
 });

apps/websocket/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "@homarr/tsconfig/base.json",
+  "compilerOptions": {
+    "baseUrl": ".",
+    "paths": {
+      "~/*": ["src/*"]
+    },
+    "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
+  },
+  "include": ["."],
+  "exclude": ["node_modules"]
+}

packages/api/package.json

@@ -4,7 +4,8 @@
   "exports": {
     ".": "./src/index.ts",
     "./client": "./src/client.ts",
-    "./server": "./src/server.ts"
+    "./server": "./src/server.ts",
+    "./websocket": "./src/websocket.ts"
   },
   "private": true,
   "main": "./index.ts",
@@ -12,12 +13,10 @@
   "license": "MIT",
   "type": "module",
   "scripts": {
-    "dev": "pnpm with-env tsx ./src/wssDevServer.ts",
     "clean": "rm -rf .turbo node_modules",
     "lint": "eslint .",
     "format": "prettier --check . --ignore-path ../../.gitignore",
-    "typecheck": "tsc --noEmit",
+    "typecheck": "tsc --noEmit"
-    "with-env": "dotenv -e ../../.env --"
   },
   "dependencies": {
     "@homarr/auth": "workspace:^0.1.0",
@@ -30,14 +29,12 @@
     "@homarr/validation": "workspace:^0.1.0",
     "@trpc/client": "next",
     "@trpc/server": "next",
-    "superjson": "2.2.1",
+    "superjson": "2.2.1"
-    "ws": "^8.16.0"
   },
   "devDependencies": {
     "@homarr/eslint-config": "workspace:^0.2.0",
     "@homarr/prettier-config": "workspace:^0.1.0",
     "@homarr/tsconfig": "workspace:^0.1.0",
-    "@types/ws": "^8.5.10",
     "eslint": "^8.57.0",
     "prettier": "^3.2.5",
     "typescript": "^5.4.4"

packages/api/src/websocket.ts

@@ -0,0 +1,2 @@
+export { appRouter } from "./root";
+export { createTRPCContext } from "./trpc";

packages/auth/configuration.ts

@@ -36,6 +36,7 @@ export const createConfiguration = (isCredentialsRequest: boolean) =>
       session: sessionCallback,
       signIn: createSignInCallback(adapter, isCredentialsRequest),
     },
+    secret: "secret-is-not-defined-yet", // TODO: This should be added later
     session: {
       strategy: "database",
       maxAge: sessionMaxAgeInSeconds,

packages/db/migrate.ts

@@ -4,7 +4,7 @@ import { migrate } from "drizzle-orm/better-sqlite3/migrator";
 
 const migrationsFolder = process.argv[2] ?? "./migrations";
 
-const sqlite = new Database(process.env.DB_URL.replace("file:", ""));
+const sqlite = new Database(process.env.DB_URL?.replace("file:", ""));
 
 const db = drizzle(sqlite);
 

packages/db/package.json

@@ -13,11 +13,12 @@
   "types": "./index.ts",
   "license": "MIT",
   "scripts": {
+    "build": "esbuild migrate.ts --bundle --platform=node --outfile=migrate.cjs",
     "clean": "rm -rf .turbo node_modules",
     "lint": "eslint .",
     "format": "prettier --check . --ignore-path ../../.gitignore",
     "migration:generate": "drizzle-kit generate:sqlite",
-    "migration:run": "node ./migrate.mjs",
+    "migration:run": "tsx ./migrate.ts",
     "push": "drizzle-kit push:sqlite",
     "studio": "drizzle-kit studio",
     "typecheck": "tsc --noEmit"

packages/log/src/redis-transport.mjs

@@ -27,7 +27,7 @@ export class RedisTransport extends Transport {
 
     this.redis
       .publish(
-        "logging",
+        "pubSub:logging",
         superjson.stringify({
           message: info.message,
           timestamp: info.timestamp,

packages/redis/src/lib/channel.ts

@@ -37,7 +37,7 @@ export const createSubPubChannel = <TData>(name: string) => {
         );
       });
       subscriber.on("message", (channel, message) => {
-        if (channel !== channelName) return;
+        if (channel !== channelName) return; // TODO: check if this is necessary - it should be handled by the redis client
 
         callback(superjson.parse(message));
       });

pnpm-lock.yaml

@@ -121,10 +121,10 @@ importers:
         version: 5.29.0(@tanstack/react-query@5.29.0)(next@14.1.4)(react@18.2.0)
       '@tiptap/extension-link':
         specifier: ^2.2.5
-        version: 2.2.5(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)
+        version: 2.2.5(@tiptap/core@2.2.5)(@tiptap/pm@2.2.4)
       '@tiptap/react':
         specifier: ^2.2.5
-        version: 2.2.5(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)(react-dom@18.2.0)(react@18.2.0)
+        version: 2.2.5(@tiptap/core@2.2.5)(@tiptap/pm@2.2.4)(react-dom@18.2.0)(react@18.2.0)
       '@tiptap/starter-kit':
         specifier: ^2.2.5
         version: 2.2.5(@tiptap/pm@2.2.4)
@@ -281,6 +281,61 @@ importers:
         specifier: ^5.4.4
         version: 5.4.4
 
+  apps/websocket:
+    dependencies:
+      '@homarr/api':
+        specifier: workspace:^0.1.0
+        version: link:../../packages/api
+      '@homarr/auth':
+        specifier: workspace:^0.1.0
+        version: link:../../packages/auth
+      '@homarr/common':
+        specifier: workspace:^0.1.0
+        version: link:../../packages/common
+      '@homarr/db':
+        specifier: workspace:^0.1.0
+        version: link:../../packages/db
+      '@homarr/definitions':
+        specifier: workspace:^0.1.0
+        version: link:../../packages/definitions
+      '@homarr/log':
+        specifier: workspace:^
+        version: link:../../packages/log
+      '@homarr/redis':
+        specifier: workspace:^0.1.0
+        version: link:../../packages/redis
+      '@homarr/validation':
+        specifier: workspace:^0.1.0
+        version: link:../../packages/validation
+      dotenv:
+        specifier: ^16.4.5
+        version: 16.4.5
+      ws:
+        specifier: ^8.16.0
+        version: 8.16.0
+    devDependencies:
+      '@homarr/eslint-config':
+        specifier: workspace:^0.2.0
+        version: link:../../tooling/eslint
+      '@homarr/prettier-config':
+        specifier: workspace:^0.1.0
+        version: link:../../tooling/prettier
+      '@homarr/tsconfig':
+        specifier: workspace:^0.1.0
+        version: link:../../tooling/typescript
+      '@types/ws':
+        specifier: ^8.5.10
+        version: 8.5.10
+      eslint:
+        specifier: ^8.57.0
+        version: 8.57.0
+      prettier:
+        specifier: ^3.2.5
+        version: 3.2.5
+      typescript:
+        specifier: ^5.4.4
+        version: 5.4.4
+
   packages/api:
     dependencies:
       '@homarr/auth':
@@ -316,9 +371,6 @@ importers:
       superjson:
         specifier: 2.2.1
         version: 2.2.1
-      ws:
-        specifier: ^8.16.0
-        version: 8.16.0
     devDependencies:
       '@homarr/eslint-config':
         specifier: workspace:^0.2.0
@@ -329,9 +381,6 @@ importers:
       '@homarr/tsconfig':
         specifier: workspace:^0.1.0
         version: link:../../tooling/typescript
-      '@types/ws':
-        specifier: ^8.5.10
-        version: 8.5.10
       eslint:
         specifier: ^8.57.0
         version: 8.57.0
@@ -2155,8 +2204,8 @@ packages:
     dependencies:
       '@mantine/core': 7.7.1(@mantine/hooks@7.7.1)(@types/react@18.2.74)(react-dom@18.2.0)(react@18.2.0)
       '@mantine/hooks': 7.7.1(react@18.2.0)
-      '@tiptap/extension-link': 2.2.5(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)
+      '@tiptap/extension-link': 2.2.5(@tiptap/core@2.2.5)(@tiptap/pm@2.2.4)
-      '@tiptap/react': 2.2.5(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)(react-dom@18.2.0)(react@18.2.0)
+      '@tiptap/react': 2.2.5(@tiptap/core@2.2.5)(@tiptap/pm@2.2.4)(react-dom@18.2.0)(react@18.2.0)
       react: 18.2.0
       react-dom: 18.2.0(react@18.2.0)
     dev: false
@@ -2600,14 +2649,6 @@ packages:
       react-error-boundary: 3.1.4(react@17.0.2)
     dev: true
 
-  /@tiptap/core@2.2.4(@tiptap/pm@2.2.4):
-    resolution: {integrity: sha512-cRrI8IlLIhCE1hacBQzXIC8dsRvGq6a4lYWQK/BaHuZg21CG7szp3Vd8Ix+ra1f5v0xPOT+Hy+QFNQooRMKMCw==}
-    peerDependencies:
-      '@tiptap/pm': ^2.0.0
-    dependencies:
-      '@tiptap/pm': 2.2.4
-    dev: false
-
   /@tiptap/core@2.2.5(@tiptap/pm@2.2.4):
     resolution: {integrity: sha512-BgF6tiXhwph0Ig16ZNppMmg30t22Z/yjZ+KnS8EALdJere3ACNw8THz9q/8iQqBWjjBOHUvhrQOgER4BJxfmGA==}
     peerDependencies:
@@ -2632,13 +2673,13 @@ packages:
       '@tiptap/core': 2.2.5(@tiptap/pm@2.2.4)
     dev: false
 
-  /@tiptap/extension-bubble-menu@2.2.5(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4):
+  /@tiptap/extension-bubble-menu@2.2.5(@tiptap/core@2.2.5)(@tiptap/pm@2.2.4):
     resolution: {integrity: sha512-JFeFaUyin93SL/DZZqfoz/DWvMxtqcZLnPPd60hZBczxcLghRZCyDH5nqVHJybrkcs/xNNPennJ1Hqft+WEHbA==}
     peerDependencies:
       '@tiptap/core': ^2.0.0
       '@tiptap/pm': ^2.0.0
     dependencies:
-      '@tiptap/core': 2.2.4(@tiptap/pm@2.2.4)
+      '@tiptap/core': 2.2.5(@tiptap/pm@2.2.4)
       '@tiptap/pm': 2.2.4
       tippy.js: 6.3.7
     dev: false
@@ -2687,13 +2728,13 @@ packages:
       '@tiptap/pm': 2.2.4
     dev: false
 
-  /@tiptap/extension-floating-menu@2.2.5(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4):
+  /@tiptap/extension-floating-menu@2.2.5(@tiptap/core@2.2.5)(@tiptap/pm@2.2.4):
     resolution: {integrity: sha512-/PfS9QHEHQsoV6ZqolJ6Rq6ZC6drD7IPJRgywuJi0zTTH0YCbLmJKrc3K2Qj69xbfwcANL8gBdsexv5zOjKAXA==}
     peerDependencies:
       '@tiptap/core': ^2.0.0
       '@tiptap/pm': ^2.0.0
     dependencies:
-      '@tiptap/core': 2.2.4(@tiptap/pm@2.2.4)
+      '@tiptap/core': 2.2.5(@tiptap/pm@2.2.4)
       '@tiptap/pm': 2.2.4
       tippy.js: 6.3.7
     dev: false
@@ -2752,13 +2793,13 @@ packages:
       '@tiptap/core': 2.2.5(@tiptap/pm@2.2.4)
     dev: false
 
-  /@tiptap/extension-link@2.2.5(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4):
+  /@tiptap/extension-link@2.2.5(@tiptap/core@2.2.5)(@tiptap/pm@2.2.4):
     resolution: {integrity: sha512-3UyEdD7eOzb31Ro9zh02EIvZhgSFKV+/rSKgAf4wU17QUFid0cYRqaXcVjG0UOSznHqccxGQKD7nNT0jxTg+cw==}
     peerDependencies:
       '@tiptap/core': ^2.0.0
       '@tiptap/pm': ^2.0.0
     dependencies:
-      '@tiptap/core': 2.2.4(@tiptap/pm@2.2.4)
+      '@tiptap/core': 2.2.5(@tiptap/pm@2.2.4)
       '@tiptap/pm': 2.2.4
       linkifyjs: 4.1.3
     dev: false
@@ -2826,7 +2867,7 @@ packages:
       prosemirror-view: 1.33.1
     dev: false
 
-  /@tiptap/react@2.2.5(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)(react-dom@18.2.0)(react@18.2.0):
+  /@tiptap/react@2.2.5(@tiptap/core@2.2.5)(@tiptap/pm@2.2.4)(react-dom@18.2.0)(react@18.2.0):
     resolution: {integrity: sha512-kpEnkvTXUnr6npLt1mQCBhdgle/ZkttGgKwtwSLdM74zPzPwj8XVeCp+3kyVBK4ME7DXWIo5dPbU0nUi/FyE+g==}
     peerDependencies:
       '@tiptap/core': ^2.0.0
@@ -2834,9 +2875,9 @@ packages:
       react: ^17.0.0 || ^18.0.0
       react-dom: ^17.0.0 || ^18.0.0
     dependencies:
-      '@tiptap/core': 2.2.4(@tiptap/pm@2.2.4)
+      '@tiptap/core': 2.2.5(@tiptap/pm@2.2.4)
-      '@tiptap/extension-bubble-menu': 2.2.5(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)
+      '@tiptap/extension-bubble-menu': 2.2.5(@tiptap/core@2.2.5)(@tiptap/pm@2.2.4)
-      '@tiptap/extension-floating-menu': 2.2.5(@tiptap/core@2.2.4)(@tiptap/pm@2.2.4)
+      '@tiptap/extension-floating-menu': 2.2.5(@tiptap/core@2.2.5)(@tiptap/pm@2.2.4)
       '@tiptap/pm': 2.2.4
       react: 18.2.0
       react-dom: 18.2.0(react@18.2.0)

scripts/run.sh

@@ -1,11 +1,13 @@
 # Run migrations
-node ./db/migrate.mjs ./db/migrations
+node ./db/migrate.cjs ./db/migrations
 
 # Start Redis
 redis-server &
 
-# Run the nestjs backend
+# Run the tasks backend
-node apps/nestjs/dist/main.mjs &
+node apps/tasks/tasks.cjs &
+
+node apps/websocket/wssServer.cjs &
 
 # Run the nextjs server
 node apps/nextjs/server.js