diff --git a/.github/workflows/renovate.yml b/.github/workflows/renovate.yml
index 77d2c04..641324e 100644
--- a/.github/workflows/renovate.yml
+++ b/.github/workflows/renovate.yml
@@ -22,11 +22,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Install Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
       - name: Install bun
         uses: oven-sh/setup-bun@v2
 
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 0000000..535cc75
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,23 @@
+name: Test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Run tests
+        run: bun test
diff --git a/__tests__/apps.test.ts b/__tests__/apps.test.ts
new file mode 100644
index 0000000..25a335f
--- /dev/null
+++ b/__tests__/apps.test.ts
@@ -0,0 +1,71 @@
+import { expect, test, describe } from "bun:test";
+import { appInfoSchema, dynamicComposeSchema } from '@runtipi/common/schemas'
+import { fromError } from 'zod-validation-error';
+import fs from 'node:fs'
+import path from 'node:path'
+
+const getApps = async () => {
+  const appsDir = await fs.promises.readdir(path.join(process.cwd(), 'apps'))
+  return appsDir
+};
+
+const getFile = async (app: string, file: string) => {
+  const filePath = path.join(process.cwd(), 'apps', app, file)
+  try {
+    const file = await fs.promises.readFile(filePath, 'utf-8')
+    return file
+  } catch (err) {
+    return null
+  }
+}
+
+describe("each app should have the required files", async () => {
+  const apps = await getApps()
+
+  for (const app of apps) {
+    const files = ['config.json', 'docker-compose.json', 'metadata/logo.jpg', 'metadata/description.md']
+
+    for (const file of files) {
+      test(`app ${app} should have ${file}`, async () => {
+        const fileContent = await getFile(app, file)
+        expect(fileContent).not.toBeNull()
+      })
+    }
+  }
+})
+
+describe("each app should have a valid config.json", async () => {
+  const apps = await getApps()
+
+  for (const app of apps) {
+    test(`app ${app} should have a valid config.json`, async () => {
+      const fileContent = await getFile(app, 'config.json')
+      const parsed = appInfoSchema.omit({ urn: true }).safeParse(JSON.parse(fileContent || '{}'))
+
+      if (!parsed.success) {
+        const validationError = fromError(parsed.error);
+        console.error(`Error parsing config.json for app ${app}:`, validationError.toString());
+      }
+
+      expect(parsed.success).toBe(true)
+    })
+  }
+})
+
+describe("each app should have a valid docker-compose.json", async () => {
+  const apps = await getApps()
+
+  for (const app of apps) {
+    test(`app ${app} should have a valid docker-compose.json`, async () => {
+      const fileContent = await getFile(app, 'docker-compose.json')
+      const parsed = dynamicComposeSchema.safeParse(JSON.parse(fileContent || '{}'))
+
+      if (!parsed.success) {
+        const validationError = fromError(parsed.error);
+        console.error(`Error parsing docker-compose.json for app ${app}:`, validationError.toString());
+      }
+
+      expect(parsed.success).toBe(true)
+    })
+  }
+});
diff --git a/bun.lockb b/bun.lockb
index f7bcbff..2c1bfe8 100755
Binary files a/bun.lockb and b/bun.lockb differ
diff --git a/package.json b/package.json
index 6223dc6..b0e7907 100644
--- a/package.json
+++ b/package.json
@@ -5,13 +5,18 @@
   "main": "index.js",
   "type": "module",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "bun test"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "devDependencies": {
-    "@runtipi/common": "^0.8.0",
+    "@types/bun": "^1.2.10",
     "@types/node": "^22.14.1"
+  },
+  "dependencies": {
+    "@runtipi/common": "^0.8.0",
+    "bun": "^1.2.10",
+    "zod-validation-error": "^3.4.0"
   }
 }
diff --git a/tsconfig.json b/tsconfig.json
index 216b3a6..01ef57a 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -17,5 +17,8 @@
     "lib": [
       "es2022"
     ]
-  }
+  },
+  "include": [
+    "**/*.ts",
+  ],
 }