diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml deleted file mode 100644 index 6cf73f2..0000000 --- a/.gitea/workflows/ci.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Tests every pull request and every push to main. When the tests pass on main, builds the -# Docker image and pushes it to this Gitea server's container registry, then (optionally) -# tells Portainer to redeploy. Secrets and variables it uses are listed in docs/runbook.md. -name: CI - -on: - push: - branches: [main] - pull_request: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: npm - - run: npm ci - - run: npm run lint - - run: npm run check - - run: npm test - - run: npm run build - - image: - needs: test - if: github.event_name == 'push' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Work out the image name - id: image - run: | - registry="${GITHUB_SERVER_URL#*://}" - registry="${registry%/}" - repository="$(echo "$GITHUB_REPOSITORY" | tr '[:upper:]' '[:lower:]')" - echo "registry=$registry" >> "$GITHUB_OUTPUT" - echo "name=$registry/$repository" >> "$GITHUB_OUTPUT" - echo "short_sha=$(echo "$GITHUB_SHA" | cut -c1-7)" >> "$GITHUB_OUTPUT" - - - name: Log in to the registry - env: - REGISTRY: ${{ steps.image.outputs.registry }} - REGISTRY_USERNAME: ${{ vars.REGISTRY_USERNAME || github.repository_owner }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - run: echo "$REGISTRY_TOKEN" | docker login "$REGISTRY" -u "$REGISTRY_USERNAME" --password-stdin - - # ORIGIN is fixed into the image at build time; leave the variable unset behind an HTTPS proxy - - name: Build - env: - IMAGE: ${{ steps.image.outputs.name }} - SHORT_SHA: ${{ steps.image.outputs.short_sha }} - ORIGIN: ${{ vars.ORIGIN }} - run: > - docker build - --build-arg ORIGIN="$ORIGIN" - --label org.opencontainers.image.revision="$GITHUB_SHA" - --tag "$IMAGE:latest" - --tag "$IMAGE:$SHORT_SHA" - . - - - name: Push - env: - IMAGE: ${{ steps.image.outputs.name }} - run: docker push --all-tags "$IMAGE" - - - name: Redeploy in Portainer - env: - PORTAINER_WEBHOOK_URL: ${{ secrets.PORTAINER_WEBHOOK_URL }} - run: | - if [ -z "$PORTAINER_WEBHOOK_URL" ]; then - echo "PORTAINER_WEBHOOK_URL is not set; redeploy the stack in Portainer by hand." - exit 0 - fi - curl --fail --silent --show-error --request POST "$PORTAINER_WEBHOOK_URL" diff --git a/README.md b/README.md index 030a523..87aad49 100644 --- a/README.md +++ b/README.md @@ -30,12 +30,13 @@ npm run dev In simulated mode, SharePoint and ACC are in-memory stand-ins. A demo project is created at startup, filled with sample files, and edited at random every 45 seconds so there's something to watch. One file (`Corrupt model.rvt`) is always rejected, so the issues list has something in it. Nothing persists: a restart starts over. -| Command | What it does | -| --------------- | ------------------------------------------------------------------------------------------------------------------- | -| `npm test` | Unit and integration tests (Vitest) | -| `npm run check` | Type check | -| `npm run lint` | Prettier + ESLint | -| `npm run build` | Production build into `build/` (`node build` runs it; run it from the repo root so `drizzle/` migrations are found) | +| Command | What it does | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `npm test` | Unit and integration tests (Vitest) | +| `npm run check` | Type check | +| `npm run lint` | Prettier + ESLint | +| `npm run build` | Production build into `build/` (`node build` runs it; run it from the repo root so `drizzle/` migrations are found) | +| `npm run image:publish` | Checks, builds and pushes the Docker image to the Gitea registry (see the runbook) | To try the Docker image locally: put `ADMIN_USERNAME`, `ADMIN_PASSWORD`, `SESSION_SECRET`, `SYNC_BACKEND=simulated` and `ORIGIN=http://localhost:3000` in `.env`, then run `docker compose up -d --build`. diff --git a/docker-compose.yml b/docker-compose.yml index 13090ad..d595837 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,13 +1,13 @@ -# Portainer stack. Portainer pulls the image that CI (.gitea/workflows/ci.yml) pushes after the -# tests pass on main; `docker compose up -d --build` builds it locally instead. Set the variables -# below in Portainer's stack environment, or in a .env file next to this one. See docs/runbook.md. +# Portainer stack. Portainer pulls the image published with `npm run image:publish`; +# `docker compose up -d --build` builds it locally instead. Set the variables below in +# Portainer's stack environment, or in a .env file next to this one. See docs/runbook.md. services: sync: build: context: . args: - # Local builds only (CI uses its ORIGIN variable): set when serving plain HTTP, to the - # exact address users open, e.g. http://sync-server:3000 + # Local builds only: set when serving plain HTTP, to the exact address users open, + # e.g. http://sync-server:3000 ORIGIN: ${ORIGIN:-} # Set IMAGE to a commit tag (e.g. ...:1a2b3c4) to roll back image: ${IMAGE:-primegit.primecontrols-dev.com/c.west/sharepoint-acc-sync:latest} diff --git a/docs/design.md b/docs/design.md index 93d2d92..6ab9107 100644 --- a/docs/design.md +++ b/docs/design.md @@ -181,8 +181,8 @@ Two settings live outside `src/env.ts`: ### Deployment - `Dockerfile`: a multi-stage build on `node:24-bookworm-slim`. The runtime stage holds `build/`, production `node_modules` (`better-sqlite3`, `zod`; everything else is bundled) and `drizzle/`, runs as `node`, and keeps the database at `/data/sync.db`. -- `.gitea/workflows/ci.yml`: every pull request and push to main runs lint, type check, tests and build. On main, a second job that needs the first builds the image and pushes it to the Gitea registry, tagged `latest` and with the short commit hash, then calls the Portainer webhook if one is set. Since CI builds the image, `ORIGIN` is a Gitea Actions variable. -- `docker-compose.yml` is the Portainer stack: one service using the CI image with `pull_policy: always` (a redeploy never reuses a stale image, and `IMAGE` pins a commit tag for rollback), the `sync-data` volume, required settings enforced with `${VAR:?}`, and `stop_grace_period: 2m` so a run in progress can finish. Its `build:` section is for local `docker compose up --build`. +- No CI: images are built on a developer's machine with `npm run image:publish` (`scripts/publish-image.ts`). It refuses uncommitted changes, runs lint, type check and tests, builds, and pushes to the Gitea registry tagged with the short commit hash and then `latest`. It reads `ORIGIN` from the shell, never from `.env`. Portainer only pulls and runs the image. +- `docker-compose.yml` is the Portainer stack: one service using the published image with `pull_policy: always` (a redeploy never reuses a stale image, and `IMAGE` pins a commit tag for rollback), the `sync-data` volume, required settings enforced with `${VAR:?}`, and `stop_grace_period: 2m` so a run in progress can finish. Its `build:` section is for local `docker compose up --build`. - `GET /healthz` backs the image's `HEALTHCHECK`. - On startup, runs left without an outcome (the process was killed mid-run) are closed as failed. The next run carries on from the saved item state. - Operations are covered in [runbook.md](runbook.md). @@ -227,7 +227,7 @@ src/lib/styles/app.css # design tokens + shared styles src/routes/ # projects, project status + history, admin, healthz drizzle/ # generated SQL migrations (npm run db:generate) Dockerfile docker-compose.yml -.gitea/workflows/ci.yml # tests on PRs and main; image build + push on main +scripts/publish-image.ts # checks, builds and pushes the image (npm run image:publish) ``` ## Open questions (resolved by spikes) @@ -248,4 +248,4 @@ Dockerfile docker-compose.yml 6. **ApsDocumentTarget** + ACC folder URL resolver, including the fallbacks chosen in step 1. 7. **Wiring (done):** coordinator, scheduler, composition root, `init` hook, history pruning, simulated mode. 8. **UI (done):** projects list, project status (title block, issues with retry, file tree), 7-day history with per-run detail, Run now with cooldown, admin login, add/edit/resume/reset/remove. -9. **Ship (done):** Docker image, Portainer stack file, Gitea Actions CI, runbook. The image was tested locally. The first CI run and Portainer deploy are still to do. +9. **Ship (done):** Docker image, Portainer stack file, publish script, runbook. The image was tested locally. The first publish and Portainer deploy are still to do. diff --git a/docs/runbook.md b/docs/runbook.md index 71b0da7..50ef9f5 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -4,7 +4,7 @@ How to deploy, update, back up and troubleshoot the sync server. ## What runs -- **One container**, from the image CI builds (see [Continuous integration](#continuous-integration)). Never run more than one copy: the locks that stop two syncs of the same project overlapping live inside the process. +- **One container**, from the image published to the Gitea registry (see [Publishing an image](#publishing-an-image)). Never run more than one copy: the locks that stop two syncs of the same project overlapping live inside the process. - **One volume**, `sync-data`, mounted at `/data`. It holds `sync.db`, the SQLite database with the projects, the record of which ACC item belongs to which SharePoint item, and the last 7 days of history. - **Port 3000** inside the container. `HOST_PORT` sets the port on the server (default 3000). - **Health check:** `GET /healthz` answers `{"status":"ok"}` once the server has started. Docker checks it every 30 seconds, and Portainer shows the result as healthy or unhealthy. @@ -15,7 +15,7 @@ How to deploy, update, back up and troubleshoot the sync server. Decide this before the first deploy, because it changes how the image is built. - **Behind an HTTPS reverse proxy** (recommended). The proxy must pass the original `Host` header on, which most do by default. Leave `ORIGIN` empty. -- **Plain HTTP** on the VPN, for example `http://sync-server:3000`. Set `ORIGIN` to exactly the address people will type, with no trailing slash. `ORIGIN` is fixed when the image is built, so the app only works at that one address: not by IP if `ORIGIN` uses a name, and not on a different port. Because CI builds the image, `ORIGIN` is a variable in Gitea, not a Portainer setting (see below). +- **Plain HTTP** on the VPN, for example `http://sync-server:3000`. Set `ORIGIN` to exactly the address people will type, with no trailing slash. `ORIGIN` is fixed when the image is built, so the app only works at that one address: not by IP if `ORIGIN` uses a name, and not on a different port. So `ORIGIN` is set when [publishing an image](#publishing-an-image), not in Portainer. If `ORIGIN` is wrong or missing over plain HTTP, pages still load, but every button that submits something fails with "Cross-site POST form submissions are forbidden". @@ -29,7 +29,7 @@ Set these in the Portainer stack's **Environment variables**, or in a `.env` fil | `ADMIN_PASSWORD` | yes | | Password for the admin pages | | `SESSION_SECRET` | yes | | Signs admin logins. Any long random string, e.g. the output of `node -e "console.log(crypto.randomBytes(32).toString('hex'))"`. Changing it logs admins out | | `HOST_PORT` | no | `3000` | Port on the server | -| `IMAGE` | no | latest from CI | Image to run. Set it to a commit tag to roll back (see [Rolling back](#rolling-back)) | +| `IMAGE` | no | latest published | Image to run. Set it to a commit tag to roll back (see [Rolling back](#rolling-back)) | | `SYNC_BACKEND` | no | `live` | `simulated` runs the in-memory demo instead (nothing is saved) | | `TZ` | no | `America/Chicago` | Time zone the status pages show times in | | `POLL_INTERVAL_SECONDS` | no | `180` | How often every project is checked for changes | @@ -39,34 +39,40 @@ Set these in the Portainer stack's **Environment variables**, or in a `.env` fil The Microsoft Graph and APS credentials will be added to this table when the live connections are built. -## Continuous integration +## Publishing an image -`.gitea/workflows/ci.yml` runs on Gitea Actions: +Images are built on a developer's machine and pushed to the Gitea server's container registry as `primegit.primecontrols-dev.com/c.west/sharepoint-acc-sync`. Portainer only pulls and runs them. -- **Every pull request and every push to main:** lint, type check, tests and a production build. A pull request whose checks fail shouldn't be merged. -- **Pushes to main, once those pass:** builds the Docker image and pushes it to this Gitea server's container registry as `primegit.primecontrols-dev.com/c.west/sharepoint-acc-sync`, tagged `latest` and with the commit's short hash (for example `1a2b3c4`). It then asks Portainer to redeploy, if a webhook is set. +Once per machine, log in to the registry with a Gitea access token that has the `write:package` scope (**User settings → Applications**). Docker asks for the username and the token: -One-time setup, in the repository's **Settings → Actions**: +```sh +docker login primegit.primecontrols-dev.com +``` -| Name | Kind | Required | What it is | -| ----------------------- | -------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | -| `REGISTRY_TOKEN` | secret | yes | Gitea access token with the `write:package` scope, from **User settings → Applications**. Used to push the image | -| `REGISTRY_USERNAME` | variable | no | The account that owns `REGISTRY_TOKEN`. Defaults to the repository owner | -| `ORIGIN` | variable | no | Only for plain HTTP (see [above](#choose-how-people-will-reach-it)). Changing it takes effect from the next image built | -| `PORTAINER_WEBHOOK_URL` | secret | no | The stack's webhook URL (see [First deploy](#first-deploy-portainer)). Without it, the image is pushed but the stack has to be redeployed by hand | +Then, from a clone with everything committed: -The repository also needs Actions turned on (**Settings → Repository → Actions**) and a runner that can run Docker commands. Gitea's own runner does by default. If the image job fails at `docker login` with "command not found" or "Cannot connect to the Docker daemon", ask whoever looks after the runner. +```sh +npm run image:publish +``` + +It stops at the first failure. In order, it: + +1. refuses to go on if there are uncommitted changes, so the image always matches a commit; +2. runs lint, type check and tests; +3. builds the image from the `Dockerfile`, pulling the newest Node base image; +4. pushes it tagged with the commit's short hash (for example `1a2b3c4`), then as `latest`. + +Over plain HTTP, set `ORIGIN` in the same shell first, for example in PowerShell `$env:ORIGIN = 'http://sync-server:3000'`, or in bash `ORIGIN=http://sync-server:3000 npm run image:publish`. The script prints the `ORIGIN` it's using before it starts. It ignores `.env`, so a local `ORIGIN=http://localhost:3000` never ends up in a published image. ## First deploy (Portainer) -1. Push to main and wait for the CI run to finish, so there's an image to pull. +1. [Publish an image](#publishing-an-image), so there's one to pull. 2. If the image isn't public, add the registry under **Registries → Add registry → Custom registry**: URL `primegit.primecontrols-dev.com`, with a Gitea access token that has the `read:package` scope. 3. **Stacks → Add stack**, give it a name such as `sharepoint-acc-sync`, and choose **Repository**. 4. Enter the repository URL and the branch (`refs/heads/main`). The compose path is `docker-compose.yml`. If the repository is private, turn on authentication and use a read-only access token. 5. Under **Environment variables**, add the settings above. -6. To redeploy automatically after each CI run, turn on **GitOps updates**, choose **Webhook** and turn on **Re-pull image**. Copy the webhook URL into the `PORTAINER_WEBHOOK_URL` secret in Gitea. -7. **Deploy the stack.** -8. Check it's working: +6. **Deploy the stack.** +7. Check it's working: - the container shows as **healthy** in Portainer; - the logs show `Sync server started` with the expected `backend`; - the projects page opens at the chosen address, and the admin login works. @@ -75,7 +81,7 @@ From the command line on the Docker host, the equivalent is `docker compose up - ## Updating -Merge to main. Once CI passes, the new image is pushed and, with the webhook set, Portainer redeploys it. Without the webhook, open the stack in Portainer and use **Pull and redeploy**. Every deploy pulls the newest image, so a redeploy never runs an old one. +[Publish an image](#publishing-an-image), then open the stack in Portainer and use **Pull and redeploy**. Every deploy pulls the newest image, so a redeploy never runs an old one. On the command line: `git pull`, then `docker compose up -d`. @@ -83,7 +89,7 @@ When the container stops, a sync in progress gets up to 2 minutes to finish. If ## Rolling back -Every image CI pushes is kept under its commit's short hash. Find the hash of the last good commit in Gitea (or under **Packages** on the repository owner's page), set the stack's `IMAGE` variable to `primegit.primecontrols-dev.com/c.west/sharepoint-acc-sync:`, and redeploy. Remove `IMAGE` again to go back to following `latest`. +Every published image is kept under its commit's short hash. Find the hash of the last good one under **Packages** on the repository owner's Gitea page, set the stack's `IMAGE` variable to `primegit.primecontrols-dev.com/c.west/sharepoint-acc-sync:`, and redeploy. Remove `IMAGE` again to go back to following `latest`. A rollback doesn't undo database changes, and an older image may not work with a database a newer one has changed. Before deploying a release that adds a migration (a new file in `drizzle/`), take a [backup](#backups), and restore it if you roll back past that release. @@ -102,14 +108,14 @@ To restore one: stop the stack, put the backup in the volume as `sync.db`, delet ## Troubleshooting -| Symptom | Likely cause and fix | -| ------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------- | -| Deploy fails with "required variable … is missing a value" | A required setting isn't set. Add it to the stack's environment variables | -| Buttons fail with "Cross-site POST form submissions are forbidden" | Plain HTTP without the right `ORIGIN`, or the app was opened at a different address. Set the `ORIGIN` variable in Gitea and rerun CI | -| Admin login appears to work but doesn't stick | Plain HTTP without `ORIGIN`: the login cookie is marked HTTPS-only, so the browser drops it. Set the `ORIGIN` variable in Gitea and rerun CI | -| Container is unhealthy or keeps restarting | Check the logs. Bad settings are reported by name at startup | -| Deploy fails pulling the image ("unauthorized" or "not found") | The registry isn't added in Portainer, its token lacks `read:package`, or CI hasn't pushed an image yet. See First deploy | -| CI's image job fails at "Log in to the registry" | `REGISTRY_TOKEN` is missing, expired or lacks `write:package`, or `REGISTRY_USERNAME` doesn't match the token's owner | -| A merge passed CI but the old version is still running | No webhook is set, or it failed (see the last step of the CI run). Use **Pull and redeploy** in Portainer | -| Times on the pages are off by some hours | `TZ` doesn't match the office's time zone | -| A project shows **Paused** | Its SharePoint folder was deleted or couldn't be found. Nothing was deleted in ACC. Fix the folder, then **Resume** it on the admin page | +| Symptom | Likely cause and fix | +| ------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| Deploy fails with "required variable … is missing a value" | A required setting isn't set. Add it to the stack's environment variables | +| Buttons fail with "Cross-site POST form submissions are forbidden" | Plain HTTP without the right `ORIGIN`, or the app was opened at a different address. Publish again with the right `ORIGIN` | +| Admin login appears to work but doesn't stick | Plain HTTP without `ORIGIN`: the login cookie is marked HTTPS-only, so the browser drops it. Publish again with `ORIGIN` set | +| Container is unhealthy or keeps restarting | Check the logs. Bad settings are reported by name at startup | +| Deploy fails pulling the image ("unauthorized" or "not found") | The registry isn't added in Portainer, its token lacks `read:package`, or no image has been published yet. See First deploy | +| `npm run image:publish` fails pushing ("unauthorized" or "denied") | Not logged in to the registry on this machine, or the token lacks `write:package` or has expired. Run `docker login` again | +| A new version was published but the old one is still running | The stack wasn't redeployed. Use **Pull and redeploy** in Portainer | +| Times on the pages are off by some hours | `TZ` doesn't match the office's time zone | +| A project shows **Paused** | Its SharePoint folder was deleted or couldn't be found. Nothing was deleted in ACC. Fix the folder, then **Resume** it on the admin page | diff --git a/package.json b/package.json index 2d61f56..7646b87 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "db:push": "drizzle-kit push", "db:generate": "drizzle-kit generate", "db:migrate": "drizzle-kit migrate", - "db:studio": "drizzle-kit studio" + "db:studio": "drizzle-kit studio", + "image:publish": "node scripts/publish-image.ts" }, "devDependencies": { "@eslint/js": "^10.0.1", diff --git a/scripts/publish-image.ts b/scripts/publish-image.ts new file mode 100644 index 0000000..3842ce1 --- /dev/null +++ b/scripts/publish-image.ts @@ -0,0 +1,63 @@ +// Checks, builds and pushes the Docker image to the Gitea registry, tagged `latest` and with the +// commit's short hash. Run with `npm run image:publish`; see docs/runbook.md, "Publishing an image". +import { execFileSync } from 'node:child_process'; + +const IMAGE = 'primegit.primecontrols-dev.com/c.west/sharepoint-acc-sync'; +const CHECKS = ['lint', 'check', 'test']; + +function run(command: string, args: string[]): void { + console.log(`\n> ${command} ${args.join(' ')}`); + execFileSync(command, args, { stdio: 'inherit' }); +} + +function read(command: string, args: string[]): string { + return execFileSync(command, args, { encoding: 'utf8' }).trim(); +} + +function npmRun(script: string): void { + // npm_execpath is set when this runs through `npm run`; going through node avoids npm.cmd on Windows + const npm = process.env.npm_execpath; + if (!npm) throw new Error('Run this with `npm run image:publish`.'); + run(process.execPath, [npm, 'run', script]); +} + +function publish(): void { + // The hash tag has to describe exactly what was built, so uncommitted changes aren't allowed + if (read('git', ['status', '--porcelain'])) { + throw new Error('There are uncommitted changes. Commit or stash them first.'); + } + const revision = read('git', ['rev-parse', 'HEAD']); + const shortRevision = read('git', ['rev-parse', '--short', 'HEAD']); + + // Fixed into the image at build time: set it only when the app is served over plain HTTP + const origin = process.env.ORIGIN ?? ''; + console.log(`Publishing ${IMAGE}:${shortRevision} with ORIGIN ${origin || '(unset, HTTPS)'}`); + + for (const check of CHECKS) npmRun(check); + + run('docker', [ + 'build', + '--pull', + '--build-arg', + `ORIGIN=${origin}`, + '--label', + `org.opencontainers.image.revision=${revision}`, + '--tag', + `${IMAGE}:${shortRevision}`, + '--tag', + `${IMAGE}:latest`, + '.' + ]); + // The hash tag goes first, so `latest` only moves once the exact build is safely stored + run('docker', ['push', `${IMAGE}:${shortRevision}`]); + run('docker', ['push', `${IMAGE}:latest`]); + + console.log(`\nPushed ${IMAGE}:${shortRevision} as latest. Redeploy the stack in Portainer.`); +} + +try { + publish(); +} catch (error) { + console.error(`\nNot published: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); +} diff --git a/tsconfig.json b/tsconfig.json index 7619945..6662e43 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,5 +4,5 @@ "strict": true, "types": ["$app/types", "node"] }, - "include": ["src", "vite.config.ts", "drizzle.config.ts"] + "include": ["src", "scripts", "vite.config.ts", "drizzle.config.ts"] }