first draft

This commit is contained in:
C-West8
2026-10-22 22:32:35 -05:00
parent 09a5f8fe0d
commit db4abbeefb
52 changed files with 8361 additions and 101 deletions

5
.env.example Normal file
View File

@@ -0,0 +1,5 @@
DATABASE_PATH=local.db
POLL_INTERVAL_SECONDS=180
MANUAL_RUN_COOLDOWN_SECONDS=30
MAX_ATTEMPTS=3
HISTORY_RETENTION_DAYS=7

25
.gitignore vendored Normal file
View File

@@ -0,0 +1,25 @@
node_modules
# Output
.output
.vercel
.netlify
.wrangler
/.svelte-kit
/build
# OS
.DS_Store
Thumbs.db
# Env
.env
.env.*
!.env.example
!.env.test
# Vite
vite.config.js.timestamp-*
vite.config.ts.timestamp-*
# SQLite
*.db

1
.npmrc Normal file
View File

@@ -0,0 +1 @@
engine-strict=true

10
.prettierignore Normal file
View File

@@ -0,0 +1,10 @@
# Package Managers
package-lock.json
pnpm-lock.yaml
yarn.lock
bun.lock
bun.lockb
# Miscellaneous
/static/
/drizzle/

3
.vscode/extensions.json vendored Normal file
View File

@@ -0,0 +1,3 @@
{
"recommendations": ["svelte.svelte-vscode", "esbenp.prettier-vscode", "dbaeumer.vscode-eslint"]
}

View File

@@ -14,7 +14,7 @@ There is also a small web page that shows each project's file tree, sync status,
## Status ## Status
Planning. See: In development. The sync core and database layer are built and tested against simulated SharePoint and ACC. The real SharePoint/ACC connections and the web UI are next. See:
- [docs/design.md](docs/design.md): architecture, interfaces, data model and build order - [docs/design.md](docs/design.md): architecture, interfaces, data model and build order
- [docs/access-setup.md](docs/access-setup.md): what IT and the ACC admin need to set up - [docs/access-setup.md](docs/access-setup.md): what IT and the ACC admin need to set up

View File

@@ -19,13 +19,13 @@ The tool only ever reads from SharePoint, so no write permission is needed.
## Autodesk / ACC (ask the ACC account admin) ## Autodesk / ACC (ask the ACC account admin)
1. **Create an APS app** at <https://aps.autodesk.com> → *My Apps*. 1. **Create an APS app** at <https://aps.autodesk.com> → _My Apps_.
- App type: **Server-to-Server** (app-only). - App type: **Server-to-Server** (app-only).
- API access: **Data Management API**. - API access: **Data Management API**.
- Send us the **Client ID** and **Client Secret**. - Send us the **Client ID** and **Client Secret**.
2. **In ACC Account Admin → Custom Integrations, add the APS app's Client ID** so it can access the account's projects. 2. **In ACC Account Admin → Custom Integrations, add the APS app's Client ID** so it can access the account's projects.
3. **Possibly: a service user.** Some ACC Docs operations may need to run "as" a user, i.e. an account that is a project member with edit/upload permission on the target folder. We'll confirm this during the build spikes before asking for it. 3. **Possibly: a service user.** Some ACC Docs operations may need to run "as" a user, i.e. an account that is a project member with edit/upload permission on the target folder. We'll confirm this during the build spikes before asking for it.
4. **For each project:** tell us which ACC project, and which folder under *Project Files*, is the sync root. 4. **For each project:** tell us which ACC project, and which folder under _Project Files_, is the sync root.
## For the build spikes ## For the build spikes

View File

@@ -3,7 +3,7 @@
## Goals and constraints ## Goals and constraints
| Decision | Choice | | Decision | Choice |
| --- | --- | | ------------- | --------------------------------------------------------------------------------------------------------------------------- |
| Direction | One-way, SharePoint → ACC | | Direction | One-way, SharePoint → ACC |
| Latency | Poll every ~3 min (configurable); "Run now" for immediate sync | | Latency | Poll every ~3 min (configurable); "Run now" for immediate sync |
| Operations | Create, update (new ACC version), rename, move, delete | | Operations | Create, update (new ACC version), rename, move, delete |
@@ -39,8 +39,8 @@ The sync logic is plain TypeScript with no SvelteKit imports. SvelteKit hosts it
### Each run has two phases ### Each run has two phases
1. **Ingest.** Read changes from SharePoint since the project's saved cursor and write them to `items` as the *desired* state (name, parent, content tag, deleted flag). Save the new cursor in the same transaction. Nothing touches ACC in this phase. 1. **Ingest.** Read changes from SharePoint since the project's saved cursor and write them to `items` as the _desired_ state (name, parent, content tag, deleted flag). Save the new cursor in the same transaction. Nothing touches ACC in this phase.
2. **Reconcile.** For every item whose desired state differs from its *synced* state, work out the operation and apply it to ACC. On success, copy desired → synced. On failure, increment `attempt_count` and record the error. 2. **Reconcile.** For every item whose desired state differs from its _synced_ state, work out the operation and apply it to ACC. On success, copy desired → synced. On failure, increment `attempt_count` and record the error.
This split gives us several things without extra code: This split gives us several things without extra code:
@@ -49,14 +49,14 @@ This split gives us several things without extra code:
- **Inferred renames and moves.** Graph delta only reports "this item now looks like X". The operation comes from comparing that to what we last synced: - **Inferred renames and moves.** Graph delta only reports "this item now looks like X". The operation comes from comparing that to what we last synced:
| Desired vs synced | Operation | | Desired vs synced | Operation |
| --- | --- | | ------------------- | ------------------------------- |
| no target id yet | create (folder) / upload (file) | | no target id yet | create (folder) / upload (file) |
| `deleted` | delete | | `deleted` | delete |
| name differs | rename | | name differs | rename |
| parent differs | move | | parent differs | move |
| content tag differs | upload new version | | content tag differs | upload new version |
A single change can need more than one of these, for example a move plus a rename. A single change can need more than one of these, for example a move plus a rename.
**Order within reconcile:** folder creates and moves from shallowest to deepest, then files, then deletes. That way a destination folder always exists before anything moves into it. **Order within reconcile:** folder creates and moves from shallowest to deepest, then files, then deletes. That way a destination folder always exists before anything moves into it.
@@ -66,69 +66,38 @@ Graph delta for SharePoint only works at the **drive root**, not on subfolders.
An item whose new parent is outside the root counts as deleted. That covers "removed from the root = deleted in ACC". An item whose new parent is outside the root counts as deleted. That covers "removed from the root = deleted in ACC".
**Safety rule:** if the project's root folder itself disappears or moves, the project is paused with a project-level issue and **nothing is deleted**. That is almost always a mistake, not an intent to wipe ACC. **Safety rule:** if the project's root folder is deleted, or is missing from a full listing, the project is paused with a project-level issue and **nothing is deleted**. That is almost always a mistake, not an intent to wipe ACC. Moving or renaming the root within the same library is harmless, because projects track it by ID.
**Restored folders:** a folder delete cascades "deleted" to everything stored under it. If a deleted folder later reappears, its contents may not be reported one by one, so ingest asks for one follow-up **full listing** in the same run, which brings them back.
## Interfaces (ports) ## Interfaces (ports)
These are sketches; final signatures get settled during the build. The source of truth is [`src/lib/server/sync/core/ports.ts`](../src/lib/server/sync/core/ports.ts):
```ts - `ChangeSource` (SharePoint): `getChanges(root, cursor)` returns changes, the next cursor and whether it was a full listing. `download(root, id)` returns the file content.
// sync/core/ports.ts - `DocumentTarget` (ACC): `createFolder`, `uploadFile`, `uploadVersion`, `rename`, `move`, `delete`, `restore`.
- `IgnorePolicy`: decides which names are skipped.
export interface SourceItem {
sourceId: string;
parentSourceId: string | null;
name: string;
isFolder: boolean;
contentTag: string | null; // Graph cTag; changes only when content changes
modifiedAt: Date;
size: number | null;
}
export type SourceChange =
| { kind: 'upsert'; item: SourceItem }
| { kind: 'delete'; sourceId: string };
export interface ChangeSource {
/** Changes since `cursor` (null = full enumeration), with the cursor to save. */
getChanges(root: SourceRoot, cursor: string | null): Promise<{ changes: SourceChange[]; cursor: string }>;
download(root: SourceRoot, sourceId: string): Promise<{ stream: ReadableStream<Uint8Array>; size: number }>;
}
export interface TargetRef { id: string; isFolder: boolean }
export interface DocumentTarget {
createFolder(parentId: string, name: string): Promise<TargetRef>;
uploadFile(parentId: string, name: string, content: FileContent): Promise<TargetRef>;
uploadVersion(ref: TargetRef, name: string, content: FileContent): Promise<void>;
rename(ref: TargetRef, name: string): Promise<void>;
/** May return a new ref if the target can only move by copy + delete. */
move(ref: TargetRef, newParentId: string): Promise<TargetRef>;
delete(ref: TargetRef): Promise<void>;
/** Bring back a previously deleted item (keeps history where possible). */
restore(ref: TargetRef, parentId: string, name: string, content?: FileContent): Promise<TargetRef>;
}
export interface IgnorePolicy {
isIgnored(name: string, isFolder: boolean): boolean;
}
```
Repositories are split by consumer (interface segregation): Repositories are split by consumer (interface segregation):
- `ProjectRepository`: CRUD, cursor, pause and reset - `ProjectRepository`: create, pause/resume, first-synced and last-run stamps, reset
- `ItemRepository`: desired/synced state, pending diff query, tree query for the UI - `ItemRepository`: load a project's items, commit an ingest with its cursor (atomically), save reconcile results
- `RunRepository`: start/finish runs, record events, prune old history - `RunRepository`: start/finish runs, record events, list history, prune old history
Adapters throw `ServiceUnavailableError` for outages (network, auth, throttling that outlasted retries). The run stops early and no item is charged an attempt; any other error counts against the item that hit it.
Both fallback strategies, move-by-copy and restore-by-reupload, live **inside the ACC adapter**. They are ACC-specific details, so the engine never branches on them. Both fallback strategies, move-by-copy and restore-by-reupload, live **inside the ACC adapter**. They are ACC-specific details, so the engine never branches on them.
### Coordinator ### Coordinator
`SyncCoordinator.requestRun(projectId, trigger)` is the only way a run starts, whether from the scheduler or from "Run now". It holds one in-process lock per project and returns one of: `SyncCoordinator` is the only way a run starts. It holds one in-process lock per project. `requestManualRun(projectId)` ("Run now") returns one of:
- `started` - `started`
- `already-running` (the caller attaches to the current run) - `already-running` (the page just shows the run in progress)
- `{ cooldown: secondsLeft }` (the last manual run was less than 30 s ago) - `cooldown` with `retryAfterSeconds` (the last manual request for this project was under 30 s ago)
- `unavailable` with a reason (unknown, disabled or paused project)
`runScheduledPass()` is what the timer calls. An `IntervalScheduler` drives it, and the history-pruning job, without ever overlapping a task with itself.
The scheduler goes through enabled projects one at a time, and one failing project never blocks the others. Graph and APS clients share one HTTP layer that handles `429`/`Retry-After` and backoff, because rate limits apply per app, not per project. The scheduler goes through enabled projects one at a time, and one failing project never blocks the others. Graph and APS clients share one HTTP layer that handles `429`/`Retry-After` and backoff, because rate limits apply per app, not per project.
@@ -145,7 +114,7 @@ items PK (project_id, source_id)
-- desired (from SharePoint) -- desired (from SharePoint)
parent_source_id, name, is_folder, content_tag, source_modified_at, size, deleted parent_source_id, name, is_folder, content_tag, source_modified_at, size, deleted
-- synced (what ACC has) -- synced (what ACC has)
target_id, synced_parent_target_id, synced_name, synced_content_tag, synced_at target_id, synced_parent_source_id, synced_name, synced_content_tag, synced_deleted, synced_at
-- bookkeeping -- bookkeeping
status ('synced' | 'pending' | 'failed' | 'deleted') status ('synced' | 'pending' | 'failed' | 'deleted')
attempt_count, last_error, last_attempt_at attempt_count, last_error, last_attempt_at
@@ -153,12 +122,15 @@ items PK (project_id, source_id)
runs runs
id, project_id, trigger ('schedule' | 'manual'), started_at, finished_at id, project_id, trigger ('schedule' | 'manual'), started_at, finished_at
outcome ('success' | 'partial' | 'failed'), error outcome ('success' | 'partial' | 'failed'), error
created, updated, renamed, moved, deleted, failed -- counts created, updated, renamed, moved, deleted, restored, failed -- counts
run_events run_events
id, run_id, project_id, source_id, path, action, outcome, message, at id, run_id, project_id, source_id, path, action, message, at
-- action: created | updated | renamed | moved | deleted | restored | failed
``` ```
The synced parent is stored as a SharePoint ID, so "was it moved?" is a plain comparison with the desired parent. The ACC folder to move into is looked up from the parent item when needed.
- IDs are stored, not paths, so renaming a root folder doesn't break a project. Display paths are shown in the UI only. - IDs are stored, not paths, so renaming a root folder doesn't break a project. Display paths are shown in the UI only.
- **Issues** = items with `status = 'failed'`, plus a project-level issue when it is paused or its latest run failed outright (bad auth, root missing). - **Issues** = items with `status = 'failed'`, plus a project-level issue when it is paused or its latest run failed outright (bad auth, root missing).
- A nightly job deletes `runs` and `run_events` older than `HISTORY_RETENTION_DAYS`. `items` is current state, so it is never pruned. - A nightly job deletes `runs` and `run_events` older than `HISTORY_RETENTION_DAYS`. `items` is current state, so it is never pruned.
@@ -169,7 +141,7 @@ run_events
## UI ## UI
| Route | Who | What | | Route | Who | What |
| --- | --- | --- | | --------------------------------------------- | ------ | ---------------------------------------------------------------------------------------------------------------------- |
| `/` | anyone | Projects list: status, last run, open issue count | | `/` | anyone | Projects list: status, last run, open issue count |
| `/projects/[id]` | anyone | File tree with status badges and last update time, an issues panel and a "Run now" button | | `/projects/[id]` | anyone | File tree with status badges and last update time, an issues panel and a "Run now" button |
| `/projects/[id]/history` | anyone | Runs from the last 7 days, expandable to per-file events | | `/projects/[id]/history` | anyone | Runs from the last 7 days, expandable to per-file events |
@@ -192,25 +164,34 @@ MAX_ATTEMPTS=3
HISTORY_RETENTION_DAYS=7 HISTORY_RETENTION_DAYS=7
``` ```
Env is validated with `zod` at startup; the server refuses to start on bad config. Run exactly **one** replica, because the locks live in-process. Env vars are declared and validated with `zod` in `src/env.ts` (SvelteKit's `defineEnvVars`); the server refuses to start on bad config. Only the variables in use so far are declared; the Graph, APS and admin ones get added along with the code that reads them. Run exactly **one** replica, because the locks live in-process.
## Source layout ## Source layout
``` ```
src/env.ts # env var declarations + validation
src/lib/server/ src/lib/server/
config.ts # zod-validated env container.ts # (todo) composition root: the only place adapters are chosen
container.ts # composition root: the only place adapters are chosen
sync/ sync/
core/ # ports, SyncEngine, reconcile planner, IgnorePolicy (pure TS) core/ # pure TS, no SvelteKit imports
coordinator.ts # locks, cooldown, scheduler loop, retention job model.ts ports.ts errors.ts
ingest.ts # SharePoint changes → desired state
plan.ts # desired vs synced → action, status, work order
reconciler.ts # applies actions to ACC
engine.ts # one run of one project
tree.ts ignore.ts
coordinator.ts # per-project lock + manual cooldown
scheduler.ts # non-overlapping interval tasks
testing/ # FakeDrive, FakeAcc, harness (real engine + in-memory SQLite)
adapters/ adapters/
graph/ # GraphChangeSource, auth (msal-node) sqlite/ # Drizzle schema, migrations runner, repositories
aps/ # ApsDocumentTarget, auth, upload flow graph/ # (todo) GraphChangeSource, auth (msal-node)
sqlite/ # Drizzle schema + repositories aps/ # (todo) ApsDocumentTarget, auth, upload flow
http/ # shared fetch wrapper: retry, backoff, Retry-After http/ # (todo) shared fetch wrapper: retry, backoff, Retry-After
auth/ # admin session cookie auth/ # (todo) admin session cookie
src/hooks.server.ts # init → container → coordinator.start(); admin route guard drizzle/ # generated SQL migrations (npm run db:generate)
src/routes/ # UI src/hooks.server.ts # (todo) init → container → scheduler; admin route guard
src/routes/ # (todo) UI
``` ```
## Open questions (resolved by spikes) ## Open questions (resolved by spikes)
@@ -224,11 +205,11 @@ src/routes/ # UI
0. **Access requests:** start now; they're the slowest part. See [access-setup.md](access-setup.md). 0. **Access requests:** start now; they're the slowest part. See [access-setup.md](access-setup.md).
1. **Spikes** against a sandbox ACC project and test SharePoint site. Answer the open questions above and record the results here. 1. **Spikes** against a sandbox ACC project and test SharePoint site. Answer the open questions above and record the results here.
2. **Scaffold:** SvelteKit + adapter-node, TypeScript strict, Drizzle + better-sqlite3, zod, Vitest, ESLint/Prettier, Dockerfile. 2. ✅ **Scaffold:** SvelteKit 3 + adapter-node, TypeScript strict, Drizzle + better-sqlite3, zod, Vitest, ESLint/Prettier. (Dockerfile moves to step 9.)
3. **Core:** ports, IgnorePolicy, reconcile planner and SyncEngine, unit-tested against in-memory fakes. Most of the logic lives here. 3. ✅ **Core:** ports, IgnorePolicy, ingest, planner, reconciler and SyncEngine, tested against FakeDrive/FakeAcc.
4. **SQLite repositories** and migrations. 4. ✅ **SQLite repositories** and migrations.
5. **GraphChangeSource.** 5. **GraphChangeSource.**
6. **ApsDocumentTarget**, including the fallbacks chosen in step 1. 6. **ApsDocumentTarget**, including the fallbacks chosen in step 1.
7. **Coordinator:** scheduler, locks, cooldown, retention job, hooked into `init`. 7. **Wiring:** ✅ coordinator, scheduler. Still to do: composition root, `init` hook, retention job schedule.
8. **UI:** status pages, Run now, admin login, project add/edit/reset. 8. **UI:** status pages, Run now, admin login, project add/edit/reset.
9. **Ship:** Docker image, Portainer stack file, runbook. 9. **Ship:** Docker image, Portainer stack file, runbook.

10
drizzle.config.ts Normal file
View File

@@ -0,0 +1,10 @@
import { defineConfig } from 'drizzle-kit';
export default defineConfig({
schema: './src/lib/server/adapters/sqlite/schema.ts',
out: './drizzle',
dialect: 'sqlite',
dbCredentials: { url: process.env.DATABASE_PATH ?? 'local.db' },
verbose: true,
strict: true
});

76
drizzle/0000_init.sql Normal file
View File

@@ -0,0 +1,76 @@
CREATE TABLE `items` (
`project_id` text NOT NULL,
`source_id` text NOT NULL,
`parent_source_id` text NOT NULL,
`name` text NOT NULL,
`is_folder` integer NOT NULL,
`content_tag` text,
`source_modified_at` integer NOT NULL,
`size` integer,
`deleted` integer NOT NULL,
`target_id` text,
`synced_parent_source_id` text,
`synced_name` text,
`synced_content_tag` text,
`synced_deleted` integer NOT NULL,
`synced_at` integer,
`status` text NOT NULL,
`attempt_count` integer DEFAULT 0 NOT NULL,
`last_error` text,
`last_attempt_at` integer,
PRIMARY KEY(`project_id`, `source_id`),
FOREIGN KEY (`project_id`) REFERENCES `projects`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `items_project_status` ON `items` (`project_id`,`status`);--> statement-breakpoint
CREATE TABLE `projects` (
`id` text PRIMARY KEY NOT NULL,
`name` text NOT NULL,
`enabled` integer DEFAULT true NOT NULL,
`paused_reason` text,
`sp_site_id` text NOT NULL,
`sp_drive_id` text NOT NULL,
`sp_root_item_id` text NOT NULL,
`sp_display_path` text NOT NULL,
`acc_hub_id` text NOT NULL,
`acc_project_id` text NOT NULL,
`acc_root_folder_id` text NOT NULL,
`acc_display_path` text NOT NULL,
`delta_cursor` text,
`first_synced_at` integer,
`last_run_at` integer,
`created_at` integer NOT NULL
);
--> statement-breakpoint
CREATE TABLE `run_events` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`run_id` text NOT NULL,
`project_id` text NOT NULL,
`source_id` text NOT NULL,
`path` text NOT NULL,
`action` text NOT NULL,
`message` text,
`at` integer NOT NULL,
FOREIGN KEY (`run_id`) REFERENCES `runs`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `run_events_run` ON `run_events` (`run_id`);--> statement-breakpoint
CREATE TABLE `runs` (
`id` text PRIMARY KEY NOT NULL,
`project_id` text NOT NULL,
`trigger` text NOT NULL,
`started_at` integer NOT NULL,
`finished_at` integer,
`outcome` text,
`error` text,
`created` integer DEFAULT 0 NOT NULL,
`updated` integer DEFAULT 0 NOT NULL,
`renamed` integer DEFAULT 0 NOT NULL,
`moved` integer DEFAULT 0 NOT NULL,
`deleted` integer DEFAULT 0 NOT NULL,
`restored` integer DEFAULT 0 NOT NULL,
`failed` integer DEFAULT 0 NOT NULL,
FOREIGN KEY (`project_id`) REFERENCES `projects`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `runs_project_started` ON `runs` (`project_id`,`started_at`);

View File

@@ -0,0 +1,542 @@
{
"version": "6",
"dialect": "sqlite",
"id": "a7309db7-6811-48f5-bb28-822a92adf5d5",
"prevId": "00000000-0000-0000-0000-000000000000",
"tables": {
"items": {
"name": "items",
"columns": {
"project_id": {
"name": "project_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"source_id": {
"name": "source_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"parent_source_id": {
"name": "parent_source_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"is_folder": {
"name": "is_folder",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"content_tag": {
"name": "content_tag",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"source_modified_at": {
"name": "source_modified_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"size": {
"name": "size",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"deleted": {
"name": "deleted",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"target_id": {
"name": "target_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"synced_parent_source_id": {
"name": "synced_parent_source_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"synced_name": {
"name": "synced_name",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"synced_content_tag": {
"name": "synced_content_tag",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"synced_deleted": {
"name": "synced_deleted",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"synced_at": {
"name": "synced_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"attempt_count": {
"name": "attempt_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"last_error": {
"name": "last_error",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"last_attempt_at": {
"name": "last_attempt_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"items_project_status": {
"name": "items_project_status",
"columns": [
"project_id",
"status"
],
"isUnique": false
}
},
"foreignKeys": {
"items_project_id_projects_id_fk": {
"name": "items_project_id_projects_id_fk",
"tableFrom": "items",
"tableTo": "projects",
"columnsFrom": [
"project_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {
"items_project_id_source_id_pk": {
"columns": [
"project_id",
"source_id"
],
"name": "items_project_id_source_id_pk"
}
},
"uniqueConstraints": {},
"checkConstraints": {}
},
"projects": {
"name": "projects",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"enabled": {
"name": "enabled",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": true
},
"paused_reason": {
"name": "paused_reason",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"sp_site_id": {
"name": "sp_site_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"sp_drive_id": {
"name": "sp_drive_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"sp_root_item_id": {
"name": "sp_root_item_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"sp_display_path": {
"name": "sp_display_path",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"acc_hub_id": {
"name": "acc_hub_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"acc_project_id": {
"name": "acc_project_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"acc_root_folder_id": {
"name": "acc_root_folder_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"acc_display_path": {
"name": "acc_display_path",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"delta_cursor": {
"name": "delta_cursor",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"first_synced_at": {
"name": "first_synced_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"last_run_at": {
"name": "last_run_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"run_events": {
"name": "run_events",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"run_id": {
"name": "run_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"project_id": {
"name": "project_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"source_id": {
"name": "source_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"path": {
"name": "path",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"action": {
"name": "action",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"message": {
"name": "message",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"at": {
"name": "at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"run_events_run": {
"name": "run_events_run",
"columns": [
"run_id"
],
"isUnique": false
}
},
"foreignKeys": {
"run_events_run_id_runs_id_fk": {
"name": "run_events_run_id_runs_id_fk",
"tableFrom": "run_events",
"tableTo": "runs",
"columnsFrom": [
"run_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"runs": {
"name": "runs",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"project_id": {
"name": "project_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"trigger": {
"name": "trigger",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"started_at": {
"name": "started_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"finished_at": {
"name": "finished_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"outcome": {
"name": "outcome",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"error": {
"name": "error",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created": {
"name": "created",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"updated": {
"name": "updated",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"renamed": {
"name": "renamed",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"moved": {
"name": "moved",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"deleted": {
"name": "deleted",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"restored": {
"name": "restored",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"failed": {
"name": "failed",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
}
},
"indexes": {
"runs_project_started": {
"name": "runs_project_started",
"columns": [
"project_id",
"started_at"
],
"isUnique": false
}
},
"foreignKeys": {
"runs_project_id_projects_id_fk": {
"name": "runs_project_id_projects_id_fk",
"tableFrom": "runs",
"tableTo": "projects",
"columnsFrom": [
"project_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
}
},
"views": {},
"enums": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
},
"internal": {
"indexes": {}
}
}

View File

@@ -0,0 +1,13 @@
{
"version": "7",
"dialect": "sqlite",
"entries": [
{
"idx": 0,
"version": "6",
"when": 1792725790862,
"tag": "0000_init",
"breakpoints": true
}
]
}

42
eslint.config.js Normal file
View File

@@ -0,0 +1,42 @@
import prettier from 'eslint-config-prettier';
import path from 'node:path';
import js from '@eslint/js';
import svelte from 'eslint-plugin-svelte';
import { defineConfig, includeIgnoreFile } from 'eslint/config';
import globals from 'globals';
import ts from 'typescript-eslint';
const gitignorePath = path.resolve(import.meta.dirname, '.gitignore');
export default defineConfig(
includeIgnoreFile(gitignorePath),
js.configs.recommended,
ts.configs.recommended,
svelte.configs.recommended,
prettier,
svelte.configs.prettier,
{
languageOptions: { globals: { ...globals.browser, ...globals.node } },
rules: {
// typescript-eslint strongly recommend that you do not use the no-undef lint rule on TypeScript projects.
// see: https://typescript-eslint.io/troubleshooting/faqs/eslint/#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors
'no-undef': 'off'
}
},
{
files: ['**/*.svelte', '**/*.svelte.ts', '**/*.svelte.js'],
languageOptions: {
parserOptions: {
projectService: true,
extraFileExtensions: ['.svelte'],
parser: ts.parser
}
}
},
{
rules: {
// Allow `const { omitted, ...rest } = obj` to drop fields.
'@typescript-eslint/no-unused-vars': ['error', { ignoreRestSiblings: true }]
}
}
);

4969
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

52
package.json Normal file
View File

@@ -0,0 +1,52 @@
{
"name": "sharepoint-acc-sync",
"private": true,
"version": "0.0.1",
"type": "module",
"scripts": {
"dev": "vite dev",
"build": "vite build",
"preview": "vite preview",
"prepare": "svelte-kit sync || echo ''",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
"lint": "prettier --check . && eslint .",
"format": "prettier --write .",
"test:unit": "vitest",
"test": "npm run test:unit -- --run",
"db:push": "drizzle-kit push",
"db:generate": "drizzle-kit generate",
"db:migrate": "drizzle-kit migrate",
"db:studio": "drizzle-kit studio"
},
"devDependencies": {
"@eslint/js": "^10.0.1",
"@sveltejs/adapter-node": "^6.0.0",
"@sveltejs/kit": "^3.0.0",
"@sveltejs/vite-plugin-svelte": "^7.2.0",
"@types/better-sqlite3": "^9.6.0",
"@types/node": "^24",
"drizzle-kit": "^0.31.10",
"drizzle-orm": "^0.45.2",
"eslint": "^10.4.1",
"eslint-config-prettier": "^10.1.8",
"eslint-plugin-svelte": "^3.19.0",
"globals": "^17.6.0",
"prettier": "^3.8.3",
"prettier-plugin-svelte": "^4.1.0",
"svelte": "^5.57.1",
"svelte-check": "^4.6.0",
"typescript": "^6.0.3",
"typescript-eslint": "^8.60.1",
"vite": "^8.3.0",
"vitest": "^4.1.8"
},
"imports": {
"#lib": "./src/lib/index.js",
"#lib/*": "./src/lib/*"
},
"dependencies": {
"better-sqlite3": "^13.0.2",
"zod": "^4.6.7"
}
}

11
prettier.config.js Normal file
View File

@@ -0,0 +1,11 @@
/** @type {import("prettier").Config} */
const config = {
useTabs: true,
singleQuote: true,
trailingComma: 'none',
printWidth: 100,
plugins: ['prettier-plugin-svelte'],
overrides: [{ files: '*.svelte', options: { parser: 'svelte' } }]
};
export default config;

13
src/app.d.ts vendored Normal file
View File

@@ -0,0 +1,13 @@
// See https://svelte.dev/docs/kit/types#app.d.ts
// for information about these interfaces
declare global {
namespace App {
// interface Error {}
// interface Locals {}
// interface PageData {}
// interface PageState {}
// interface Platform {}
}
}
export {};

12
src/app.html Normal file
View File

@@ -0,0 +1,12 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="text-scale" content="scale" />
%sveltekit.head%
</head>
<body data-sveltekit-preload-data="hover">
<div style="display: contents">%sveltekit.body%</div>
</body>
</html>

28
src/env.ts Normal file
View File

@@ -0,0 +1,28 @@
import { defineEnvVars } from '@sveltejs/kit/env';
import { z } from 'zod';
const positiveInt = (fallback: number) =>
z.coerce.number<string | undefined>().int().positive().default(fallback);
export const variables = defineEnvVars({
DATABASE_PATH: {
description: 'Path to the SQLite database file (mount a volume here in Docker).',
schema: z.string().min(1).default('local.db')
},
POLL_INTERVAL_SECONDS: {
description: 'How often every project is checked for SharePoint changes.',
schema: positiveInt(180)
},
MANUAL_RUN_COOLDOWN_SECONDS: {
description: 'Minimum gap between "Run now" requests for the same project.',
schema: positiveInt(30)
},
MAX_ATTEMPTS: {
description: 'Failed attempts before an item is reported as an issue.',
schema: positiveInt(3)
},
HISTORY_RETENTION_DAYS: {
description: 'How long run history is kept.',
schema: positiveInt(7)
}
});

View File

@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="107" height="128" viewBox="0 0 107 128"><title>svelte-logo</title><path d="M94.157 22.819c-10.4-14.885-30.94-19.297-45.792-9.835L22.282 29.608A29.92 29.92 0 0 0 8.764 49.65a31.5 31.5 0 0 0 3.108 20.231 30 30 0 0 0-4.477 11.183 31.9 31.9 0 0 0 5.448 24.116c10.402 14.887 30.942 19.297 45.791 9.835l26.083-16.624A29.92 29.92 0 0 0 98.235 78.35a31.53 31.53 0 0 0-3.105-20.232 30 30 0 0 0 4.474-11.182 31.88 31.88 0 0 0-5.447-24.116" style="fill:#ff3e00"/><path d="M45.817 106.582a20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.503 18 18 0 0 1 .624-2.435l.49-1.498 1.337.981a33.6 33.6 0 0 0 10.203 5.098l.97.294-.09.968a5.85 5.85 0 0 0 1.052 3.878 6.24 6.24 0 0 0 6.695 2.485 5.8 5.8 0 0 0 1.603-.704L69.27 76.28a5.43 5.43 0 0 0 2.45-3.631 5.8 5.8 0 0 0-.987-4.371 6.24 6.24 0 0 0-6.698-2.487 5.7 5.7 0 0 0-1.6.704l-9.953 6.345a19 19 0 0 1-5.296 2.326 20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.502 17.99 17.99 0 0 1 8.13-12.052l26.081-16.623a19 19 0 0 1 5.3-2.329 20.72 20.72 0 0 1 22.237 8.243 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-.624 2.435l-.49 1.498-1.337-.98a33.6 33.6 0 0 0-10.203-5.1l-.97-.294.09-.968a5.86 5.86 0 0 0-1.052-3.878 6.24 6.24 0 0 0-6.696-2.485 5.8 5.8 0 0 0-1.602.704L37.73 51.72a5.42 5.42 0 0 0-2.449 3.63 5.79 5.79 0 0 0 .986 4.372 6.24 6.24 0 0 0 6.698 2.486 5.8 5.8 0 0 0 1.602-.704l9.952-6.342a19 19 0 0 1 5.295-2.328 20.72 20.72 0 0 1 22.237 8.242 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-8.13 12.053l-26.081 16.622a19 19 0 0 1-5.3 2.328" style="fill:#fff"/></svg>

After

Width:  |  Height:  |  Size: 1.5 KiB

1
src/lib/index.ts Normal file
View File

@@ -0,0 +1 @@
// place files you want to import through the `#lib` alias in this folder.

View File

@@ -0,0 +1,16 @@
import Database from 'better-sqlite3';
import { drizzle, type BetterSQLite3Database } from 'drizzle-orm/better-sqlite3';
import { migrate } from 'drizzle-orm/better-sqlite3/migrator';
import * as schema from './schema';
export type Db = BetterSQLite3Database<typeof schema>;
/** Opens (or creates) the database and brings its schema up to date. */
export function openDatabase(path: string, migrationsFolder = 'drizzle'): Db {
const client = new Database(path);
client.pragma('journal_mode = WAL');
client.pragma('foreign_keys = ON');
const db = drizzle(client, { schema });
migrate(db, { migrationsFolder });
return db;
}

View File

@@ -0,0 +1,88 @@
import { eq } from 'drizzle-orm';
import type { ItemState } from '../../sync/core/model';
import type { ItemRepository } from '../../sync/core/ports';
import type { Db } from './database';
import { items, projects } from './schema';
type ItemRow = typeof items.$inferSelect;
type Tx = Parameters<Parameters<Db['transaction']>[0]>[0];
export class SqliteItemRepository implements ItemRepository {
constructor(private readonly db: Db) {}
async listByProject(projectId: string): Promise<ItemState[]> {
return this.db.select().from(items).where(eq(items.projectId, projectId)).all().map(toItem);
}
async commitIngest(projectId: string, states: ItemState[], cursor: string): Promise<void> {
this.db.transaction((tx) => {
upsert(tx, projectId, states);
tx.update(projects).set({ deltaCursor: cursor }).where(eq(projects.id, projectId)).run();
});
}
async save(projectId: string, states: ItemState[]): Promise<void> {
this.db.transaction((tx) => upsert(tx, projectId, states));
}
}
function upsert(tx: Tx, projectId: string, states: ItemState[]): void {
for (const state of states) {
const row = toRow(projectId, state);
tx.insert(items)
.values(row)
.onConflictDoUpdate({ target: [items.projectId, items.sourceId], set: row })
.run();
}
}
function toRow(projectId: string, { sourceId, desired, synced, ...rest }: ItemState): ItemRow {
return {
projectId,
sourceId,
parentSourceId: desired.parentSourceId,
name: desired.name,
isFolder: desired.isFolder,
contentTag: desired.contentTag,
sourceModifiedAt: desired.modifiedAt,
size: desired.size,
deleted: desired.deleted,
targetId: synced.targetId,
syncedParentSourceId: synced.parentSourceId,
syncedName: synced.name,
syncedContentTag: synced.contentTag,
syncedDeleted: synced.deleted,
syncedAt: synced.at,
status: rest.status,
attemptCount: rest.attemptCount,
lastError: rest.lastError,
lastAttemptAt: rest.lastAttemptAt
};
}
function toItem(row: ItemRow): ItemState {
return {
sourceId: row.sourceId,
desired: {
parentSourceId: row.parentSourceId,
name: row.name,
isFolder: row.isFolder,
contentTag: row.contentTag,
modifiedAt: row.sourceModifiedAt,
size: row.size,
deleted: row.deleted
},
synced: {
targetId: row.targetId,
parentSourceId: row.syncedParentSourceId,
name: row.syncedName,
contentTag: row.syncedContentTag,
deleted: row.syncedDeleted,
at: row.syncedAt
},
status: row.status,
attemptCount: row.attemptCount,
lastError: row.lastError,
lastAttemptAt: row.lastAttemptAt
};
}

View File

@@ -0,0 +1,95 @@
import { eq } from 'drizzle-orm';
import type { NewProject, Project } from '../../sync/core/model';
import type { Clock, ProjectRepository } from '../../sync/core/ports';
import type { Db } from './database';
import { items, projects } from './schema';
type ProjectRow = typeof projects.$inferSelect;
export class SqliteProjectRepository implements ProjectRepository {
constructor(
private readonly db: Db,
private readonly clock: Clock
) {}
async get(id: string): Promise<Project | null> {
const row = this.db.select().from(projects).where(eq(projects.id, id)).get();
return row ? toProject(row) : null;
}
async list(): Promise<Project[]> {
return this.db.select().from(projects).orderBy(projects.name).all().map(toProject);
}
async create(project: NewProject): Promise<Project> {
const row = this.db
.insert(projects)
.values({
id: crypto.randomUUID(),
name: project.name,
spSiteId: project.source.siteId,
spDriveId: project.source.driveId,
spRootItemId: project.source.rootItemId,
spDisplayPath: project.sourceDisplayPath,
accHubId: project.target.hubId,
accProjectId: project.target.projectId,
accRootFolderId: project.target.rootFolderId,
accDisplayPath: project.targetDisplayPath,
createdAt: this.clock()
})
.returning()
.get();
return toProject(row);
}
async pause(id: string, reason: string): Promise<void> {
this.update(id, { pausedReason: reason });
}
async resume(id: string): Promise<void> {
this.update(id, { pausedReason: null });
}
async markFirstSynced(id: string, at: Date): Promise<void> {
this.update(id, { firstSyncedAt: at });
}
async setLastRunAt(id: string, at: Date): Promise<void> {
this.update(id, { lastRunAt: at });
}
async reset(id: string): Promise<void> {
this.db.transaction((tx) => {
tx.delete(items).where(eq(items.projectId, id)).run();
tx.update(projects)
.set({ deltaCursor: null, firstSyncedAt: null, pausedReason: null })
.where(eq(projects.id, id))
.run();
});
}
private update(id: string, values: Partial<ProjectRow>): void {
this.db.update(projects).set(values).where(eq(projects.id, id)).run();
}
}
function toProject(row: ProjectRow): Project {
return {
id: row.id,
name: row.name,
enabled: row.enabled,
pausedReason: row.pausedReason,
source: { siteId: row.spSiteId, driveId: row.spDriveId, rootItemId: row.spRootItemId },
sourceDisplayPath: row.spDisplayPath,
target: {
hubId: row.accHubId,
projectId: row.accProjectId,
rootFolderId: row.accRootFolderId
},
targetDisplayPath: row.accDisplayPath,
deltaCursor: row.deltaCursor,
firstSyncedAt: row.firstSyncedAt,
lastRunAt: row.lastRunAt,
createdAt: row.createdAt
};
}

View File

@@ -0,0 +1,28 @@
import { describe, expect, it } from 'vitest';
import { createHarness } from '../../sync/testing/harness';
describe('SqliteRunRepository', () => {
it('prunes old runs together with their events', async () => {
const h = await createHarness();
h.drive.addFile(h.rootId, 'a.txt');
const old = await h.sync();
const cutoff = new Date(Date.UTC(2026, 9, 2));
const recent = await h.runs.start(h.project.id, 'manual', new Date(Date.UTC(2026, 9, 3)));
expect(await h.runs.pruneBefore(cutoff)).toBe(1);
expect((await h.runs.listSince(h.project.id, new Date(0))).map((r) => r.runId)).toEqual([
recent
]);
expect(await h.runs.listEvents(old.runId)).toEqual([]);
});
it('reports an unfinished run with no outcome', async () => {
const h = await createHarness();
const runId = await h.runs.start(h.project.id, 'manual', new Date());
const [run] = await h.runs.listSince(h.project.id, new Date(0));
expect(run).toMatchObject({ runId, outcome: null, finishedAt: null });
});
});

View File

@@ -0,0 +1,71 @@
import { and, asc, desc, eq, gte, lt } from 'drizzle-orm';
import type { RunCounts, RunEvent, RunOutcome, RunRecord, RunTrigger } from '../../sync/core/model';
import type { RunRepository } from '../../sync/core/ports';
import type { Db } from './database';
import { runEvents, runs } from './schema';
type RunRow = typeof runs.$inferSelect;
export class SqliteRunRepository implements RunRepository {
constructor(private readonly db: Db) {}
async start(projectId: string, trigger: RunTrigger, at: Date): Promise<string> {
const id = crypto.randomUUID();
this.db.insert(runs).values({ id, projectId, trigger, startedAt: at }).run();
return id;
}
async finish(
runId: string,
result: { finishedAt: Date; outcome: RunOutcome; error: string | null; counts: RunCounts }
): Promise<void> {
const { finishedAt, outcome, error, counts } = result;
this.db
.update(runs)
.set({ finishedAt, outcome, error, ...counts })
.where(eq(runs.id, runId))
.run();
}
async addEvent(event: RunEvent): Promise<void> {
this.db.insert(runEvents).values(event).run();
}
async listSince(projectId: string, since: Date): Promise<RunRecord[]> {
return this.db
.select()
.from(runs)
.where(and(eq(runs.projectId, projectId), gte(runs.startedAt, since)))
.orderBy(desc(runs.startedAt))
.all()
.map(toRunRecord);
}
async listEvents(runId: string): Promise<RunEvent[]> {
return this.db
.select()
.from(runEvents)
.where(eq(runEvents.runId, runId))
.orderBy(asc(runEvents.id))
.all()
.map(({ id, ...event }) => event);
}
async pruneBefore(cutoff: Date): Promise<number> {
return this.db.delete(runs).where(lt(runs.startedAt, cutoff)).run().changes;
}
}
function toRunRecord(row: RunRow): RunRecord {
const { id, projectId, trigger, startedAt, finishedAt, outcome, error, ...counts } = row;
return {
runId: id,
projectId,
trigger,
startedAt,
finishedAt,
outcome,
error,
counts
};
}

View File

@@ -0,0 +1,99 @@
import { index, integer, primaryKey, sqliteTable, text } from 'drizzle-orm/sqlite-core';
import { EVENT_ACTIONS } from '../../sync/core/model';
const timestamp = (name: string) => integer(name, { mode: 'timestamp_ms' });
const flag = (name: string) => integer(name, { mode: 'boolean' });
const count = (name: string) => integer(name).notNull().default(0);
export const projects = sqliteTable('projects', {
id: text('id').primaryKey(),
name: text('name').notNull(),
enabled: flag('enabled').notNull().default(true),
pausedReason: text('paused_reason'),
spSiteId: text('sp_site_id').notNull(),
spDriveId: text('sp_drive_id').notNull(),
spRootItemId: text('sp_root_item_id').notNull(),
spDisplayPath: text('sp_display_path').notNull(),
accHubId: text('acc_hub_id').notNull(),
accProjectId: text('acc_project_id').notNull(),
accRootFolderId: text('acc_root_folder_id').notNull(),
accDisplayPath: text('acc_display_path').notNull(),
deltaCursor: text('delta_cursor'),
firstSyncedAt: timestamp('first_synced_at'),
lastRunAt: timestamp('last_run_at'),
createdAt: timestamp('created_at').notNull()
});
export const items = sqliteTable(
'items',
{
projectId: text('project_id')
.notNull()
.references(() => projects.id, { onDelete: 'cascade' }),
sourceId: text('source_id').notNull(),
// desired (from SharePoint)
parentSourceId: text('parent_source_id').notNull(),
name: text('name').notNull(),
isFolder: flag('is_folder').notNull(),
contentTag: text('content_tag'),
sourceModifiedAt: timestamp('source_modified_at').notNull(),
size: integer('size'),
deleted: flag('deleted').notNull(),
// synced (what ACC has)
targetId: text('target_id'),
syncedParentSourceId: text('synced_parent_source_id'),
syncedName: text('synced_name'),
syncedContentTag: text('synced_content_tag'),
syncedDeleted: flag('synced_deleted').notNull(),
syncedAt: timestamp('synced_at'),
// bookkeeping
status: text('status', { enum: ['pending', 'synced', 'failed', 'deleted'] }).notNull(),
attemptCount: count('attempt_count'),
lastError: text('last_error'),
lastAttemptAt: timestamp('last_attempt_at')
},
(t) => [
primaryKey({ columns: [t.projectId, t.sourceId] }),
index('items_project_status').on(t.projectId, t.status)
]
);
export const runs = sqliteTable(
'runs',
{
id: text('id').primaryKey(),
projectId: text('project_id')
.notNull()
.references(() => projects.id, { onDelete: 'cascade' }),
trigger: text('trigger', { enum: ['schedule', 'manual'] }).notNull(),
startedAt: timestamp('started_at').notNull(),
finishedAt: timestamp('finished_at'),
outcome: text('outcome', { enum: ['success', 'partial', 'failed'] }),
error: text('error'),
created: count('created'),
updated: count('updated'),
renamed: count('renamed'),
moved: count('moved'),
deleted: count('deleted'),
restored: count('restored'),
failed: count('failed')
},
(t) => [index('runs_project_started').on(t.projectId, t.startedAt)]
);
export const runEvents = sqliteTable(
'run_events',
{
id: integer('id').primaryKey({ autoIncrement: true }),
runId: text('run_id')
.notNull()
.references(() => runs.id, { onDelete: 'cascade' }),
projectId: text('project_id').notNull(),
sourceId: text('source_id').notNull(),
path: text('path').notNull(),
action: text('action', { enum: EVENT_ACTIONS }).notNull(),
message: text('message'),
at: timestamp('at').notNull()
},
(t) => [index('run_events_run').on(t.runId)]
);

View File

@@ -0,0 +1,115 @@
import { describe, expect, it } from 'vitest';
import type { ProjectSyncRunner } from './core/engine';
import { emptyCounts, type Project, type RunTrigger } from './core/model';
import type { ProjectRepository } from './core/ports';
import { SyncCoordinator } from './coordinator';
import { silentLogger } from './testing/harness';
function project(id: string, overrides: Partial<Project> = {}): Project {
return { id, enabled: true, pausedReason: null, ...overrides } as Project;
}
/** A runner whose runs stay in flight until `finish` is called. */
class ControlledRunner implements ProjectSyncRunner {
readonly started: Array<[string, RunTrigger]> = [];
private readonly pending: Array<() => void> = [];
runProject(projectId: string, trigger: RunTrigger) {
this.started.push([projectId, trigger]);
return new Promise<never>((resolve) => {
this.pending.push(() =>
resolve({ runId: 'r', outcome: 'success', error: null, counts: emptyCounts() } as never)
);
});
}
finishAll() {
this.pending.splice(0).forEach((finish) => finish());
}
}
function setup(projects: Project[]) {
let now = 0;
const runner = new ControlledRunner();
const repo = {
get: async (id: string) => projects.find((p) => p.id === id) ?? null,
list: async () => projects
} as ProjectRepository;
const coordinator = new SyncCoordinator({
runner,
projects: repo,
clock: () => new Date(now),
logger: silentLogger,
manualCooldownMs: 30_000
});
return { runner, coordinator, advance: (ms: number) => (now += ms) };
}
describe('SyncCoordinator', () => {
it('starts a manual run and reports it as already running on a second click', async () => {
const { runner, coordinator } = setup([project('p1')]);
expect(await coordinator.requestManualRun('p1')).toEqual({ status: 'started' });
expect(await coordinator.requestManualRun('p1')).toEqual({ status: 'already-running' });
expect(runner.started).toEqual([['p1', 'manual']]);
});
it('enforces the manual cooldown after a run finishes', async () => {
const { runner, coordinator, advance } = setup([project('p1')]);
await coordinator.requestManualRun('p1');
runner.finishAll();
await coordinator.drain();
advance(10_000);
expect(await coordinator.requestManualRun('p1')).toEqual({
status: 'cooldown',
retryAfterSeconds: 20
});
advance(20_000);
expect(await coordinator.requestManualRun('p1')).toEqual({ status: 'started' });
});
it('keeps cooldowns per project', async () => {
const { coordinator } = setup([project('p1'), project('p2')]);
await coordinator.requestManualRun('p1');
expect(await coordinator.requestManualRun('p2')).toEqual({ status: 'started' });
});
it('refuses manual runs for disabled, paused or unknown projects', async () => {
const { coordinator } = setup([
project('off', { enabled: false }),
project('paused', { pausedReason: 'Root deleted' })
]);
expect(await coordinator.requestManualRun('off')).toMatchObject({ status: 'unavailable' });
expect(await coordinator.requestManualRun('paused')).toEqual({
status: 'unavailable',
reason: 'Root deleted'
});
expect(await coordinator.requestManualRun('nope')).toMatchObject({ status: 'unavailable' });
});
it('runs scheduled passes sequentially, skipping busy, disabled and paused projects', async () => {
const { runner, coordinator } = setup([
project('busy'),
project('off', { enabled: false }),
project('paused', { pausedReason: 'x' }),
project('a'),
project('b')
]);
await coordinator.requestManualRun('busy');
const pass = coordinator.runScheduledPass();
await Promise.resolve();
await Promise.resolve();
expect(runner.started.map(([id]) => id)).toEqual(['busy', 'a']);
runner.finishAll();
await new Promise((r) => setTimeout(r));
expect(runner.started.map(([id]) => id)).toEqual(['busy', 'a', 'b']);
runner.finishAll();
await pass;
});
});

View File

@@ -0,0 +1,79 @@
import type { ProjectSyncRunner } from './core/engine';
import { errorMessage } from './core/errors';
import type { RunTrigger } from './core/model';
import type { Clock, Logger, ProjectRepository } from './core/ports';
export type RunRequestResult =
| { status: 'started' }
| { status: 'already-running' }
| { status: 'cooldown'; retryAfterSeconds: number }
| { status: 'unavailable'; reason: string };
export interface SyncCoordinatorDeps {
runner: ProjectSyncRunner;
projects: ProjectRepository;
clock: Clock;
logger: Logger;
manualCooldownMs: number;
}
/**
* The only way a sync run starts. Guarantees at most one run per project at a
* time and rate-limits manual "Run now" requests per project.
*/
export class SyncCoordinator {
private readonly active = new Map<string, Promise<void>>();
private readonly lastManualRequest = new Map<string, number>();
constructor(private readonly deps: SyncCoordinatorDeps) {}
isRunning(projectId: string): boolean {
return this.active.has(projectId);
}
async requestManualRun(projectId: string): Promise<RunRequestResult> {
const project = await this.deps.projects.get(projectId);
if (!project) return { status: 'unavailable', reason: 'Project not found.' };
if (!project.enabled) return { status: 'unavailable', reason: 'Project is disabled.' };
if (project.pausedReason) return { status: 'unavailable', reason: project.pausedReason };
if (this.isRunning(projectId)) return { status: 'already-running' };
const now = this.deps.clock().getTime();
const elapsed = now - (this.lastManualRequest.get(projectId) ?? -Infinity);
if (elapsed < this.deps.manualCooldownMs) {
return {
status: 'cooldown',
retryAfterSeconds: Math.ceil((this.deps.manualCooldownMs - elapsed) / 1000)
};
}
this.lastManualRequest.set(projectId, now);
void this.launch(projectId, 'manual');
return { status: 'started' };
}
/** Syncs every enabled, unpaused, idle project, one after another. */
async runScheduledPass(): Promise<void> {
for (const project of await this.deps.projects.list()) {
if (!project.enabled || project.pausedReason || this.isRunning(project.id)) continue;
await this.launch(project.id, 'schedule');
}
}
/** Resolves once every in-flight run has finished. */
async drain(): Promise<void> {
await Promise.all(this.active.values());
}
private launch(projectId: string, trigger: RunTrigger): Promise<void> {
const run = this.deps.runner
.runProject(projectId, trigger)
.then(() => undefined)
.catch((error) => {
this.deps.logger.error('Sync run crashed', { projectId, error: errorMessage(error) });
})
.finally(() => this.active.delete(projectId));
this.active.set(projectId, run);
return run;
}
}

View File

@@ -0,0 +1,243 @@
import { describe, expect, it } from 'vitest';
import { createHarness } from '../testing/harness';
import { ServiceUnavailableError } from './errors';
import { ROOT_DELETED } from './ingest';
import { WAITING_FOR_PARENT } from './reconciler';
describe('SyncEngine', () => {
it('mirrors the initial tree and ignores temp files', async () => {
const h = await createHarness();
const drawings = h.drive.addFolder(h.rootId, 'Drawings');
h.drive.addFile(drawings, 'plan.pdf');
h.drive.addFile(drawings, '~$plan.docx');
h.drive.addFile(h.rootId, 'notes.txt');
h.drive.addFile(h.drive.driveRootId, 'outside-root.txt');
const result = await h.sync();
expect(result.outcome).toBe('success');
expect(result.counts.created).toBe(3);
expect(h.acc.tree()).toEqual(['Drawings/', 'Drawings/plan.pdf@1', 'notes.txt@1']);
});
it('uploads edits as new versions', async () => {
const h = await createHarness();
const file = h.drive.addFile(h.rootId, 'spec.docx');
await h.sync();
h.drive.edit(file, 'spec v2');
const result = await h.sync();
expect(result.counts.updated).toBe(1);
expect(h.acc.tree()).toEqual(['spec.docx@2']);
expect(h.acc.latestContent('spec.docx')).toBe('spec v2');
});
it('applies a move and rename of the same file', async () => {
const h = await createHarness();
const a = h.drive.addFolder(h.rootId, 'A');
const b = h.drive.addFolder(h.rootId, 'B');
const file = h.drive.addFile(a, 'old.txt');
await h.sync();
h.drive.move(file, b);
h.drive.rename(file, 'new.txt');
const result = await h.sync();
expect(result.counts).toMatchObject({ moved: 1, renamed: 1, created: 0 });
expect(h.acc.tree()).toEqual(['A/', 'B/', 'B/new.txt@1']);
});
it('creates a folder before files moved into it', async () => {
const h = await createHarness();
const file = h.drive.addFile(h.rootId, 'report.pdf');
await h.sync();
const archive = h.drive.addFolder(h.rootId, 'Archive');
h.drive.move(file, archive);
await h.sync();
expect(h.acc.tree()).toEqual(['Archive/', 'Archive/report.pdf@1']);
});
it('deletes a folder with a single ACC call that covers its contents', async () => {
const h = await createHarness();
const folder = h.drive.addFolder(h.rootId, 'Old');
h.drive.addFile(folder, 'a.txt');
h.drive.addFile(folder, 'b.txt');
await h.sync();
h.drive.remove(folder);
const result = await h.sync();
expect(result.counts.deleted).toBe(1);
expect(h.acc.calls.filter((c) => c.startsWith('delete'))).toEqual(['delete Old']);
expect(h.acc.tree()).toEqual([]);
expect((await h.itemNamed('a.txt')).status).toBe('deleted');
});
it('treats items moved out of the root as deleted', async () => {
const h = await createHarness();
const file = h.drive.addFile(h.rootId, 'leaving.txt');
await h.sync();
h.drive.move(file, h.drive.driveRootId);
await h.sync();
expect(h.acc.tree()).toEqual([]);
});
it('treats a rename to an ignored name as a delete', async () => {
const h = await createHarness();
const file = h.drive.addFile(h.rootId, 'draft.docx');
await h.sync();
h.drive.rename(file, 'draft.tmp');
await h.sync();
expect(h.acc.tree()).toEqual([]);
});
it('revives the same ACC items when a deleted folder is restored', async () => {
const h = await createHarness();
const folder = h.drive.addFolder(h.rootId, 'Specs');
h.drive.addFile(folder, 'spec.docx');
await h.sync();
const before = await h.itemNamed('spec.docx');
h.drive.remove(folder);
await h.sync();
h.drive.restore(folder);
const result = await h.sync();
expect(result.counts.restored).toBe(2);
expect(result.counts.created).toBe(0);
expect(h.acc.tree()).toEqual(['Specs/', 'Specs/spec.docx@1']);
expect((await h.itemNamed('spec.docx')).synced.targetId).toBe(before.synced.targetId);
});
it('never touches files added to ACC by hand', async () => {
const h = await createHarness();
const folder = h.drive.addFolder(h.rootId, 'Shared');
await h.sync();
const accFolder = (await h.itemNamed('Shared')).synced.targetId!;
h.acc.addByHand(h.acc.rootFolderId, 'manual.pdf');
h.acc.addByHand(accFolder, 'inside.pdf');
h.drive.addFile(h.rootId, 'synced.txt');
h.drive.rename(folder, 'Shared Renamed');
await h.sync();
expect(h.acc.tree()).toEqual([
'Shared Renamed/',
'Shared Renamed/inside.pdf@1',
'manual.pdf@1',
'synced.txt@1'
]);
});
describe('failures', () => {
it('retries a failing item, then reports it after the last attempt', async () => {
const h = await createHarness({ maxAttempts: 3 });
h.drive.addFile(h.rootId, 'bad.dwg');
h.drive.addFile(h.rootId, 'good.txt');
h.acc.failOn('bad.dwg', new Error('Upload rejected'));
const first = await h.sync();
expect(first.outcome).toBe('partial');
expect((await h.itemNamed('bad.dwg')).status).toBe('pending');
await h.sync();
await h.sync();
const failed = await h.itemNamed('bad.dwg');
expect(failed).toMatchObject({
status: 'failed',
attemptCount: 3,
lastError: 'Upload rejected'
});
const uploadsBefore = h.acc.calls.length;
await h.sync();
expect(h.acc.calls.length).toBe(uploadsBefore);
expect(h.acc.tree()).toEqual(['good.txt@1']);
});
it('gives a failed item fresh attempts when it changes in SharePoint', async () => {
const h = await createHarness({ maxAttempts: 1 });
const file = h.drive.addFile(h.rootId, 'bad.dwg');
h.acc.failOn('bad.dwg', new Error('Upload rejected'), 1);
await h.sync();
expect((await h.itemNamed('bad.dwg')).status).toBe('failed');
h.drive.edit(file, 'fixed');
await h.sync();
expect((await h.itemNamed('bad.dwg')).status).toBe('synced');
expect(h.acc.latestContent('bad.dwg')).toBe('fixed');
});
it('holds children without charging attempts while their folder fails', async () => {
const h = await createHarness();
const folder = h.drive.addFolder(h.rootId, 'Broken');
h.drive.addFile(folder, 'child.txt');
h.acc.failOn('Broken', new Error('Folder create failed'), 1);
await h.sync();
expect(await h.itemNamed('child.txt')).toMatchObject({
status: 'pending',
attemptCount: 0,
lastError: WAITING_FOR_PARENT
});
await h.sync();
expect(h.acc.tree()).toEqual(['Broken/', 'Broken/child.txt@1']);
});
it('stops the run on an outage without charging attempts', async () => {
const h = await createHarness();
h.drive.addFile(h.rootId, 'a.txt');
h.drive.addFile(h.rootId, 'b.txt');
h.acc.failOn('a.txt', new ServiceUnavailableError('ACC is down'), 1);
const result = await h.sync();
expect(result).toMatchObject({ outcome: 'failed', error: 'ACC is down' });
expect((await h.itemNamed('a.txt')).attemptCount).toBe(0);
expect(h.acc.calls).toEqual(['uploadFile a.txt']);
await h.sync();
expect(h.acc.tree()).toEqual(['a.txt@1', 'b.txt@1']);
});
});
describe('root safety', () => {
it('pauses instead of deleting when the root folder is deleted', async () => {
const h = await createHarness();
h.drive.addFile(h.rootId, 'keep.txt');
await h.sync();
h.drive.remove(h.rootId);
const result = await h.sync();
expect(result).toMatchObject({ outcome: 'failed', error: ROOT_DELETED });
expect((await h.projects.get(h.project.id))!.pausedReason).toBe(ROOT_DELETED);
expect(h.acc.tree()).toEqual(['keep.txt@1']);
});
});
it('records run history and per-file events', async () => {
const h = await createHarness();
const folder = h.drive.addFolder(h.rootId, 'Docs');
h.drive.addFile(folder, 'a.txt');
const result = await h.sync();
const [run] = await h.runs.listSince(h.project.id, new Date(0));
expect(run).toMatchObject({ runId: result.runId, trigger: 'schedule', outcome: 'success' });
const events = await h.runs.listEvents(result.runId);
expect(events.map((e) => `${e.action} ${e.path}`)).toEqual([
'created Docs',
'created Docs/a.txt'
]);
});
});

View File

@@ -0,0 +1,104 @@
import { errorMessage } from './errors';
import { ingestChanges } from './ingest';
import {
emptyCounts,
type ItemState,
type Project,
type RunOutcome,
type RunResult,
type RunTrigger
} from './model';
import type {
ChangeSource,
Clock,
IgnorePolicy,
ItemRepository,
Logger,
ProjectRepository,
RunRepository
} from './ports';
import type { Reconciler } from './reconciler';
export interface ProjectSyncRunner {
runProject(projectId: string, trigger: RunTrigger): Promise<RunResult>;
}
export interface SyncEngineDeps {
source: ChangeSource;
projects: ProjectRepository;
items: ItemRepository;
runs: RunRepository;
reconciler: Reconciler;
ignore: IgnorePolicy;
clock: Clock;
logger: Logger;
maxAttempts: number;
}
/** Runs one sync of one project: ingest SharePoint changes, then reconcile ACC. */
export class SyncEngine implements ProjectSyncRunner {
constructor(private readonly deps: SyncEngineDeps) {}
async runProject(projectId: string, trigger: RunTrigger): Promise<RunResult> {
const { projects, items, runs, clock, logger } = this.deps;
const project = await projects.get(projectId);
if (!project) throw new Error(`Project ${projectId} not found.`);
const runId = await runs.start(projectId, trigger, clock());
let counts = emptyCounts();
let outcome: RunOutcome = 'failed';
let error: string | null;
try {
const stored = await items.listByProject(projectId);
const current = new Map(stored.map((item) => [item.sourceId, item]));
const pausedReason = await this.ingest(project, current);
if (pausedReason) {
await projects.pause(projectId, pausedReason);
error = pausedReason;
logger.warn('Project paused', { projectId, reason: pausedReason });
} else {
if (!project.firstSyncedAt) await projects.markFirstSynced(projectId, clock());
const result = await this.deps.reconciler.reconcile(project, runId, current);
counts = result.counts;
error = result.abortedWith;
outcome = result.abortedWith ? 'failed' : counts.failed > 0 ? 'partial' : 'success';
}
} catch (caught) {
error = errorMessage(caught);
logger.error('Sync run failed', { projectId, runId, error });
}
const finishedAt = clock();
await runs.finish(runId, { finishedAt, outcome, error, counts });
await projects.setLastRunAt(projectId, finishedAt);
return { runId, outcome, error, counts };
}
/**
* Reads SharePoint changes into `current` and commits them with their cursor.
* Follows up with one full listing when ingest asks for it. Returns a pause
* reason if the batch was rejected.
*/
private async ingest(project: Project, current: Map<string, ItemState>): Promise<string | null> {
let cursor = project.deltaCursor;
for (;;) {
const batch = await this.deps.source.getChanges(project.source, cursor);
const result = ingestChanges({
rootId: project.source.rootItemId,
items: current,
changes: batch.changes,
fullSync: batch.fullSync,
ignore: this.deps.ignore,
maxAttempts: this.deps.maxAttempts
});
if (result.kind === 'paused') return result.reason;
await this.deps.items.commitIngest(project.id, result.changed, batch.cursor);
for (const item of result.changed) current.set(item.sourceId, item);
if (!result.needsFullListing) return null;
cursor = null;
}
}
}

View File

@@ -0,0 +1,15 @@
/**
* A remote service can't be used right now (outage, network, auth, throttling
* that outlasted retries). Adapters throw this so the run stops early instead of
* charging a failed attempt to every item it would otherwise touch.
*/
export class ServiceUnavailableError extends Error {
constructor(message: string, options?: ErrorOptions) {
super(message, options);
this.name = 'ServiceUnavailableError';
}
}
export function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}

View File

@@ -0,0 +1,26 @@
import { describe, expect, it } from 'vitest';
import { PatternIgnorePolicy } from './ignore';
describe('PatternIgnorePolicy', () => {
const policy = new PatternIgnorePolicy();
it.each([
'~$Report.docx',
'~WRL0003.tmp',
'upload.TMP',
'.~lock.plan.odt#',
'thumbs.db',
'.DS_Store'
])('ignores %s', (name) => expect(policy.isIgnored(name)).toBe(true));
it.each(['Report.docx', 'tmp-notes.txt', 'plan.dwg', 'desktop.ini.bak'])('keeps %s', (name) =>
expect(policy.isIgnored(name)).toBe(false)
);
it('accepts custom patterns', () => {
const custom = new PatternIgnorePolicy(['*.bak', 'draft?.txt']);
expect(custom.isIgnored('a.bak')).toBe(true);
expect(custom.isIgnored('draft1.txt')).toBe(true);
expect(custom.isIgnored('draft10.txt')).toBe(false);
});
});

View File

@@ -0,0 +1,35 @@
import type { IgnorePolicy } from './ports';
/** Office lock files, temp/partial downloads and OS metadata. */
export const DEFAULT_IGNORE_PATTERNS = [
'~$*',
'~*.tmp',
'*.tmp',
'.~lock.*#',
'*.crdownload',
'*.partial',
'Thumbs.db',
'desktop.ini',
'.DS_Store'
];
/** Matches names against case-insensitive `*` / `?` wildcard patterns. */
export class PatternIgnorePolicy implements IgnorePolicy {
private readonly patterns: RegExp[];
constructor(patterns: readonly string[] = DEFAULT_IGNORE_PATTERNS) {
this.patterns = patterns.map(toRegExp);
}
isIgnored(name: string): boolean {
return this.patterns.some((pattern) => pattern.test(name));
}
}
function toRegExp(pattern: string): RegExp {
const body = pattern
.split('')
.map((ch) => (ch === '*' ? '.*' : ch === '?' ? '.' : ch.replace(/[.+^${}()|[\]\\#]/g, '\\$&')))
.join('');
return new RegExp(`^${body}$`, 'i');
}

View File

@@ -0,0 +1,123 @@
import { describe, expect, it } from 'vitest';
import { PatternIgnorePolicy } from './ignore';
import { ingestChanges, ROOT_MISSING, type IngestInput } from './ingest';
import type { ItemState, SourceChange, SourceItem } from './model';
const ROOT = 'root';
function source(sourceId: string, parentSourceId: string, overrides: Partial<SourceItem> = {}) {
return {
sourceId,
parentSourceId,
name: sourceId,
isFolder: false,
contentTag: 'c1',
modifiedAt: new Date(0),
size: 1,
...overrides
} satisfies SourceItem;
}
const upsert = (item: SourceItem): SourceChange => ({ kind: 'upsert', item });
const folder = (id: string, parent: string) =>
source(id, parent, { isFolder: true, contentTag: null });
function run(
changes: SourceChange[],
items: ItemState[] = [],
overrides: Partial<IngestInput> = {}
) {
const result = ingestChanges({
rootId: ROOT,
items: new Map(items.map((i) => [i.sourceId, i])),
changes,
fullSync: false,
ignore: new PatternIgnorePolicy(),
maxAttempts: 3,
...overrides
});
if (result.kind !== 'ok') throw new Error(result.reason);
return result;
}
const byId = (items: ItemState[]) => Object.fromEntries(items.map((i) => [i.sourceId, i]));
describe('ingestChanges', () => {
it('accepts children reported before their parent', () => {
const { changed } = run([upsert(source('file', 'dir')), upsert(folder('dir', ROOT))]);
expect(changed.map((i) => i.sourceId).sort()).toEqual(['dir', 'file']);
});
it('skips untracked items outside the root', () => {
const { changed } = run([upsert(source('elsewhere', 'other-folder'))]);
expect(changed).toEqual([]);
});
it('keeps the last change when an item appears twice', () => {
const { changed } = run([
upsert(source('f', ROOT, { name: 'first' })),
upsert(source('f', ROOT, { name: 'second' }))
]);
expect(changed[0].desired.name).toBe('second');
});
it('skips everything under an ignored folder', () => {
const ignore = { isIgnored: (name: string) => name === 'skip' };
const { changed } = run([upsert(folder('skip', ROOT)), upsert(source('f', 'skip'))], [], {
ignore
});
expect(changed).toEqual([]);
});
it('cascades a folder delete to everything tracked beneath it', () => {
const first = run([
upsert(folder('a', ROOT)),
upsert(folder('b', 'a')),
upsert(source('f', 'b'))
]).changed;
const { changed } = run([{ kind: 'delete', sourceId: 'a' }], first);
expect(changed.map((i) => [i.sourceId, i.desired.deleted]).sort()).toEqual([
['a', true],
['b', true],
['f', true]
]);
});
it('deletes tracked items missing from a full listing', () => {
const first = run([upsert(source('gone', ROOT)), upsert(source('kept', ROOT))]).changed;
const { changed } = run([upsert(folder(ROOT, 'drive')), upsert(source('kept', ROOT))], first, {
fullSync: true
});
expect(byId(changed).gone.desired.deleted).toBe(true);
expect(byId(changed).kept).toBeUndefined();
});
it('pauses when a full listing does not include the root', () => {
const result = ingestChanges({
rootId: ROOT,
items: new Map(),
changes: [],
fullSync: true,
ignore: new PatternIgnorePolicy(),
maxAttempts: 3
});
expect(result).toEqual({ kind: 'paused', reason: ROOT_MISSING });
});
it('keeps attempts for metadata-only changes but resets them for real ones', () => {
const [tracked] = run([upsert(source('f', ROOT))]).changed;
const failing = { ...tracked, attemptCount: 3, lastError: 'boom' };
const touched = run([upsert(source('f', ROOT, { modifiedAt: new Date(5) }))], [failing]);
expect(touched.changed[0].attemptCount).toBe(3);
const edited = run([upsert(source('f', ROOT, { contentTag: 'c2' }))], [failing]);
expect(edited.changed[0]).toMatchObject({ attemptCount: 0, lastError: null });
});
it('asks for a full listing when a deleted folder comes back', () => {
const first = run([upsert(folder('a', ROOT))]).changed;
const deleted = run([{ kind: 'delete', sourceId: 'a' }], first).changed;
expect(run([upsert(folder('a', ROOT))], deleted).needsFullListing).toBe(true);
});
});

View File

@@ -0,0 +1,200 @@
import type { DesiredState, ItemState, SourceChange, SourceItem } from './model';
import { withStatus } from './plan';
import type { IgnorePolicy } from './ports';
import { ItemTree } from './tree';
export const ROOT_DELETED = 'The SharePoint root folder was deleted.';
export const ROOT_MISSING = 'The SharePoint root folder was not found in a full listing.';
export interface IngestInput {
rootId: string;
items: ReadonlyMap<string, ItemState>;
changes: readonly SourceChange[];
fullSync: boolean;
ignore: IgnorePolicy;
maxAttempts: number;
}
export type IngestResult =
| {
kind: 'ok';
changed: ItemState[];
/**
* A deleted folder came back. Its contents may not be reported individually,
* so the caller should follow up with a full listing to revive them.
*/
needsFullListing: boolean;
}
| { kind: 'paused'; reason: string };
/**
* Folds a batch of SharePoint changes into the desired state of the project's
* items. Pure: returns only the items that changed and never touches ACC.
*
* Anything that leaves the root's scope (moved out, under an ignored or deleted
* folder, renamed to an ignored name, or absent from a full listing) is treated
* as deleted. If the root itself disappears the batch is rejected instead, so a
* misconfiguration can never cascade into deleting everything in ACC.
*/
export function ingestChanges(input: IngestInput): IngestResult {
const { rootId, changes, fullSync, ignore, maxAttempts } = input;
const latest = new Map<string, SourceChange>();
for (const change of changes) latest.set(changeId(change), change);
const rootChange = latest.get(rootId);
if (rootChange?.kind === 'delete') return { kind: 'paused', reason: ROOT_DELETED };
if (fullSync && !rootChange) return { kind: 'paused', reason: ROOT_MISSING };
latest.delete(rootId);
const working = new Map(input.items);
const touched = new Set<string>();
let revivedFolder = false;
const setDesired = (sourceId: string, desired: DesiredState) => {
const current = working.get(sourceId);
if (current && sameDesired(current.desired, desired)) return;
const base = current ?? newItem(sourceId, desired);
const resetAttempts = !current || actionableChange(current.desired, desired);
if (current?.desired.deleted && !desired.deleted && desired.isFolder) revivedFolder = true;
working.set(sourceId, {
...base,
desired,
...(resetAttempts && { attemptCount: 0, lastError: null })
});
touched.add(sourceId);
};
const markDeleted = (sourceId: string) => {
const current = working.get(sourceId);
if (current && !current.desired.deleted) {
setDesired(sourceId, { ...current.desired, deleted: true });
}
};
const upserts = new Map<string, SourceItem>();
for (const change of latest.values()) {
if (change.kind === 'upsert') upserts.set(change.item.sourceId, change.item);
else markDeleted(change.sourceId);
}
if (fullSync) {
for (const sourceId of input.items.keys()) if (!latest.has(sourceId)) markDeleted(sourceId);
}
const inScope = scopeResolver(rootId, upserts, working, ignore);
for (const item of upserts.values()) {
if (inScope(item.sourceId)) setDesired(item.sourceId, toDesired(item));
else markDeleted(item.sourceId);
}
const tree = new ItemTree(rootId, working);
for (const sourceId of [...touched]) {
const item = working.get(sourceId)!;
if (item.desired.deleted && item.desired.isFolder) {
for (const descendant of tree.desiredDescendants(sourceId)) markDeleted(descendant.sourceId);
}
}
return {
kind: 'ok',
changed: [...touched].map((id) => withStatus(working.get(id)!, maxAttempts)),
needsFullListing: revivedFolder && !fullSync
};
}
/**
* An item is in scope when every ancestor up to the root is a live, non-ignored
* folder. Newer information from this batch wins over stored state.
*/
function scopeResolver(
rootId: string,
upserts: ReadonlyMap<string, SourceItem>,
known: ReadonlyMap<string, ItemState>,
ignore: IgnorePolicy
): (sourceId: string) => boolean {
const memo = new Map<string, boolean>();
const resolve = (sourceId: string): boolean => {
if (sourceId === rootId) return true;
const cached = memo.get(sourceId);
if (cached !== undefined) return cached;
memo.set(sourceId, false); // guards against parent cycles
const fresh = upserts.get(sourceId);
const stored = known.get(sourceId);
let result: boolean;
if (fresh) {
result =
!ignore.isIgnored(fresh.name, fresh.isFolder) &&
fresh.parentSourceId !== null &&
resolveParent(fresh.parentSourceId);
} else {
result = !!stored && !stored.desired.deleted && resolveParent(stored.desired.parentSourceId);
}
memo.set(sourceId, result);
return result;
};
const resolveParent = (parentId: string): boolean => {
if (parentId === rootId) return true;
const isFolder = upserts.get(parentId)?.isFolder ?? known.get(parentId)?.desired.isFolder;
return isFolder === true && resolve(parentId);
};
return resolve;
}
function changeId(change: SourceChange): string {
return change.kind === 'upsert' ? change.item.sourceId : change.sourceId;
}
function toDesired(item: SourceItem): DesiredState {
return {
parentSourceId: item.parentSourceId!,
name: item.name,
isFolder: item.isFolder,
contentTag: item.contentTag,
modifiedAt: item.modifiedAt,
size: item.size,
deleted: false
};
}
function newItem(sourceId: string, desired: DesiredState): ItemState {
return {
sourceId,
desired,
synced: {
targetId: null,
parentSourceId: null,
name: null,
contentTag: null,
deleted: false,
at: null
},
status: 'pending',
attemptCount: 0,
lastError: null,
lastAttemptAt: null
};
}
/** Changes that alter what needs doing in ACC (as opposed to display metadata). */
function actionableChange(a: DesiredState, b: DesiredState): boolean {
return (
a.parentSourceId !== b.parentSourceId ||
a.name !== b.name ||
a.contentTag !== b.contentTag ||
a.deleted !== b.deleted
);
}
function sameDesired(a: DesiredState, b: DesiredState): boolean {
return (
!actionableChange(a, b) &&
a.isFolder === b.isFolder &&
a.size === b.size &&
a.modifiedAt.getTime() === b.modifiedAt.getTime()
);
}

View File

@@ -0,0 +1,145 @@
/** Where a project's files come from in SharePoint (all Graph IDs). */
export interface SourceRoot {
siteId: string;
driveId: string;
/** The folder whose contents are mirrored. Never stored as an item itself. */
rootItemId: string;
}
/** Where a project's files go in ACC (all APS IDs). */
export interface TargetRoot {
hubId: string;
projectId: string;
rootFolderId: string;
}
export interface Project {
id: string;
name: string;
enabled: boolean;
/** Set when the sync refuses to continue (e.g. the SharePoint root vanished). */
pausedReason: string | null;
source: SourceRoot;
sourceDisplayPath: string;
target: TargetRoot;
targetDisplayPath: string;
deltaCursor: string | null;
firstSyncedAt: Date | null;
lastRunAt: Date | null;
createdAt: Date;
}
export type NewProject = Pick<
Project,
'name' | 'source' | 'sourceDisplayPath' | 'target' | 'targetDisplayPath'
>;
/** An item as SharePoint currently describes it. */
export interface SourceItem {
sourceId: string;
parentSourceId: string | null;
name: string;
isFolder: boolean;
/** Changes only when file content changes (Graph `cTag`). Null for folders. */
contentTag: string | null;
modifiedAt: Date;
size: number | null;
}
export type SourceChange =
{ kind: 'upsert'; item: SourceItem } | { kind: 'delete'; sourceId: string };
export interface FileContent {
size: number;
stream: ReadableStream<Uint8Array>;
}
export interface TargetRef {
id: string;
isFolder: boolean;
}
/** What SharePoint says the item should look like. */
export interface DesiredState {
parentSourceId: string;
name: string;
isFolder: boolean;
contentTag: string | null;
modifiedAt: Date;
size: number | null;
deleted: boolean;
}
/** What we last successfully wrote to ACC. */
export interface SyncedState {
targetId: string | null;
parentSourceId: string | null;
name: string | null;
contentTag: string | null;
deleted: boolean;
at: Date | null;
}
export type ItemStatus = 'pending' | 'synced' | 'failed' | 'deleted';
export interface ItemState {
sourceId: string;
desired: DesiredState;
synced: SyncedState;
status: ItemStatus;
attemptCount: number;
lastError: string | null;
lastAttemptAt: Date | null;
}
export type SyncAction =
| { kind: 'create' }
| { kind: 'restore' }
| { kind: 'delete' }
| { kind: 'update'; move: boolean; rename: boolean; content: boolean };
export type RunTrigger = 'schedule' | 'manual';
export type RunOutcome = 'success' | 'partial' | 'failed';
export const EVENT_ACTIONS = [
'created',
'updated',
'renamed',
'moved',
'deleted',
'restored',
'failed'
] as const;
export type EventAction = (typeof EVENT_ACTIONS)[number];
export type RunCounts = Record<EventAction, number>;
export function emptyCounts(): RunCounts {
return Object.fromEntries(EVENT_ACTIONS.map((a) => [a, 0])) as RunCounts;
}
export interface RunResult {
runId: string;
outcome: RunOutcome;
error: string | null;
counts: RunCounts;
}
export interface RunRecord extends Omit<RunResult, 'outcome'> {
/** Null while the run is still in progress. */
outcome: RunOutcome | null;
projectId: string;
trigger: RunTrigger;
startedAt: Date;
finishedAt: Date | null;
}
export interface RunEvent {
runId: string;
projectId: string;
sourceId: string;
path: string;
action: EventAction;
message: string | null;
at: Date;
}

View File

@@ -0,0 +1,67 @@
import { describe, expect, it } from 'vitest';
import type { DesiredState, ItemState, SyncedState } from './model';
import { computeStatus, planAction } from './plan';
const desired: DesiredState = {
parentSourceId: 'p',
name: 'a.txt',
isFolder: false,
contentTag: 'c1',
modifiedAt: new Date(0),
size: 1,
deleted: false
};
const synced: SyncedState = {
targetId: 't',
parentSourceId: 'p',
name: 'a.txt',
contentTag: 'c1',
deleted: false,
at: new Date(0)
};
function item(d: Partial<DesiredState> = {}, s: Partial<SyncedState> = {}, attemptCount = 0) {
return {
sourceId: 'x',
desired: { ...desired, ...d },
synced: { ...synced, ...s },
status: 'pending',
attemptCount,
lastError: null,
lastAttemptAt: null
} satisfies ItemState;
}
describe('planAction', () => {
it.each([
['in sync', item(), null],
['never synced', item({}, { targetId: null }), { kind: 'create' }],
['deleted in ACC, alive in SharePoint', item({}, { deleted: true }), { kind: 'restore' }],
['deleted in SharePoint', item({ deleted: true }), { kind: 'delete' }],
['deleted before it was ever synced', item({ deleted: true }, { targetId: null }), null],
['already deleted in both', item({ deleted: true }, { deleted: true }), null],
[
'moved, renamed and edited',
item({ parentSourceId: 'q', name: 'b.txt', contentTag: 'c2' }),
{ kind: 'update', move: true, rename: true, content: true }
],
[
'folder (content tags ignored)',
item({ isFolder: true, contentTag: null }, { contentTag: null }),
null
]
])('%s', (_label, state, expected) => {
expect(planAction(state)).toEqual(expected);
});
});
describe('computeStatus', () => {
it.each([
[item(), 'synced'],
[item({ deleted: true }, { deleted: true }), 'deleted'],
[item({ name: 'new' }, {}, 2), 'pending'],
[item({ name: 'new' }, {}, 3), 'failed']
] as const)('%#: %s', (state, expected) => {
expect(computeStatus(state, 3)).toBe(expected);
});
});

View File

@@ -0,0 +1,39 @@
import type { ItemState, ItemStatus, SyncAction } from './model';
import type { ItemTree } from './tree';
/** The ACC operation needed to make the synced state match the desired state, if any. */
export function planAction({ desired, synced }: ItemState): SyncAction | null {
if (desired.deleted) return synced.targetId && !synced.deleted ? { kind: 'delete' } : null;
if (!synced.targetId) return { kind: 'create' };
if (synced.deleted) return { kind: 'restore' };
const move = desired.parentSourceId !== synced.parentSourceId;
const rename = desired.name !== synced.name;
const content = !desired.isFolder && desired.contentTag !== synced.contentTag;
return move || rename || content ? { kind: 'update', move, rename, content } : null;
}
export function computeStatus(item: ItemState, maxAttempts: number): ItemStatus {
if (!planAction(item)) return item.desired.deleted ? 'deleted' : 'synced';
return item.attemptCount >= maxAttempts ? 'failed' : 'pending';
}
export function withStatus(item: ItemState, maxAttempts: number): ItemState {
return { ...item, status: computeStatus(item, maxAttempts) };
}
/**
* Items with outstanding work and attempts left, ordered so parents exist before
* children: folders (shallow first), then files, then deletes (shallow first, so
* a folder delete covers its contents).
*/
export function workQueue(
items: Iterable<ItemState>,
tree: ItemTree,
maxAttempts: number
): ItemState[] {
const rank = (item: ItemState) => (item.desired.deleted ? 2 : item.desired.isFolder ? 0 : 1);
return [...items]
.filter((item) => computeStatus(item, maxAttempts) === 'pending')
.sort((a, b) => rank(a) - rank(b) || tree.depth(a.sourceId) - tree.depth(b.sourceId));
}

View File

@@ -0,0 +1,93 @@
import type {
FileContent,
ItemState,
NewProject,
Project,
RunEvent,
RunOutcome,
RunCounts,
RunRecord,
RunTrigger,
SourceChange,
SourceRoot,
TargetRef
} from './model';
export interface ChangeBatch {
changes: SourceChange[];
/** Cursor to pass next time. */
cursor: string;
/** True when this is a complete listing: anything tracked but absent has been removed. */
fullSync: boolean;
}
export interface ChangeSource {
/** Changes since `cursor`; a null cursor (or an expired one) yields a full listing. */
getChanges(root: SourceRoot, cursor: string | null): Promise<ChangeBatch>;
download(root: SourceRoot, sourceId: string): Promise<FileContent>;
}
/**
* The destination document store. Implementations own any fallbacks they need
* (e.g. move by copy + delete), so callers never branch on target quirks.
*/
export interface DocumentTarget {
createFolder(parentId: string, name: string): Promise<TargetRef>;
uploadFile(parentId: string, name: string, content: FileContent): Promise<TargetRef>;
uploadVersion(ref: TargetRef, name: string, content: FileContent): Promise<void>;
rename(ref: TargetRef, name: string): Promise<void>;
/** May return a different ref if the move had to be done by copy + delete. */
move(ref: TargetRef, newParentId: string): Promise<TargetRef>;
delete(ref: TargetRef): Promise<void>;
/** Bring back a deleted item, keeping its history where possible. */
restore(
ref: TargetRef,
parentId: string,
name: string,
content?: FileContent
): Promise<TargetRef>;
}
export interface IgnorePolicy {
isIgnored(name: string, isFolder: boolean): boolean;
}
export interface ProjectRepository {
get(id: string): Promise<Project | null>;
list(): Promise<Project[]>;
create(project: NewProject): Promise<Project>;
pause(id: string, reason: string): Promise<void>;
resume(id: string): Promise<void>;
markFirstSynced(id: string, at: Date): Promise<void>;
setLastRunAt(id: string, at: Date): Promise<void>;
/** Forget all sync state so the project starts over from a full listing. */
reset(id: string): Promise<void>;
}
export interface ItemRepository {
listByProject(projectId: string): Promise<ItemState[]>;
/** Atomically store ingested changes and the cursor they were read up to. */
commitIngest(projectId: string, items: ItemState[], cursor: string): Promise<void>;
save(projectId: string, items: ItemState[]): Promise<void>;
}
export interface RunRepository {
start(projectId: string, trigger: RunTrigger, at: Date): Promise<string>;
finish(
runId: string,
result: { finishedAt: Date; outcome: RunOutcome; error: string | null; counts: RunCounts }
): Promise<void>;
addEvent(event: RunEvent): Promise<void>;
listSince(projectId: string, since: Date): Promise<RunRecord[]>;
listEvents(runId: string): Promise<RunEvent[]>;
/** Delete runs (and their events) started before `cutoff`; returns how many runs were removed. */
pruneBefore(cutoff: Date): Promise<number>;
}
export type Clock = () => Date;
export interface Logger {
info(message: string, meta?: Record<string, unknown>): void;
warn(message: string, meta?: Record<string, unknown>): void;
error(message: string, meta?: Record<string, unknown>): void;
}

View File

@@ -0,0 +1,237 @@
import { errorMessage, ServiceUnavailableError } from './errors';
import {
emptyCounts,
type EventAction,
type ItemState,
type Project,
type RunCounts,
type SyncAction,
type SyncedState,
type TargetRef
} from './model';
import { planAction, withStatus, workQueue } from './plan';
import type { ChangeSource, Clock, DocumentTarget, ItemRepository, RunRepository } from './ports';
import { ItemTree } from './tree';
export const WAITING_FOR_PARENT = 'Waiting for its parent folder to sync.';
export interface ReconcilerDeps {
source: ChangeSource;
target: DocumentTarget;
items: ItemRepository;
runs: RunRepository;
clock: Clock;
maxAttempts: number;
}
export interface ReconcileResult {
counts: RunCounts;
/** Set when a service outage stopped the run early. */
abortedWith: string | null;
}
type RecordEvent = (action: EventAction, message?: string) => Promise<void>;
/** Pushes every pending item's desired state to ACC, one item at a time. */
export class Reconciler {
constructor(private readonly deps: ReconcilerDeps) {}
async reconcile(
project: Project,
runId: string,
items: Map<string, ItemState>
): Promise<ReconcileResult> {
const { maxAttempts } = this.deps;
const tree = new ItemTree(project.source.rootItemId, items);
const counts = emptyCounts();
for (const { sourceId } of workQueue(items.values(), tree, maxAttempts)) {
// Re-read: an earlier folder delete may already have covered this item.
const item = items.get(sourceId)!;
const action = planAction(item);
if (!action) continue;
const record: RecordEvent = async (eventAction, message) => {
counts[eventAction]++;
await this.deps.runs.addEvent({
runId,
projectId: project.id,
sourceId,
path: tree.path(sourceId),
action: eventAction,
message: message ?? null,
at: this.deps.clock()
});
};
try {
await this.apply(project, items, tree, item, action, record);
} catch (error) {
if (error instanceof ServiceUnavailableError) {
return { counts, abortedWith: error.message };
}
throw error;
}
}
return { counts, abortedWith: null };
}
private async apply(
project: Project,
items: Map<string, ItemState>,
tree: ItemTree,
item: ItemState,
action: SyncAction,
record: RecordEvent
): Promise<void> {
const parentTargetId = this.targetParentId(project, items, item);
const needsParent = action.kind !== 'delete' && (action.kind !== 'update' || action.move);
if (needsParent && !parentTargetId) {
await this.store(project, items, [{ ...item, lastError: WAITING_FOR_PARENT }]);
return;
}
const synced: SyncedState = { ...item.synced };
const now = this.deps.clock();
try {
await this.execute(project, item, action, parentTargetId!, synced, record);
const done: ItemState = {
...item,
synced: { ...synced, at: now },
attemptCount: 0,
lastError: null,
lastAttemptAt: now
};
await this.store(project, items, [done]);
if (action.kind === 'delete' && item.desired.isFolder) {
await this.coverDeletedContents(project, items, tree, item.sourceId);
}
} catch (error) {
if (error instanceof ServiceUnavailableError) {
// Keep partial progress but don't charge an attempt for an outage.
await this.store(project, items, [{ ...item, synced }]);
throw error;
}
const message = errorMessage(error);
await this.store(project, items, [
{
...item,
synced,
attemptCount: item.attemptCount + 1,
lastError: message,
lastAttemptAt: now
}
]);
await record('failed', message);
}
}
/** Runs the target calls for one action, recording progress into `synced` as each succeeds. */
private async execute(
project: Project,
item: ItemState,
action: SyncAction,
parentTargetId: string,
synced: SyncedState,
record: RecordEvent
): Promise<void> {
const { target } = this.deps;
const { desired } = item;
const adopt = (ref: TargetRef) =>
Object.assign(synced, {
targetId: ref.id,
parentSourceId: desired.parentSourceId,
name: desired.name,
contentTag: desired.contentTag,
deleted: false
});
switch (action.kind) {
case 'create': {
const ref = desired.isFolder
? await target.createFolder(parentTargetId, desired.name)
: await target.uploadFile(
parentTargetId,
desired.name,
await this.download(project, item)
);
adopt(ref);
return record('created');
}
case 'restore': {
const stale = !desired.isFolder && desired.contentTag !== synced.contentTag;
const content = stale ? await this.download(project, item) : undefined;
adopt(await target.restore(refOf(item), parentTargetId, desired.name, content));
return record('restored');
}
case 'delete': {
await target.delete(refOf(item));
synced.deleted = true;
return record('deleted');
}
case 'update': {
let ref = refOf(item);
if (action.move) {
ref = await target.move(ref, parentTargetId);
Object.assign(synced, { targetId: ref.id, parentSourceId: desired.parentSourceId });
await record('moved');
}
if (action.rename) {
await target.rename(ref, desired.name);
synced.name = desired.name;
await record('renamed');
}
if (action.content) {
await target.uploadVersion(ref, desired.name, await this.download(project, item));
synced.contentTag = desired.contentTag;
await record('updated');
}
}
}
}
/** Deleting a folder in ACC takes its contents with it; reflect that in their synced state. */
private async coverDeletedContents(
project: Project,
items: Map<string, ItemState>,
tree: ItemTree,
folderId: string
): Promise<void> {
const covered = tree
.syncedDescendants(folderId)
.filter((d) => d.synced.targetId && !d.synced.deleted)
.map((d) => ({ ...d, synced: { ...d.synced, deleted: true } }));
await this.store(project, items, covered);
}
private targetParentId(
project: Project,
items: ReadonlyMap<string, ItemState>,
item: ItemState
): string | null {
const parentId = item.desired.parentSourceId;
if (parentId === project.source.rootItemId) return project.target.rootFolderId;
const parent = items.get(parentId);
return parent?.synced.targetId && !parent.synced.deleted ? parent.synced.targetId : null;
}
private download(project: Project, item: ItemState) {
return this.deps.source.download(project.source, item.sourceId);
}
/** Persists items with fresh statuses and mirrors them into the in-memory map. */
private async store(
project: Project,
items: Map<string, ItemState>,
updated: ItemState[]
): Promise<void> {
if (updated.length === 0) return;
const withStatuses = updated.map((item) => withStatus(item, this.deps.maxAttempts));
await this.deps.items.save(project.id, withStatuses);
for (const item of withStatuses) items.set(item.sourceId, item);
}
}
function refOf(item: ItemState): TargetRef {
return { id: item.synced.targetId!, isFolder: item.desired.isFolder };
}

View File

@@ -0,0 +1,73 @@
import type { ItemState } from './model';
type ParentOf = (item: ItemState) => string | null;
const desiredParent: ParentOf = (item) => item.desired.parentSourceId;
const syncedParent: ParentOf = (item) => item.synced.parentSourceId;
/**
* Read-only view over a project's items. Reads the map live, so it reflects
* entries replaced after construction.
*/
export class ItemTree {
constructor(
private readonly rootId: string,
private readonly items: ReadonlyMap<string, ItemState>
) {}
/** Desired ancestors, nearest first, stopping before the root. Cycle-safe. */
ancestors(sourceId: string): ItemState[] {
const result: ItemState[] = [];
const seen = new Set([sourceId]);
let parentId = this.items.get(sourceId)?.desired.parentSourceId;
while (parentId && parentId !== this.rootId && !seen.has(parentId)) {
const parent = this.items.get(parentId);
if (!parent) break;
result.push(parent);
seen.add(parentId);
parentId = parent.desired.parentSourceId;
}
return result;
}
depth(sourceId: string): number {
return this.ancestors(sourceId).length;
}
/** Display path relative to the project root, e.g. `Drawings/Level 1/plan.pdf`. */
path(sourceId: string): string {
const item = this.items.get(sourceId);
if (!item) return sourceId;
return [...this.ancestors(sourceId).reverse(), item].map((i) => i.desired.name).join('/');
}
/** Items under `folderId` according to SharePoint. */
desiredDescendants(folderId: string): ItemState[] {
return this.descendants(folderId, desiredParent);
}
/** Items under `folderId` as they currently sit in ACC. */
syncedDescendants(folderId: string): ItemState[] {
return this.descendants(folderId, syncedParent);
}
private descendants(folderId: string, parentOf: ParentOf): ItemState[] {
const children = new Map<string, ItemState[]>();
for (const item of this.items.values()) {
const parentId = parentOf(item);
if (parentId) children.set(parentId, [...(children.get(parentId) ?? []), item]);
}
const result: ItemState[] = [];
const seen = new Set([folderId]);
const queue = [folderId];
while (queue.length > 0) {
for (const child of children.get(queue.shift()!) ?? []) {
if (seen.has(child.sourceId)) continue;
seen.add(child.sourceId);
result.push(child);
queue.push(child.sourceId);
}
}
return result;
}
}

View File

@@ -0,0 +1,38 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { IntervalScheduler } from './scheduler';
import { silentLogger } from './testing/harness';
describe('IntervalScheduler', () => {
beforeEach(() => vi.useFakeTimers());
afterEach(() => vi.useRealTimers());
it('waits for a slow task to finish before scheduling the next run', async () => {
const scheduler = new IntervalScheduler(silentLogger);
let runs = 0;
scheduler.every('slow', 1000, async () => {
runs++;
await new Promise((resolve) => setTimeout(resolve, 5000));
});
await vi.advanceTimersByTimeAsync(1000);
expect(runs).toBe(1);
await vi.advanceTimersByTimeAsync(4000);
expect(runs).toBe(1);
await vi.advanceTimersByTimeAsync(2000);
expect(runs).toBe(2);
scheduler.stop();
});
it('keeps going after a task throws, and stops cleanly', async () => {
const scheduler = new IntervalScheduler(silentLogger);
const task = vi.fn().mockRejectedValueOnce(new Error('boom')).mockResolvedValue(undefined);
scheduler.every('flaky', 1000, task);
await vi.advanceTimersByTimeAsync(2000);
expect(task).toHaveBeenCalledTimes(2);
scheduler.stop();
await vi.advanceTimersByTimeAsync(5000);
expect(task).toHaveBeenCalledTimes(2);
});
});

View File

@@ -0,0 +1,39 @@
import { errorMessage } from './core/errors';
import type { Logger } from './core/ports';
/**
* Runs async tasks on fixed intervals. The next run of a task is scheduled only
* after the previous one settles, so a slow task never overlaps itself.
*/
export class IntervalScheduler {
private readonly timers = new Set<ReturnType<typeof setTimeout>>();
private stopped = false;
constructor(private readonly logger: Logger) {}
every(name: string, intervalMs: number, task: () => Promise<unknown>): void {
const tick = async () => {
try {
await task();
} catch (error) {
this.logger.error('Scheduled task failed', { task: name, error: errorMessage(error) });
}
schedule();
};
const schedule = () => {
if (this.stopped) return;
const timer = setTimeout(() => {
this.timers.delete(timer);
void tick();
}, intervalMs);
this.timers.add(timer);
};
schedule();
}
stop(): void {
this.stopped = true;
for (const timer of this.timers) clearTimeout(timer);
this.timers.clear();
}
}

View File

@@ -0,0 +1,145 @@
import type { FileContent, TargetRef } from '../core/model';
import type { DocumentTarget } from '../core/ports';
interface AccNode {
id: string;
parentId: string | null;
name: string;
isFolder: boolean;
deleted: boolean;
versions: string[];
}
/**
* In-memory stand-in for ACC Docs. Deleting a folder hides its contents (as in
* ACC); restoring a folder does not bring its contents back.
*/
export class FakeAcc implements DocumentTarget {
readonly rootFolderId = 'acc-root';
readonly calls: string[] = [];
private readonly nodes = new Map<string, AccNode>();
private readonly failures = new Map<string, { error: Error; remaining: number }>();
private nextId = 1;
constructor() {
this.nodes.set(this.rootFolderId, {
id: this.rootFolderId,
parentId: null,
name: 'Project Files',
isFolder: true,
deleted: false,
versions: []
});
}
/** Make the next `times` operations on an item with this name throw `error`. */
failOn(name: string, error: Error, times = Infinity): void {
this.failures.set(name, { error, remaining: times });
}
/** Adds a file directly, as if someone uploaded it by hand in ACC. */
addByHand(parentId: string, name: string): string {
return this.insert(parentId, name, false, ['by hand']);
}
/** Live paths, e.g. `Drawings/`, `Drawings/plan.pdf@2` (file@version count). */
tree(): string[] {
return [...this.nodes.values()]
.filter((node) => node.id !== this.rootFolderId && this.isAlive(node))
.map((node) => this.path(node) + (node.isFolder ? '/' : `@${node.versions.length}`))
.sort();
}
latestContent(path: string): string | undefined {
const node = [...this.nodes.values()].find((n) => this.isAlive(n) && this.path(n) === path);
return node?.versions.at(-1);
}
async createFolder(parentId: string, name: string): Promise<TargetRef> {
this.enter('createFolder', name);
return { id: this.insert(parentId, name, true, []), isFolder: true };
}
async uploadFile(parentId: string, name: string, content: FileContent): Promise<TargetRef> {
this.enter('uploadFile', name);
return { id: this.insert(parentId, name, false, [await read(content)]), isFolder: false };
}
async uploadVersion(ref: TargetRef, name: string, content: FileContent): Promise<void> {
const node = this.node(ref, 'uploadVersion');
node.versions.push(await read(content));
node.name = name;
}
async rename(ref: TargetRef, name: string): Promise<void> {
this.node(ref, 'rename').name = name;
}
async move(ref: TargetRef, newParentId: string): Promise<TargetRef> {
this.node(ref, 'move').parentId = newParentId;
return ref;
}
async delete(ref: TargetRef): Promise<void> {
this.node(ref, 'delete').deleted = true;
}
async restore(
ref: TargetRef,
parentId: string,
name: string,
content?: FileContent
): Promise<TargetRef> {
const node = this.node(ref, 'restore');
Object.assign(node, { deleted: false, parentId, name });
if (content) node.versions.push(await read(content));
return ref;
}
private insert(parentId: string, name: string, isFolder: boolean, versions: string[]): string {
if (!this.nodes.has(parentId)) throw new Error(`No ACC folder ${parentId}`);
const id = `acc-${this.nextId++}`;
this.nodes.set(id, { id, parentId, name, isFolder, deleted: false, versions });
return id;
}
private node(ref: TargetRef, operation: string): AccNode {
const node = this.nodes.get(ref.id);
if (!node) throw new Error(`No ACC item ${ref.id}`);
this.enter(operation, node.name);
return node;
}
private enter(operation: string, name: string): void {
this.calls.push(`${operation} ${name}`);
const failure = this.failures.get(name);
if (failure && failure.remaining > 0) {
failure.remaining--;
throw failure.error;
}
}
private isAlive(node: AccNode): boolean {
for (
let n: AccNode | undefined = node;
n;
n = n.parentId ? this.nodes.get(n.parentId) : undefined
) {
if (n.deleted) return false;
}
return true;
}
private path(node: AccNode): string {
const names: string[] = [];
for (let n: AccNode | undefined = node; n && n.id !== this.rootFolderId;) {
names.unshift(n.name);
n = n.parentId ? this.nodes.get(n.parentId) : undefined;
}
return names.join('/');
}
}
async function read(content: FileContent): Promise<string> {
return new Response(content.stream).text();
}

View File

@@ -0,0 +1,127 @@
import type { FileContent, SourceChange, SourceItem, SourceRoot } from '../core/model';
import type { ChangeBatch, ChangeSource } from '../core/ports';
interface DriveNode {
id: string;
parentId: string | null;
name: string;
isFolder: boolean;
content: string;
version: number;
deleted: boolean;
}
/**
* In-memory stand-in for a SharePoint document library that behaves like Graph
* delta: a null cursor lists everything; otherwise each item changed since the
* cursor is reported once, in its latest state.
*/
export class FakeDrive implements ChangeSource {
readonly driveRootId = 'drive-root';
private readonly nodes = new Map<string, DriveNode>();
private readonly log: string[] = [];
private nextId = 1;
constructor() {
this.nodes.set(this.driveRootId, {
id: this.driveRootId,
parentId: null,
name: 'root',
isFolder: true,
content: '',
version: 0,
deleted: false
});
}
addFolder(parentId: string, name: string): string {
return this.add(parentId, name, true, '');
}
addFile(parentId: string, name: string, content = `${name} v1`): string {
return this.add(parentId, name, false, content);
}
edit(id: string, content: string): void {
this.change(id, (node) => {
node.content = content;
node.version++;
});
}
rename(id: string, name: string): void {
this.change(id, (node) => (node.name = name));
}
move(id: string, parentId: string): void {
this.change(id, (node) => (node.parentId = parentId));
}
/** Reports only the removed item, not its contents: the harder case for the engine. */
remove(id: string): void {
this.change(id, (node) => (node.deleted = true));
}
/** Reports only the restored item, not its contents. */
restore(id: string): void {
this.change(id, (node) => (node.deleted = false));
}
async getChanges(_root: SourceRoot, cursor: string | null): Promise<ChangeBatch> {
const end = String(this.log.length);
if (cursor === null) {
const changes = [...this.nodes.values()]
.filter((node) => this.isAlive(node))
.map((node) => this.toChange(node));
return { changes, cursor: end, fullSync: true };
}
const ids = [...new Set(this.log.slice(Number(cursor)).reverse())].reverse();
return {
changes: ids.map((id) => this.toChange(this.nodes.get(id)!)),
cursor: end,
fullSync: false
};
}
async download(_root: SourceRoot, sourceId: string): Promise<FileContent> {
const bytes = new TextEncoder().encode(this.nodes.get(sourceId)!.content);
return { size: bytes.length, stream: new Response(bytes).body! };
}
private add(parentId: string, name: string, isFolder: boolean, content: string): string {
const id = `sp-${this.nextId++}`;
this.nodes.set(id, { id, parentId, name, isFolder, content, version: 1, deleted: false });
this.log.push(id);
return id;
}
private change(id: string, mutate: (node: DriveNode) => void): void {
mutate(this.nodes.get(id)!);
this.log.push(id);
}
private isAlive(node: DriveNode): boolean {
for (
let n: DriveNode | undefined = node;
n;
n = n.parentId ? this.nodes.get(n.parentId) : undefined
) {
if (n.deleted) return false;
}
return true;
}
private toChange(node: DriveNode): SourceChange {
if (node.deleted) return { kind: 'delete', sourceId: node.id };
const item: SourceItem = {
sourceId: node.id,
parentSourceId: node.parentId,
name: node.name,
isFolder: node.isFolder,
contentTag: node.isFolder ? null : `c${node.version}`,
modifiedAt: new Date(Date.UTC(2026, 0, 1, 0, 0, node.version)),
size: node.isFolder ? null : node.content.length
};
return { kind: 'upsert', item };
}
}

View File

@@ -0,0 +1,70 @@
import { openDatabase } from '../../adapters/sqlite/database';
import { SqliteItemRepository } from '../../adapters/sqlite/item-repository';
import { SqliteProjectRepository } from '../../adapters/sqlite/project-repository';
import { SqliteRunRepository } from '../../adapters/sqlite/run-repository';
import { SyncEngine } from '../core/engine';
import { PatternIgnorePolicy } from '../core/ignore';
import type { Logger } from '../core/ports';
import { Reconciler } from '../core/reconciler';
import { FakeAcc } from './fake-acc';
import { FakeDrive } from './fake-drive';
export const silentLogger: Logger = { info() {}, warn() {}, error() {} };
/** A real engine over in-memory SQLite, a fake SharePoint drive and a fake ACC. */
export async function createHarness({ maxAttempts = 3 } = {}) {
let now = Date.UTC(2026, 9, 1);
const clock = () => new Date((now += 1000));
const db = openDatabase(':memory:');
const projects = new SqliteProjectRepository(db, clock);
const items = new SqliteItemRepository(db);
const runs = new SqliteRunRepository(db);
const drive = new FakeDrive();
const acc = new FakeAcc();
const rootId = drive.addFolder(drive.driveRootId, 'Project X');
const reconciler = new Reconciler({
source: drive,
target: acc,
items,
runs,
clock,
maxAttempts
});
const engine = new SyncEngine({
source: drive,
projects,
items,
runs,
reconciler,
ignore: new PatternIgnorePolicy(),
clock,
logger: silentLogger,
maxAttempts
});
const project = await projects.create({
name: 'Project X',
source: { siteId: 'site', driveId: 'drive', rootItemId: rootId },
sourceDisplayPath: '/sites/ProjectX/Documents/Project X',
target: { hubId: 'hub', projectId: 'acc-project', rootFolderId: acc.rootFolderId },
targetDisplayPath: 'Project X/Project Files'
});
return {
db,
drive,
acc,
rootId,
projects,
items,
runs,
engine,
project,
sync: () => engine.runProject(project.id, 'schedule'),
itemNamed: async (name: string) =>
(await items.listByProject(project.id)).find((i) => i.desired.name === name)!
};
}

12
src/routes/+layout.svelte Normal file
View File

@@ -0,0 +1,12 @@
<script lang="ts">
import favicon from '#lib/assets/favicon.svg';
import type { LayoutProps } from './$types';
let { children }: LayoutProps = $props();
</script>
<svelte:head>
<link rel="icon" href={favicon} />
</svelte:head>
{@render children()}

2
src/routes/+page.svelte Normal file
View File

@@ -0,0 +1,2 @@
<h1>Welcome to SvelteKit</h1>
<p>Visit <a href="https://svelte.dev/docs/kit">svelte.dev/docs/kit</a> to read the documentation</p>

3
static/robots.txt Normal file
View File

@@ -0,0 +1,3 @@
# allow crawling everything by default
User-agent: *
Disallow:

8
tsconfig.json Normal file
View File

@@ -0,0 +1,8 @@
{
"extends": "$app/tsconfig",
"compilerOptions": {
"strict": true,
"types": ["$app/types", "node"]
},
"include": ["src", "vite.config.ts", "drizzle.config.ts"]
}

30
vite.config.ts Normal file
View File

@@ -0,0 +1,30 @@
import { defineConfig } from 'vitest/config';
import adapter from '@sveltejs/adapter-node';
import { sveltekit } from '@sveltejs/kit/vite';
export default defineConfig({
plugins: [
sveltekit({
compilerOptions: {
// Force runes mode for the project, except for libraries. Can be removed in svelte 6.
runes: ({ filename }) =>
filename.split(/[/\\]/).includes('node_modules') ? undefined : true
},
adapter: adapter()
})
],
test: {
expect: { requireAssertions: true },
projects: [
{
extends: './vite.config.ts',
test: {
name: 'server',
environment: 'node',
include: ['src/**/*.{test,spec}.{js,ts}'],
exclude: ['src/**/*.svelte.{test,spec}.{js,ts}']
}
}
]
}
});