allowing admin to choose which folders get synced

This commit is contained in:
C-West8
2026-10-23 12:55:14 -05:00
parent fbd81b0b45
commit 3ca266acc3
30 changed files with 1109 additions and 120 deletions

View File

@@ -9,6 +9,7 @@ There is also a small web page that shows each project's file tree, sync status,
- **One-way sync.** SharePoint is the source of truth. Nothing is ever written back to SharePoint.
- **Every file operation carries over.** Creates, edits (as new ACC versions), renames, moves and deletes are all mirrored. Removing something from the project's SharePoint root counts as deleting it.
- **Files added to ACC by hand are left alone.** The sync only ever changes ACC items it created.
- **Admins can leave folders out.** Every folder syncs by default. Unticking one on the project's admin page deletes the copies the sync made of it in ACC; ticking it again brings them back.
- **Temp and lock files are ignored.** That covers `~$*`, `*.tmp`, `Thumbs.db` and similar.
- **Failures are retried 3 times**, then reported as an issue on the status page.

View File

@@ -10,6 +10,7 @@
| Scope | Many projects per server; each project is a 1:1 pair of SharePoint root and ACC root |
| ACC ownership | Only touch ACC items this tool created; hand-added ACC content is left alone |
| Ignored files | Temp/lock/system files are skipped silently (not issues) |
| Excluded | Admins can untick folders per project (all sync by default); excluding deletes the tool's ACC copies, re-including restores |
| Retries | 3 attempts across runs, then surface as an issue |
| Files | Small/medium documents; no large Revit models |
| Hosting | One Docker container (Portainer), SQLite on a volume |
@@ -70,6 +71,8 @@ An item whose new parent is outside the root counts as deleted. That covers "rem
**Restored folders:** a folder delete cascades "deleted" to everything stored under it. If a deleted folder later reappears, its contents may not be reported one by one, so ingest asks for one follow-up **full listing** in the same run, which brings them back.
**Excluded folders:** an admin can untick folders on the project's admin page; their IDs are stored on the project. Ingest still tracks everything under them, so `desired` stays accurate. Between ingest and reconcile, each run sets `excluded` on every item that is, or sits under, an excluded folder (by SharePoint's current layout, so moves in and out count). Reconcile treats excluded like deleted: the folder's ACC copy is deleted with one call, which covers its contents. Unticking it again clears the flag, and the tombstoned ACC items are restored. Saving the choice updates the flags straight away, so the status page shows the change before the next run.
## Interfaces (ports)
The source of truth is [`src/lib/server/sync/core/ports.ts`](../src/lib/server/sync/core/ports.ts):
@@ -108,6 +111,7 @@ projects
id, name, enabled, paused_reason
sp_site_id, sp_drive_id, sp_root_item_id, sp_display_path
acc_hub_id, acc_project_id, acc_root_folder_id, acc_display_path
excluded_folder_ids -- JSON array of SharePoint folder IDs
delta_cursor, first_synced_at, last_run_at, created_at
items PK (project_id, source_id)
@@ -116,7 +120,8 @@ items PK (project_id, source_id)
-- synced (what ACC has)
target_id, synced_parent_source_id, synced_name, synced_content_tag, synced_deleted, synced_at
-- bookkeeping
status ('synced' | 'pending' | 'failed' | 'deleted')
excluded -- in or under a folder an admin excluded
status ('synced' | 'pending' | 'failed' | 'deleted' | 'excluded')
attempt_count, last_error, last_attempt_at
runs

View File

@@ -0,0 +1,2 @@
ALTER TABLE `items` ADD `excluded` integer DEFAULT false NOT NULL;--> statement-breakpoint
ALTER TABLE `projects` ADD `excluded_folder_ids` text DEFAULT '[]' NOT NULL;

View File

@@ -0,0 +1,558 @@
{
"version": "6",
"dialect": "sqlite",
"id": "c2d87a43-5e1d-4ccd-a21f-3ef03a52589a",
"prevId": "e1cc8fc1-3ac7-41c4-915b-3d703eccd229",
"tables": {
"items": {
"name": "items",
"columns": {
"project_id": {
"name": "project_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"source_id": {
"name": "source_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"parent_source_id": {
"name": "parent_source_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"is_folder": {
"name": "is_folder",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"content_tag": {
"name": "content_tag",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"source_modified_at": {
"name": "source_modified_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"size": {
"name": "size",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"deleted": {
"name": "deleted",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"target_id": {
"name": "target_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"synced_parent_source_id": {
"name": "synced_parent_source_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"synced_name": {
"name": "synced_name",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"synced_content_tag": {
"name": "synced_content_tag",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"synced_deleted": {
"name": "synced_deleted",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"synced_at": {
"name": "synced_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"excluded": {
"name": "excluded",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"attempt_count": {
"name": "attempt_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"last_error": {
"name": "last_error",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"last_attempt_at": {
"name": "last_attempt_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"items_project_status": {
"name": "items_project_status",
"columns": [
"project_id",
"status"
],
"isUnique": false
}
},
"foreignKeys": {
"items_project_id_projects_id_fk": {
"name": "items_project_id_projects_id_fk",
"tableFrom": "items",
"tableTo": "projects",
"columnsFrom": [
"project_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {
"items_project_id_source_id_pk": {
"columns": [
"project_id",
"source_id"
],
"name": "items_project_id_source_id_pk"
}
},
"uniqueConstraints": {},
"checkConstraints": {}
},
"projects": {
"name": "projects",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"enabled": {
"name": "enabled",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": true
},
"paused_reason": {
"name": "paused_reason",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"sp_site_id": {
"name": "sp_site_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"sp_drive_id": {
"name": "sp_drive_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"sp_root_item_id": {
"name": "sp_root_item_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"sp_display_path": {
"name": "sp_display_path",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"acc_hub_id": {
"name": "acc_hub_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"acc_project_id": {
"name": "acc_project_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"acc_root_folder_id": {
"name": "acc_root_folder_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"acc_display_path": {
"name": "acc_display_path",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"excluded_folder_ids": {
"name": "excluded_folder_ids",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'[]'"
},
"delta_cursor": {
"name": "delta_cursor",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"first_synced_at": {
"name": "first_synced_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"last_run_at": {
"name": "last_run_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"run_events": {
"name": "run_events",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"run_id": {
"name": "run_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"project_id": {
"name": "project_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"source_id": {
"name": "source_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"path": {
"name": "path",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"action": {
"name": "action",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"message": {
"name": "message",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"at": {
"name": "at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"run_events_run": {
"name": "run_events_run",
"columns": [
"run_id"
],
"isUnique": false
}
},
"foreignKeys": {
"run_events_run_id_runs_id_fk": {
"name": "run_events_run_id_runs_id_fk",
"tableFrom": "run_events",
"tableTo": "runs",
"columnsFrom": [
"run_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"runs": {
"name": "runs",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"project_id": {
"name": "project_id",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"trigger": {
"name": "trigger",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"started_at": {
"name": "started_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"finished_at": {
"name": "finished_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"outcome": {
"name": "outcome",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"error": {
"name": "error",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created": {
"name": "created",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"updated": {
"name": "updated",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"renamed": {
"name": "renamed",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"moved": {
"name": "moved",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"deleted": {
"name": "deleted",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"restored": {
"name": "restored",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"failed": {
"name": "failed",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
}
},
"indexes": {
"runs_project_started": {
"name": "runs_project_started",
"columns": [
"project_id",
"started_at"
],
"isUnique": false
}
},
"foreignKeys": {
"runs_project_id_projects_id_fk": {
"name": "runs_project_id_projects_id_fk",
"tableFrom": "runs",
"tableTo": "projects",
"columnsFrom": [
"project_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
}
},
"views": {},
"enums": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
},
"internal": {
"indexes": {}
}
}

View File

@@ -8,6 +8,13 @@
"when": 1792772657596,
"tag": "0000_init",
"breakpoints": true
},
{
"idx": 1,
"version": "6",
"when": 1792775179343,
"tag": "0001_excluded_folders",
"breakpoints": true
}
]
}

View File

@@ -2,6 +2,7 @@
import { SvelteSet } from 'svelte/reactivity';
import Badge from '#lib/components/Badge.svelte';
import TimeAgo from '#lib/components/TimeAgo.svelte';
import TreeName from '#lib/components/TreeName.svelte';
import { ITEM_STATUS } from '#lib/format.ts';
import type { TreeNode } from '#lib/server/status/tree-view.ts';
@@ -15,8 +16,9 @@
depth: number;
}
const settled = new Set<TreeNode['status']>(['synced', 'excluded']);
const needsAttention = (node: TreeNode): boolean =>
node.status !== 'synced' || node.children.some(needsAttention);
!settled.has(node.status) || node.children.some(needsAttention);
const rows = $derived.by(() => {
const result: Row[] = [];
@@ -60,7 +62,7 @@
</p>
{:else}
<div class="table-wrap">
<table class="register">
<table class="register tree">
<thead>
<tr>
<th>Name</th>
@@ -72,35 +74,19 @@
<tbody>
{#each rows as { node, depth } (node.sourceId)}
{@const status = statusOf(node)}
<tr class:folder={node.isFolder}>
<td class="name" style:--depth={depth}>
<div class="name-line">
{#if node.isFolder && !attentionOnly}
<button
type="button"
class="twisty"
aria-expanded={!collapsed.has(node.sourceId)}
aria-label="{collapsed.has(node.sourceId) ? 'Expand' : 'Collapse'} {node.name}"
onclick={() => toggle(node.sourceId)}
<tr>
<TreeName
name={node.name}
isFolder={node.isFolder}
{depth}
expanded={node.isFolder && !attentionOnly ? !collapsed.has(node.sourceId) : undefined}
ontoggle={() => toggle(node.sourceId)}
title={node.error ?? undefined}
>
<svg viewBox="0 0 16 16" aria-hidden="true"><path d="M6 4l4 4-4 4" /></svg>
</button>
{:else}
<span class="twisty-spacer"></span>
{/if}
<svg class="icon" viewBox="0 0 16 16" aria-hidden="true">
{#if node.isFolder}
<path d="M1.5 3.5h5l1.5 1.5h6.5v8h-13z" />
{:else}
<path d="M3.5 1.5h6l3 3v10h-9z M9.5 1.5v3h3" />
{/if}
</svg>
<span title={node.error ?? undefined}>{node.name}</span>
{#if node.problems > 0}
<span class="problems">{node.problems} failed inside</span>
{/if}
</div>
</td>
</TreeName>
<td><Badge {...status} /></td>
<td class="small"><TimeAgo date={node.isFolder ? null : node.modifiedAt} empty="" /></td
>
@@ -124,69 +110,6 @@
text-align: center;
}
/* Every row is one line, so cells centre on each other rather than sharing a baseline
(the arrow and icons have none). */
td {
vertical-align: middle;
}
/* The cell stays a table cell so its height and border match the row; the flex layout
lives on the line inside it. */
.name {
padding-left: calc(0.5rem + var(--depth) * 1.25rem);
min-width: 18rem;
}
.name-line {
display: flex;
align-items: center;
gap: 0.35rem;
}
.folder .name {
font-weight: 600;
}
.twisty,
.twisty-spacer {
flex: none;
width: 1.25rem;
height: 1.25rem;
}
.twisty {
display: grid;
place-items: center;
padding: 0;
border: 0;
border-radius: var(--radius);
background: none;
color: var(--pencil);
cursor: pointer;
}
.twisty svg {
width: 0.85rem;
height: 0.85rem;
fill: none;
stroke: currentColor;
stroke-width: 1.75;
transition: transform 120ms ease;
}
.twisty[aria-expanded='true'] svg {
transform: rotate(90deg);
}
.icon {
flex: none;
width: 1rem;
height: 1rem;
fill: none;
stroke: var(--pencil);
stroke-width: 1.1;
}
.problems {
margin-left: 0.25rem;
font-size: var(--step--1);

View File

@@ -0,0 +1,132 @@
<script lang="ts">
import { enhance } from '$app/forms';
import { SvelteSet } from 'svelte/reactivity';
import TreeName from '#lib/components/TreeName.svelte';
import type { TreeNode } from '#lib/server/status/tree-view.ts';
/** The admin form for choosing which of a project's folders sync. */
let { nodes, excluded }: { nodes: TreeNode[]; excluded: string[] } = $props();
/** Folders left unticked. Starts from what's saved, and starts over when that changes. */
const unticked = $derived(new SvelteSet(excluded));
const collapsed = new SvelteSet<string>();
interface Row {
folder: TreeNode;
depth: number;
hasSubfolders: boolean;
/** A folder above is unticked, so this one is left out whatever its own box says. */
covered: boolean;
/** A folder above is collapsed. The row stays in the form so its setting is still sent. */
hidden: boolean;
}
const rows = $derived.by(() => {
const result: Row[] = [];
const walk = (list: TreeNode[], depth: number, covered: boolean, hidden: boolean) => {
for (const folder of list) {
if (!folder.isFolder) continue;
const hasSubfolders = folder.children.some((child) => child.isFolder);
result.push({ folder, depth, hasSubfolders, covered, hidden });
walk(
folder.children,
depth + 1,
covered || unticked.has(folder.sourceId),
hidden || collapsed.has(folder.sourceId)
);
}
};
walk(nodes, 0, false, false);
return result;
});
function toggle(set: SvelteSet<string>, id: string) {
if (set.has(id)) set.delete(id);
else set.add(id);
}
</script>
{#if rows.length === 0}
<p class="empty muted">No folders yet. They appear here after the first sync.</p>
{:else}
<form
method="POST"
action="?/folders"
class="folders"
use:enhance={({ cancel }) => {
const newlyOff = [...unticked].filter((id) => !excluded.includes(id)).length;
const question = `Stop syncing ${newlyOff === 1 ? '1 folder' : `${newlyOff} folders`}? The copies in ACC are deleted on the next sync.`;
if (newlyOff > 0 && !confirm(question)) cancel();
return ({ update }) => update({ reset: false });
}}
>
<div class="table-wrap">
<table class="register tree">
<thead>
<tr>
<th>Folder</th>
<th>Sync</th>
</tr>
</thead>
<tbody>
{#each rows as { folder, depth, hasSubfolders, covered, hidden } (folder.sourceId)}
{@const ticked = !unticked.has(folder.sourceId)}
<tr {hidden} class:covered>
<TreeName
name={folder.name}
isFolder
{depth}
expanded={hasSubfolders ? !collapsed.has(folder.sourceId) : undefined}
ontoggle={() => toggle(collapsed, folder.sourceId)}
/>
<td class="small">
<!-- Inert rows still submit their values, so a folder keeps its own setting
for when the one above is ticked again. -->
<input type="hidden" name="folder" value={folder.sourceId} />
<label class="check" inert={covered}>
<input
type="checkbox"
name="sync"
value={folder.sourceId}
checked={ticked}
aria-label="Sync {folder.name}"
onchange={() => toggle(unticked, folder.sourceId)}
/>
{#if covered}
Not synced (a folder above is unticked)
{:else}
{ticked ? 'Syncs' : 'Not synced'}
{/if}
</label>
</td>
</tr>
{/each}
</tbody>
</table>
</div>
<div class="actions">
<button class="button primary">Save folders</button>
</div>
</form>
{/if}
<style>
.folders {
display: grid;
gap: 0.75rem;
}
.empty {
padding: 1.5rem;
border: 1px dashed var(--rule);
text-align: center;
}
.covered {
color: var(--pencil);
}
.covered input {
opacity: 0.5;
}
</style>

View File

@@ -0,0 +1,111 @@
<script lang="ts">
import type { Snippet } from 'svelte';
/** The name cell of a tree table row: indent, expand arrow, icon and name. */
let {
name,
isFolder,
depth,
expanded,
ontoggle,
title,
children
}: {
name: string;
isFolder: boolean;
depth: number;
/** Whether the folder is open. Leave unset for rows that can't expand. */
expanded?: boolean;
ontoggle?: () => void;
/** Hover text for the name. */
title?: string;
/** Anything shown after the name. */
children?: Snippet;
} = $props();
</script>
<td class="name" class:folder={isFolder} style:--depth={depth}>
<!-- The cell stays a table cell so its height and border match the row; the flex
layout lives on the line inside it. -->
<div class="name-line">
{#if expanded === undefined}
<span class="twisty-spacer"></span>
{:else}
<button
type="button"
class="twisty"
aria-expanded={expanded}
aria-label="{expanded ? 'Collapse' : 'Expand'} {name}"
onclick={ontoggle}
>
<svg viewBox="0 0 16 16" aria-hidden="true"><path d="M6 4l4 4-4 4" /></svg>
</button>
{/if}
<svg class="icon" viewBox="0 0 16 16" aria-hidden="true">
{#if isFolder}
<path d="M1.5 3.5h5l1.5 1.5h6.5v8h-13z" />
{:else}
<path d="M3.5 1.5h6l3 3v10h-9z M9.5 1.5v3h3" />
{/if}
</svg>
<span {title}>{name}</span>
{@render children?.()}
</div>
</td>
<style>
.name {
padding-left: calc(0.5rem + var(--depth) * 1.25rem);
min-width: 18rem;
}
.folder {
font-weight: 600;
}
.name-line {
display: flex;
align-items: center;
gap: 0.35rem;
}
.twisty,
.twisty-spacer {
flex: none;
width: 1.25rem;
height: 1.25rem;
}
.twisty {
display: grid;
place-items: center;
padding: 0;
border: 0;
border-radius: var(--radius);
background: none;
color: var(--pencil);
cursor: pointer;
}
.twisty svg {
width: 0.85rem;
height: 0.85rem;
fill: none;
stroke: currentColor;
stroke-width: 1.75;
transition: transform 120ms ease;
}
.twisty[aria-expanded='true'] svg {
transform: rotate(90deg);
}
.icon {
flex: none;
width: 1rem;
height: 1rem;
fill: none;
stroke: var(--pencil);
stroke-width: 1.1;
}
</style>

View File

@@ -23,7 +23,8 @@ export const ITEM_STATUS: Record<ItemStatus, { label: string; tone: Tone }> = {
synced: { label: 'Synced', tone: 'ok' },
pending: { label: 'Pending', tone: 'info' },
failed: { label: 'Failed', tone: 'danger' },
deleted: { label: 'Deleted', tone: 'muted' }
deleted: { label: 'Deleted', tone: 'muted' },
excluded: { label: 'Not synced', tone: 'muted' }
};
export const OUTCOME: Record<RunOutcome, { label: string; tone: Tone }> = {

View File

@@ -23,6 +23,16 @@ describe('SimulatedBackend', () => {
expect(names).toContain(REJECTED_FILE);
});
it('puts sample folders that share a parent under one folder', async () => {
const backend = new SimulatedBackend();
const { root } = await backend.resolveSourceRoot('Demo');
const { changes } = await backend.drive.getChanges(root, null);
const names = changes.flatMap((c) => (c.kind === 'upsert' ? [c.item.name] : []));
expect(names.filter((name) => name === 'Models')).toHaveLength(1);
});
it('rejects a blank location', async () => {
await expect(new SimulatedBackend().resolveTargetRoot(' / ')).rejects.toThrow(
'Enter a folder path or URL.'

View File

@@ -87,11 +87,18 @@ export class SimulatedBackend implements SourceRootResolver, TargetRootResolver
}
private seed(rootId: string): void {
for (const [folderPath, files] of Object.entries(SAMPLE_TREE)) {
let parent = rootId;
for (const segment of folderPath.split('/').filter(Boolean)) {
parent = this.drive.addFolder(parent, segment);
const folders = new Map<string, string>([['', rootId]]);
const folderAt = (path: string): string => {
let id = folders.get(path);
if (!id) {
const cut = path.lastIndexOf('/');
id = this.drive.addFolder(folderAt(path.slice(0, Math.max(cut, 0))), path.slice(cut + 1));
folders.set(path, id);
}
return id;
};
for (const [folderPath, files] of Object.entries(SAMPLE_TREE)) {
const parent = folderAt(folderPath);
for (const file of files) this.drive.addFile(parent, file);
}
}

View File

@@ -78,6 +78,7 @@ function toRow(projectId: number, { sourceId, desired, synced, ...rest }: ItemSt
syncedContentTag: synced.contentTag,
syncedDeleted: synced.deleted,
syncedAt: synced.at,
excluded: rest.excluded,
status: rest.status,
attemptCount: rest.attemptCount,
lastError: rest.lastError,
@@ -105,6 +106,7 @@ function toItem(row: ItemRow): ItemState {
deleted: row.syncedDeleted,
at: row.syncedAt
},
excluded: row.excluded,
status: row.status,
attemptCount: row.attemptCount,
lastError: row.lastError,

View File

@@ -75,10 +75,12 @@ export class SqliteProjectRepository implements ProjectRepository {
/** Maps the editable project fields to columns, leaving out anything not provided. */
function toColumns(changes: ProjectChanges): Partial<ProjectRow> {
const { name, enabled, source, sourceDisplayPath, target, targetDisplayPath } = changes;
const { name, enabled, source, sourceDisplayPath, target, targetDisplayPath, excludedFolderIds } =
changes;
return {
name,
enabled,
excludedFolderIds,
spSiteId: source?.siteId,
spDriveId: source?.driveId,
spRootItemId: source?.rootItemId,
@@ -104,6 +106,7 @@ function toProject(row: ProjectRow): Project {
rootFolderId: row.accRootFolderId
},
targetDisplayPath: row.accDisplayPath,
excludedFolderIds: row.excludedFolderIds,
deltaCursor: row.deltaCursor,
firstSyncedAt: row.firstSyncedAt,
lastRunAt: row.lastRunAt,

View File

@@ -1,5 +1,6 @@
import { sql } from 'drizzle-orm';
import { index, integer, primaryKey, sqliteTable, text } from 'drizzle-orm/sqlite-core';
import { EVENT_ACTIONS } from '#lib/server/sync/core/model.ts';
import { EVENT_ACTIONS, ITEM_STATUSES } from '#lib/server/sync/core/model.ts';
const timestamp = (name: string) => integer(name, { mode: 'timestamp_ms' });
const flag = (name: string) => integer(name, { mode: 'boolean' });
@@ -22,6 +23,10 @@ export const projects = sqliteTable('projects', {
accProjectId: text('acc_project_id').notNull(),
accRootFolderId: text('acc_root_folder_id').notNull(),
accDisplayPath: text('acc_display_path').notNull(),
excludedFolderIds: text('excluded_folder_ids', { mode: 'json' })
.$type<string[]>()
.notNull()
.default(sql`'[]'`),
deltaCursor: text('delta_cursor'),
firstSyncedAt: timestamp('first_synced_at'),
lastRunAt: timestamp('last_run_at'),
@@ -51,7 +56,8 @@ export const items = sqliteTable(
syncedDeleted: flag('synced_deleted').notNull(),
syncedAt: timestamp('synced_at'),
// bookkeeping
status: text('status', { enum: ['pending', 'synced', 'failed', 'deleted'] }).notNull(),
excluded: flag('excluded').notNull().default(false),
status: text('status', { enum: ITEM_STATUSES }).notNull(),
attemptCount: count('attempt_count'),
lastError: text('last_error'),
lastAttemptAt: timestamp('last_attempt_at')

View File

@@ -25,3 +25,8 @@ export function field(data: FormData, name: string): string {
const value = data.get(name);
return typeof value === 'string' ? value : '';
}
/** Every value of a repeated form field (e.g. ticked checkboxes), skipping files. */
export function fields(data: FormData, name: string): string[] {
return data.getAll(name).filter((value) => typeof value === 'string');
}

View File

@@ -5,6 +5,7 @@ import type {
ProjectRepository,
RunRepository
} from '#lib/server/sync/core/ports.ts';
import { isWanted } from '#lib/server/sync/core/plan.ts';
import { ItemTree } from '#lib/server/sync/core/tree.ts';
import { buildTree, type TreeNode } from './tree-view.ts';
@@ -72,7 +73,7 @@ export class StatusService {
.map((item) => ({
sourceId: item.sourceId,
path: paths.path(item.sourceId),
message: `${item.desired.deleted ? 'Delete failed: ' : ''}${item.lastError ?? 'Unknown error'}`,
message: `${isWanted(item) ? '' : 'Delete failed: '}${item.lastError ?? 'Unknown error'}`,
attempts: item.attemptCount,
lastAttemptAt: item.lastAttemptAt
}))

View File

@@ -1,8 +1,8 @@
import { describe, expect, it } from 'vitest';
import { createHarness } from '#lib/server/sync/testing/harness.ts';
import { createHarness, type Harness } from '#lib/server/sync/testing/harness.ts';
import { ServiceUnavailableError } from './errors.ts';
import { ROOT_DELETED } from './ingest.ts';
import { WAITING_FOR_PARENT } from './reconciler.ts';
import { EXCLUDED_BY_ADMIN, WAITING_FOR_PARENT } from './reconciler.ts';
describe('SyncEngine', () => {
it('mirrors the initial tree and ignores temp files', async () => {
@@ -136,6 +136,56 @@ describe('SyncEngine', () => {
]);
});
describe('excluded folders', () => {
const exclude = (h: Harness, folder: string) =>
h.projectService.setFolderSync(h.project.id, { shown: [folder], synced: [] });
const include = (h: Harness, folder: string) =>
h.projectService.setFolderSync(h.project.id, { shown: [folder], synced: [folder] });
it('deletes an excluded folder from ACC and brings the same items back when included', async () => {
const h = await createHarness();
const folder = h.drive.addFolder(h.rootId, 'Archive');
h.drive.addFile(folder, 'old.pdf');
h.drive.addFile(h.rootId, 'keep.txt');
await h.sync();
const before = await h.itemNamed('old.pdf');
await exclude(h, folder);
const removed = await h.sync();
expect(h.acc.calls.filter((c) => c.startsWith('delete'))).toEqual(['delete Archive']);
expect(h.acc.tree()).toEqual(['keep.txt@1']);
expect((await h.itemNamed('old.pdf')).status).toBe('excluded');
const events = await h.runs.listEvents(removed.runId);
expect(events.map((e) => `${e.action} ${e.path}: ${e.message}`)).toEqual([
`deleted Archive: ${EXCLUDED_BY_ADMIN}`
]);
await include(h, folder);
const restored = await h.sync();
expect(restored.counts).toMatchObject({ restored: 2, created: 0 });
expect(h.acc.tree()).toEqual(['Archive/', 'Archive/old.pdf@1', 'keep.txt@1']);
expect((await h.itemNamed('old.pdf')).synced.targetId).toBe(before.synced.targetId);
});
it('keeps new and moved-in items out of ACC while their folder is excluded', async () => {
const h = await createHarness();
const folder = h.drive.addFolder(h.rootId, 'Private');
const moving = h.drive.addFile(h.rootId, 'moving.txt');
await h.sync();
await exclude(h, folder);
await h.sync();
h.drive.addFile(folder, 'new.txt');
h.drive.move(moving, folder);
await h.sync();
expect(h.acc.tree()).toEqual([]);
expect((await h.itemNamed('new.txt')).status).toBe('excluded');
});
});
describe('failures', () => {
it('retries a failing item, then reports it after the last attempt', async () => {
const h = await createHarness({ maxAttempts: 3 });

View File

@@ -1,4 +1,5 @@
import { errorMessage } from './errors.ts';
import { applyExclusions } from './exclusion.ts';
import { ingestChanges } from './ingest.ts';
import {
emptyCounts,
@@ -60,6 +61,7 @@ export class SyncEngine implements ProjectSyncRunner {
logger.warn('Project paused', { projectId, reason: pausedReason });
} else {
if (!project.firstSyncedAt) await projects.markFirstSynced(projectId, clock());
await this.applyExclusions(project, current);
const result = await this.deps.reconciler.reconcile(project, runId, current);
counts = result.counts;
error = result.abortedWith;
@@ -101,4 +103,16 @@ export class SyncEngine implements ProjectSyncRunner {
cursor = null;
}
}
/** Brings the excluded flags in `current` in line with the project's excluded folders. */
private async applyExclusions(project: Project, current: Map<string, ItemState>): Promise<void> {
const changed = applyExclusions(
project.source.rootItemId,
current,
project.excludedFolderIds,
this.deps.maxAttempts
);
await this.deps.items.save(project.id, changed);
for (const item of changed) current.set(item.sourceId, item);
}
}

View File

@@ -0,0 +1,30 @@
import type { ItemState } from './model.ts';
import { withStatus } from './plan.ts';
import { ItemTree } from './tree.ts';
/**
* Flags every item that is, or sits under, one of the excluded folders, and clears the
* flag everywhere else. Follows SharePoint's current layout, so moving something into
* or out of an excluded folder changes it too. Returns only the items whose flag
* changed, with fresh attempts, since each change means new work in ACC.
*/
export function applyExclusions(
rootId: string,
items: ReadonlyMap<string, ItemState>,
excludedFolderIds: readonly string[],
maxAttempts: number
): ItemState[] {
const excluded = new Set(excludedFolderIds);
const tree = new ItemTree(rootId, items);
const changed: ItemState[] = [];
for (const item of items.values()) {
const flag =
excluded.has(item.sourceId) ||
tree.ancestors(item.sourceId).some((ancestor) => excluded.has(ancestor.sourceId));
if (flag === item.excluded) continue;
changed.push(
withStatus({ ...item, excluded: flag, attemptCount: 0, lastError: null }, maxAttempts)
);
}
return changed;
}

View File

@@ -173,6 +173,7 @@ function newItem(sourceId: string, desired: DesiredState): ItemState {
deleted: false,
at: null
},
excluded: false,
status: 'pending',
attemptCount: 0,
lastError: null,

View File

@@ -23,6 +23,8 @@ export interface Project {
sourceDisplayPath: string;
target: TargetRoot;
targetDisplayPath: string;
/** SharePoint folders an admin has taken out of the sync, with everything inside them. */
excludedFolderIds: string[];
deltaCursor: string | null;
firstSyncedAt: Date | null;
lastRunAt: Date | null;
@@ -37,7 +39,13 @@ export type NewProject = Pick<
export type ProjectChanges = Partial<
Pick<
Project,
'name' | 'enabled' | 'source' | 'sourceDisplayPath' | 'target' | 'targetDisplayPath'
| 'name'
| 'enabled'
| 'source'
| 'sourceDisplayPath'
| 'target'
| 'targetDisplayPath'
| 'excludedFolderIds'
>
>;
@@ -87,13 +95,18 @@ export interface SyncedState {
at: Date | null;
}
export const ITEM_STATUSES = ['pending', 'synced', 'failed', 'deleted'] as const;
export const ITEM_STATUSES = ['pending', 'synced', 'failed', 'deleted', 'excluded'] as const;
export type ItemStatus = (typeof ITEM_STATUSES)[number];
export interface ItemState {
sourceId: string;
desired: DesiredState;
synced: SyncedState;
/**
* In or under a folder an admin excluded. Kept out of ACC like a deleted item, but
* `desired` still tracks SharePoint so including the folder again brings it back.
*/
excluded: boolean;
status: ItemStatus;
attemptCount: number;
lastError: string | null;

View File

@@ -20,11 +20,17 @@ const synced: SyncedState = {
at: new Date(0)
};
function item(d: Partial<DesiredState> = {}, s: Partial<SyncedState> = {}, attemptCount = 0) {
function item(
d: Partial<DesiredState> = {},
s: Partial<SyncedState> = {},
attemptCount = 0,
excluded = false
) {
return {
sourceId: 'x',
desired: { ...desired, ...d },
synced: { ...synced, ...s },
excluded,
status: 'pending',
attemptCount,
lastError: null,
@@ -40,6 +46,8 @@ describe('planAction', () => {
['deleted in SharePoint', item({ deleted: true }), { kind: 'delete' }],
['deleted before it was ever synced', item({ deleted: true }, { targetId: null }), null],
['already deleted in both', item({ deleted: true }, { deleted: true }), null],
['excluded while in ACC', item({}, {}, 0, true), { kind: 'delete' }],
['excluded and already out of ACC', item({}, { deleted: true }, 0, true), null],
[
'moved, renamed and edited',
item({ parentSourceId: 'q', name: 'b.txt', contentTag: 'c2' }),
@@ -59,6 +67,8 @@ describe('computeStatus', () => {
it.each([
[item(), 'synced'],
[item({ deleted: true }, { deleted: true }), 'deleted'],
[item({}, { deleted: true }, 0, true), 'excluded'],
[item({ deleted: true }, { deleted: true }, 0, true), 'deleted'],
[item({ name: 'new' }, {}, 2), 'pending'],
[item({ name: 'new' }, {}, 3), 'failed']
] as const)('%#: %s', (state, expected) => {

View File

@@ -1,9 +1,15 @@
import type { ItemState, ItemStatus, SyncAction } from './model.ts';
import type { ItemTree } from './tree.ts';
/** Whether the item should exist in ACC: not deleted in SharePoint and not excluded by an admin. */
export function isWanted(item: ItemState): boolean {
return !item.desired.deleted && !item.excluded;
}
/** The ACC operation needed to make the synced state match the desired state, if any. */
export function planAction({ desired, synced }: ItemState): SyncAction | null {
if (desired.deleted) return synced.targetId && !synced.deleted ? { kind: 'delete' } : null;
export function planAction(item: ItemState): SyncAction | null {
const { desired, synced } = item;
if (!isWanted(item)) return synced.targetId && !synced.deleted ? { kind: 'delete' } : null;
if (!synced.targetId) return { kind: 'create' };
if (synced.deleted) return { kind: 'restore' };
@@ -14,7 +20,10 @@ export function planAction({ desired, synced }: ItemState): SyncAction | null {
}
export function computeStatus(item: ItemState, maxAttempts: number): ItemStatus {
if (!planAction(item)) return item.desired.deleted ? 'deleted' : 'synced';
if (!planAction(item)) {
if (item.desired.deleted) return 'deleted';
return item.excluded ? 'excluded' : 'synced';
}
return item.attemptCount >= maxAttempts ? 'failed' : 'pending';
}
@@ -32,7 +41,7 @@ export function workQueue(
tree: ItemTree,
maxAttempts: number
): ItemState[] {
const rank = (item: ItemState) => (item.desired.deleted ? 2 : item.desired.isFolder ? 0 : 1);
const rank = (item: ItemState) => (!isWanted(item) ? 2 : item.desired.isFolder ? 0 : 1);
return [...items]
.filter((item) => computeStatus(item, maxAttempts) === 'pending')
.sort((a, b) => rank(a) - rank(b) || tree.depth(a.sourceId) - tree.depth(b.sourceId));

View File

@@ -20,6 +20,7 @@ import type {
import { ItemTree } from './tree.ts';
export const WAITING_FOR_PARENT = 'Waiting for its parent folder to sync.';
export const EXCLUDED_BY_ADMIN = 'Excluded from the sync by an admin.';
export interface ReconcilerDeps {
source: ChangeSource;
@@ -173,7 +174,7 @@ export class Reconciler {
case 'delete': {
await target.delete(refOf(item));
synced.deleted = true;
return record('deleted');
return record('deleted', item.excluded ? EXCLUDED_BY_ADMIN : undefined);
}
case 'update': {
let ref = refOf(item);

View File

@@ -70,13 +70,33 @@ describe('ProjectService', () => {
expect(await h.items.listByProject(h.project.id)).toEqual([]);
});
it('refuses to reset, delete or retry while a run is in progress', async () => {
it('refuses to reset, delete, retry or change folders while a run is in progress', async () => {
const h = await createHarness();
h.running.add(h.project.id);
const noChange = { shown: [], synced: [] };
await expect(h.projectService.reset(h.project.id)).rejects.toThrow(BUSY);
await expect(h.projectService.delete(h.project.id)).rejects.toThrow(BUSY);
await expect(h.projectService.retryItem(h.project.id, 'x')).rejects.toThrow(BUSY);
await expect(h.projectService.setFolderSync(h.project.id, noChange)).rejects.toThrow(BUSY);
});
it('saves folder choices, keeping folders that were not on the form', async () => {
const h = await createHarness();
const [a, b] = ['A', 'B'].map((name) => h.drive.addFolder(h.rootId, name));
const file = h.drive.addFile(a, 'inside.txt');
await h.sync();
const excludedIds = async () => (await h.projects.get(h.project.id))!.excludedFolderIds;
await h.projectService.setFolderSync(h.project.id, { shown: [a, b], synced: [b] });
expect(await excludedIds()).toEqual([a]);
// Still in ACC until the next sync deletes it.
expect((await h.itemNamed('inside.txt')).status).toBe('pending');
// A wasn't on this form, and a file can't be excluded on its own.
await h.projectService.setFolderSync(h.project.id, { shown: [b, file], synced: [] });
expect(await excludedIds()).toEqual([a, b]);
expect((await h.itemNamed('inside.txt')).excluded).toBe(true);
});
it('deletes a project with its items and history', async () => {

View File

@@ -1,4 +1,5 @@
import { InvalidRequestError } from './core/errors.ts';
import { applyExclusions } from './core/exclusion.ts';
import type { Project, ProjectChanges } from './core/model.ts';
import { withStatus } from './core/plan.ts';
import type {
@@ -26,6 +27,13 @@ export interface ProjectEdit {
targetInput?: string;
}
export interface FolderSyncChoice {
/** Every folder the form offered. */
shown: string[];
/** The ones left ticked. */
synced: string[];
}
export interface ProjectServiceDeps {
projects: ProjectRepository;
items: ItemRepository;
@@ -89,6 +97,31 @@ export class ProjectService {
await this.deps.projects.delete(id);
}
/**
* Saves which of the folders on the admin's form sync. Folders that weren't on the form
* (e.g. deleted in SharePoint since) keep their setting. The status view updates straight
* away; ACC catches up on the next run.
*/
async setFolderSync(id: number, choice: FolderSyncChoice): Promise<void> {
const project = await this.requireIdle(id);
const items = new Map(
(await this.deps.items.listByProject(id)).map((item) => [item.sourceId, item])
);
const shown = new Set(choice.shown);
const synced = new Set(choice.synced);
const excludedFolderIds = [
...project.excludedFolderIds.filter((folderId) => !shown.has(folderId)),
...[...shown].filter(
(folderId) => !synced.has(folderId) && items.get(folderId)?.desired.isFolder
)
];
await this.deps.projects.update(id, { excludedFolderIds });
const rootId = project.source.rootItemId;
const { maxAttempts } = this.deps;
await this.deps.items.save(id, applyExclusions(rootId, items, excludedFolderIds, maxAttempts));
}
/** Give a failed item a fresh set of attempts on the next run. */
async retryItem(projectId: number, sourceId: string): Promise<void> {
await this.requireIdle(projectId);

View File

@@ -17,6 +17,8 @@ import { FakeDrive } from '#lib/server/adapters/simulated/fake-drive.ts';
export const silentLogger: Logger = { info() {}, warn() {}, error() {} };
export type Harness = Awaited<ReturnType<typeof createHarness>>;
/** A real engine over in-memory SQLite, a fake SharePoint drive and a fake ACC. */
export async function createHarness({ maxAttempts = 3 } = {}) {
let now = Date.UTC(2026, 9, 1);

View File

@@ -251,6 +251,12 @@ a {
border-bottom: 0;
}
/* Tree tables (files, folders) have one-line rows, so cells centre on each other rather
than sharing a baseline (the arrows and icons have none). */
.register.tree td {
vertical-align: middle;
}
.table-wrap {
overflow-x: auto;
}

View File

@@ -1,13 +1,13 @@
import { error, redirect } from '@sveltejs/kit';
import { services } from '#lib/server/app.ts';
import { attempt, field } from '#lib/server/forms.ts';
import { attempt, field, fields } from '#lib/server/forms.ts';
import { idParam } from '#lib/server/params.ts';
import type { Actions, PageServerLoad } from './$types';
export const load: PageServerLoad = async ({ params }) => {
const summary = await services().status.getSummary(idParam(params.id));
if (!summary) error(404, 'Project not found');
return { summary };
const detail = await services().status.getProject(idParam(params.id));
if (!detail) error(404, 'Project not found');
return { detail };
};
export const actions: Actions = {
@@ -28,6 +28,15 @@ export const actions: Actions = {
);
},
folders: async ({ params, request }) => {
const data = await request.formData();
const choice = { shown: fields(data, 'folder'), synced: fields(data, 'sync') };
return attempt(async () => {
await services().projectService.setFolderSync(idParam(params.id), choice);
return { foldersSaved: true };
});
},
resume: ({ params }) =>
attempt(async () => {
await services().projectService.resume(idParam(params.id));

View File

@@ -1,11 +1,12 @@
<script lang="ts">
import { enhance, type SubmitFunction } from '$app/forms';
import { resolve } from '$app/paths';
import FolderSyncForm from '#lib/components/FolderSyncForm.svelte';
import ProjectFields from '#lib/components/ProjectFields.svelte';
import type { PageProps } from './$types';
let { data, form }: PageProps = $props();
const project = $derived(data.summary.project);
const project = $derived(data.detail.project);
const values = $derived(form && 'values' in form && form.values ? form.values : project);
const enabled = $derived(
form && 'values' in form && form.values ? form.values.enabled : project.enabled
@@ -43,6 +44,8 @@
</div>
{:else if form && 'resumed' in form}
<div class="notice tone-ok" role="status">Project resumed. It syncs again on the next run.</div>
{:else if form && 'foldersSaved' in form}
<div class="notice tone-ok" role="status">Folders saved. ACC catches up on the next sync.</div>
{/if}
{#if project.pausedReason}
@@ -86,6 +89,16 @@
</form>
</section>
<section class="section">
<h2>Folders</h2>
<p class="muted intro">
Untick a folder to leave it, and everything in it, out of the sync. Copies the sync made in
ACC are deleted on the next sync, and come back if you tick the folder again. New folders sync
unless they're inside an unticked one.
</p>
<FolderSyncForm nodes={data.detail.tree} excluded={project.excludedFolderIds} />
</section>
<section class="section danger-zone">
<h2>Reset or remove</h2>
<div class="option">
@@ -126,6 +139,10 @@
</div>
<style>
.intro {
max-width: 60ch;
}
.danger-zone {
border-top: 1px solid var(--rule);
padding-top: 1.5rem;