building out next phase of sync

This commit is contained in:
C-West8
2026-10-22 22:42:07 -05:00
parent db4abbeefb
commit 42a1ce7d30
34 changed files with 934 additions and 107 deletions

View File

@@ -0,0 +1,75 @@
import { InvalidRequestError, ServiceUnavailableError } from '#lib/server/sync/core/errors.ts';
import type {
FileContent,
SourceRoot,
TargetRef,
TargetRoot
} from '#lib/server/sync/core/model.ts';
import type {
ChangeBatch,
ChangeSource,
DocumentTarget,
ResolvedRoot,
SourceRootResolver,
TargetRootResolver
} from '#lib/server/sync/core/ports.ts';
const MESSAGE =
'The SharePoint and ACC connections are not built yet. Set SYNC_BACKEND=simulated to try the app.';
/**
* Stands in for the real Graph and APS adapters until they exist. Sync runs
* fail as an outage (no item is charged an attempt); project forms show the
* message to the admin.
*/
export class NotConnectedBackend
implements ChangeSource, DocumentTarget, SourceRootResolver, TargetRootResolver
{
getChanges(): Promise<ChangeBatch> {
return unavailable();
}
download(): Promise<FileContent> {
return unavailable();
}
createFolder(): Promise<TargetRef> {
return unavailable();
}
uploadFile(): Promise<TargetRef> {
return unavailable();
}
uploadVersion(): Promise<void> {
return unavailable();
}
rename(): Promise<void> {
return unavailable();
}
move(): Promise<TargetRef> {
return unavailable();
}
delete(): Promise<void> {
return unavailable();
}
restore(): Promise<TargetRef> {
return unavailable();
}
async resolveSourceRoot(): Promise<ResolvedRoot<SourceRoot>> {
throw new InvalidRequestError(MESSAGE);
}
async resolveTargetRoot(): Promise<ResolvedRoot<TargetRoot>> {
throw new InvalidRequestError(MESSAGE);
}
}
async function unavailable(): Promise<never> {
throw new ServiceUnavailableError(MESSAGE);
}

View File

@@ -1,5 +1,5 @@
import type { FileContent, TargetRef } from '../core/model';
import type { DocumentTarget } from '../core/ports';
import type { FileContent, TargetRef } from '#lib/server/sync/core/model.ts';
import type { DocumentTarget } from '#lib/server/sync/core/ports.ts';
interface AccNode {
id: string;
@@ -42,6 +42,11 @@ export class FakeAcc implements DocumentTarget {
return this.insert(parentId, name, false, ['by hand']);
}
/** Adds a folder directly, as if someone created it by hand in ACC. */
addFolderByHand(parentId: string, name: string): string {
return this.insert(parentId, name, true, []);
}
/** Live paths, e.g. `Drawings/`, `Drawings/plan.pdf@2` (file@version count). */
tree(): string[] {
return [...this.nodes.values()]

View File

@@ -1,5 +1,10 @@
import type { FileContent, SourceChange, SourceItem, SourceRoot } from '../core/model';
import type { ChangeBatch, ChangeSource } from '../core/ports';
import type {
FileContent,
SourceChange,
SourceItem,
SourceRoot
} from '#lib/server/sync/core/model.ts';
import type { ChangeBatch, ChangeSource, Clock } from '#lib/server/sync/core/ports.ts';
interface DriveNode {
id: string;
@@ -8,6 +13,7 @@ interface DriveNode {
isFolder: boolean;
content: string;
version: number;
modifiedAt: Date;
deleted: boolean;
}
@@ -22,7 +28,7 @@ export class FakeDrive implements ChangeSource {
private readonly log: string[] = [];
private nextId = 1;
constructor() {
constructor(private readonly clock: Clock = () => new Date()) {
this.nodes.set(this.driveRootId, {
id: this.driveRootId,
parentId: null,
@@ -30,6 +36,7 @@ export class FakeDrive implements ChangeSource {
isFolder: true,
content: '',
version: 0,
modifiedAt: clock(),
deleted: false
});
}
@@ -57,6 +64,17 @@ export class FakeDrive implements ChangeSource {
this.change(id, (node) => (node.parentId = parentId));
}
/** IDs of live files anywhere under `folderId`. */
filesUnder(folderId: string): string[] {
return [...this.nodes.values()]
.filter((node) => !node.isFolder && this.isAlive(node) && this.isUnder(node, folderId))
.map((node) => node.id);
}
nameOf(id: string): string {
return this.nodes.get(id)!.name;
}
/** Reports only the removed item, not its contents: the harder case for the engine. */
remove(id: string): void {
this.change(id, (node) => (node.deleted = true));
@@ -90,25 +108,41 @@ export class FakeDrive implements ChangeSource {
private add(parentId: string, name: string, isFolder: boolean, content: string): string {
const id = `sp-${this.nextId++}`;
this.nodes.set(id, { id, parentId, name, isFolder, content, version: 1, deleted: false });
const modifiedAt = this.clock();
this.nodes.set(id, {
id,
parentId,
name,
isFolder,
content,
version: 1,
modifiedAt,
deleted: false
});
this.log.push(id);
return id;
}
private change(id: string, mutate: (node: DriveNode) => void): void {
mutate(this.nodes.get(id)!);
const node = this.nodes.get(id)!;
mutate(node);
node.modifiedAt = this.clock();
this.log.push(id);
}
private isAlive(node: DriveNode): boolean {
for (
let n: DriveNode | undefined = node;
n;
n = n.parentId ? this.nodes.get(n.parentId) : undefined
) {
if (n.deleted) return false;
/** The node followed by its ancestors up to the drive root. */
private *lineage(node: DriveNode): Generator<DriveNode> {
for (let n: DriveNode | undefined = node; n; n = n.parentId ? this.nodes.get(n.parentId) : undefined) {
yield n;
}
return true;
}
private isAlive(node: DriveNode): boolean {
return [...this.lineage(node)].every((n) => !n.deleted);
}
private isUnder(node: DriveNode, folderId: string): boolean {
return [...this.lineage(node)].some((n) => n !== node && n.id === folderId);
}
private toChange(node: DriveNode): SourceChange {
@@ -119,7 +153,7 @@ export class FakeDrive implements ChangeSource {
name: node.name,
isFolder: node.isFolder,
contentTag: node.isFolder ? null : `c${node.version}`,
modifiedAt: new Date(Date.UTC(2026, 0, 1, 0, 0, node.version)),
modifiedAt: node.modifiedAt,
size: node.isFolder ? null : node.content.length
};
return { kind: 'upsert', item };

View File

@@ -0,0 +1,107 @@
import { InvalidRequestError } from '#lib/server/sync/core/errors.ts';
import type { SourceRoot, TargetRoot } from '#lib/server/sync/core/model.ts';
import type {
ResolvedRoot,
SourceRootResolver,
TargetRootResolver
} from '#lib/server/sync/core/ports.ts';
import { FakeAcc } from './fake-acc.ts';
import { FakeDrive } from './fake-drive.ts';
/** A file the simulated ACC always rejects, so the issues list has something in it. */
export const REJECTED_FILE = 'Corrupt model.rvt';
const SAMPLE_TREE: Record<string, string[]> = {
Drawings: ['A-101 Floor Plan.pdf', 'A-201 Elevations.pdf', 'S-101 Foundations.dwg'],
Specifications: ['Division 08 - Openings.docx', 'Division 26 - Electrical.docx'],
'Submittals/Approved': ['Door hardware.pdf'],
'': ['Project Directory.xlsx', REJECTED_FILE, '~$Project Directory.xlsx']
};
/**
* An in-memory SharePoint library and ACC project for trying the app without
* credentials. Any pasted path resolves to a folder named after its last
* segment; new SharePoint folders come pre-filled with sample files.
*/
export class SimulatedBackend implements SourceRootResolver, TargetRootResolver {
readonly drive = new FakeDrive();
readonly acc = new FakeAcc();
private readonly sourceFolders = new Map<string, string>();
private readonly targetFolders = new Map<string, string>();
constructor(private readonly random: () => number = Math.random) {
this.acc.failOn(REJECTED_FILE, new Error('ACC rejected the upload (simulated failure).'));
}
async resolveSourceRoot(input: string): Promise<ResolvedRoot<SourceRoot>> {
const name = lastSegment(input);
let id = this.sourceFolders.get(name);
if (!id) {
id = this.drive.addFolder(this.drive.driveRootId, name);
this.seed(id);
this.sourceFolders.set(name, id);
}
return {
root: { siteId: 'simulated-site', driveId: 'simulated-drive', rootItemId: id },
displayPath: `Simulated SharePoint / ${name}`
};
}
async resolveTargetRoot(input: string): Promise<ResolvedRoot<TargetRoot>> {
const name = lastSegment(input);
let id = this.targetFolders.get(name);
if (!id) {
id = this.acc.addFolderByHand(this.acc.rootFolderId, name);
this.targetFolders.set(name, id);
}
return {
root: { hubId: 'simulated-hub', projectId: 'simulated-project', rootFolderId: id },
displayPath: `Simulated ACC / Project Files / ${name}`
};
}
/** Makes one random change (edit, add or rename) somewhere under a synced root. */
simulateActivity(): void {
const roots = [...this.sourceFolders.values()];
if (roots.length === 0) return;
const root = this.pick(roots);
const files = this.drive.filesUnder(root).filter((id) => this.drive.nameOf(id) !== REJECTED_FILE);
const roll = this.random();
if (files.length === 0 || roll < 0.25) {
this.drive.addFile(root, `Field report ${Math.floor(this.random() * 1000)}.pdf`);
} else if (roll < 0.85) {
const file = this.pick(files);
this.drive.edit(file, `${this.drive.nameOf(file)} edited ${new Date().toISOString()}`);
} else {
const file = this.pick(files);
this.drive.rename(file, `Rev ${Math.floor(this.random() * 10)} - ${this.drive.nameOf(file)}`);
}
}
private seed(rootId: string): void {
for (const [folderPath, files] of Object.entries(SAMPLE_TREE)) {
let parent = rootId;
for (const segment of folderPath.split('/').filter(Boolean)) {
parent = this.drive.addFolder(parent, segment);
}
for (const file of files) this.drive.addFile(parent, file);
}
}
private pick<T>(values: T[]): T {
return values[Math.floor(this.random() * values.length)];
}
}
/** `https://x/sites/A/Shared%20Documents/Project%20X?web=1` → `Project X` */
function lastSegment(input: string): string {
const path = input.trim().split(/[?#]/)[0];
const name = path.split(/[/\\]/).filter(Boolean).at(-1);
if (!name) throw new InvalidRequestError('Enter a folder path or URL.');
try {
return decodeURIComponent(name);
} catch {
return name;
}
}

View File

@@ -1,7 +1,7 @@
import Database from 'better-sqlite3';
import { drizzle, type BetterSQLite3Database } from 'drizzle-orm/better-sqlite3';
import { migrate } from 'drizzle-orm/better-sqlite3/migrator';
import * as schema from './schema';
import * as schema from './schema.ts';
export type Db = BetterSQLite3Database<typeof schema>;

View File

@@ -1,8 +1,12 @@
import { eq } from 'drizzle-orm';
import type { ItemState } from '../../sync/core/model';
import type { ItemRepository } from '../../sync/core/ports';
import type { Db } from './database';
import { items, projects } from './schema';
import { and, count, eq } from 'drizzle-orm';
import {
emptyStatusCounts,
type ItemState,
type StatusCounts
} from '#lib/server/sync/core/model.ts';
import type { ItemRepository } from '#lib/server/sync/core/ports.ts';
import type { Db } from './database.ts';
import { items, projects } from './schema.ts';
type ItemRow = typeof items.$inferSelect;
type Tx = Parameters<Parameters<Db['transaction']>[0]>[0];
@@ -10,10 +14,31 @@ type Tx = Parameters<Parameters<Db['transaction']>[0]>[0];
export class SqliteItemRepository implements ItemRepository {
constructor(private readonly db: Db) {}
async get(projectId: string, sourceId: string): Promise<ItemState | null> {
const row = this.db
.select()
.from(items)
.where(and(eq(items.projectId, projectId), eq(items.sourceId, sourceId)))
.get();
return row ? toItem(row) : null;
}
async listByProject(projectId: string): Promise<ItemState[]> {
return this.db.select().from(items).where(eq(items.projectId, projectId)).all().map(toItem);
}
async countByStatus(projectId: string): Promise<StatusCounts> {
const rows = this.db
.select({ status: items.status, total: count() })
.from(items)
.where(eq(items.projectId, projectId))
.groupBy(items.status)
.all();
const counts = emptyStatusCounts();
for (const { status, total } of rows) counts[status] = total;
return counts;
}
async commitIngest(projectId: string, states: ItemState[], cursor: string): Promise<void> {
this.db.transaction((tx) => {
upsert(tx, projectId, states);

View File

@@ -1,10 +1,11 @@
import { eq } from 'drizzle-orm';
import type { NewProject, Project } from '../../sync/core/model';
import type { Clock, ProjectRepository } from '../../sync/core/ports';
import type { Db } from './database';
import { items, projects } from './schema';
import type { NewProject, Project, ProjectChanges } from '#lib/server/sync/core/model.ts';
import type { Clock, ProjectRepository } from '#lib/server/sync/core/ports.ts';
import type { Db } from './database.ts';
import { items, projects } from './schema.ts';
type ProjectRow = typeof projects.$inferSelect;
type ProjectInsert = typeof projects.$inferInsert;
export class SqliteProjectRepository implements ProjectRepository {
constructor(
@@ -24,38 +25,39 @@ export class SqliteProjectRepository implements ProjectRepository {
async create(project: NewProject): Promise<Project> {
const row = this.db
.insert(projects)
// A NewProject supplies every editable field, so toColumns returns them all.
.values({
...toColumns(project),
id: crypto.randomUUID(),
name: project.name,
spSiteId: project.source.siteId,
spDriveId: project.source.driveId,
spRootItemId: project.source.rootItemId,
spDisplayPath: project.sourceDisplayPath,
accHubId: project.target.hubId,
accProjectId: project.target.projectId,
accRootFolderId: project.target.rootFolderId,
accDisplayPath: project.targetDisplayPath,
createdAt: this.clock()
})
} as ProjectInsert)
.returning()
.get();
return toProject(row);
}
async update(id: string, changes: ProjectChanges): Promise<void> {
this.set(id, toColumns(changes));
}
async delete(id: string): Promise<void> {
this.db.delete(projects).where(eq(projects.id, id)).run();
}
async pause(id: string, reason: string): Promise<void> {
this.update(id, { pausedReason: reason });
this.set(id, { pausedReason: reason });
}
async resume(id: string): Promise<void> {
this.update(id, { pausedReason: null });
this.set(id, { pausedReason: null });
}
async markFirstSynced(id: string, at: Date): Promise<void> {
this.update(id, { firstSyncedAt: at });
this.set(id, { firstSyncedAt: at });
}
async setLastRunAt(id: string, at: Date): Promise<void> {
this.update(id, { lastRunAt: at });
this.set(id, { lastRunAt: at });
}
async reset(id: string): Promise<void> {
@@ -68,11 +70,34 @@ export class SqliteProjectRepository implements ProjectRepository {
});
}
private update(id: string, values: Partial<ProjectRow>): void {
this.db.update(projects).set(values).where(eq(projects.id, id)).run();
private set(id: string, values: Partial<ProjectRow>): void {
const defined = Object.entries(values).filter(([, value]) => value !== undefined);
if (defined.length === 0) return;
this.db
.update(projects)
.set(Object.fromEntries(defined))
.where(eq(projects.id, id))
.run();
}
}
/** Maps the editable project fields to columns, leaving out anything not provided. */
function toColumns(changes: ProjectChanges): Partial<ProjectRow> {
const { name, enabled, source, sourceDisplayPath, target, targetDisplayPath } = changes;
return {
name,
enabled,
spSiteId: source?.siteId,
spDriveId: source?.driveId,
spRootItemId: source?.rootItemId,
spDisplayPath: sourceDisplayPath,
accHubId: target?.hubId,
accProjectId: target?.projectId,
accRootFolderId: target?.rootFolderId,
accDisplayPath: targetDisplayPath
};
}
function toProject(row: ProjectRow): Project {
return {
id: row.id,

View File

@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
import { createHarness } from '../../sync/testing/harness';
import { createHarness } from '#lib/server/sync/testing/harness.ts';
describe('SqliteRunRepository', () => {
it('prunes old runs together with their events', async () => {

View File

@@ -1,8 +1,14 @@
import { and, asc, desc, eq, gte, lt } from 'drizzle-orm';
import type { RunCounts, RunEvent, RunOutcome, RunRecord, RunTrigger } from '../../sync/core/model';
import type { RunRepository } from '../../sync/core/ports';
import type { Db } from './database';
import { runEvents, runs } from './schema';
import type {
RunCounts,
RunEvent,
RunOutcome,
RunRecord,
RunTrigger
} from '#lib/server/sync/core/model.ts';
import type { RunRepository } from '#lib/server/sync/core/ports.ts';
import type { Db } from './database.ts';
import { runEvents, runs } from './schema.ts';
type RunRow = typeof runs.$inferSelect;
@@ -31,6 +37,17 @@ export class SqliteRunRepository implements RunRepository {
this.db.insert(runEvents).values(event).run();
}
async latest(projectId: string): Promise<RunRecord | null> {
const row = this.db
.select()
.from(runs)
.where(eq(runs.projectId, projectId))
.orderBy(desc(runs.startedAt))
.limit(1)
.get();
return row ? toRunRecord(row) : null;
}
async listSince(projectId: string, since: Date): Promise<RunRecord[]> {
return this.db
.select()

View File

@@ -1,5 +1,5 @@
import { index, integer, primaryKey, sqliteTable, text } from 'drizzle-orm/sqlite-core';
import { EVENT_ACTIONS } from '../../sync/core/model';
import { EVENT_ACTIONS } from '#lib/server/sync/core/model.ts';
const timestamp = (name: string) => integer(name, { mode: 'timestamp_ms' });
const flag = (name: string) => integer(name, { mode: 'boolean' });

View File

@@ -0,0 +1,50 @@
import { describe, expect, it } from 'vitest';
import { AdminAuth } from './admin-auth.ts';
function auth(overrides: Partial<ConstructorParameters<typeof AdminAuth>[0]> = {}) {
let now = 1_000_000;
const instance = new AdminAuth({
username: 'admin',
password: 'hunter2',
sessionSecret: 'secret',
sessionTtlMs: 60_000,
clock: () => new Date(now),
...overrides
});
return { instance, advance: (ms: number) => (now += ms) };
}
describe('AdminAuth', () => {
it('accepts only the configured credentials', () => {
const { instance } = auth();
expect(instance.checkCredentials('admin', 'hunter2')).toBe(true);
expect(instance.checkCredentials('admin', 'wrong')).toBe(false);
expect(instance.checkCredentials('Admin', 'hunter2')).toBe(false);
});
it('is disabled, and rejects everything, without credentials configured', () => {
const { instance } = auth({ password: undefined });
expect(instance.enabled).toBe(false);
expect(instance.checkCredentials('admin', '')).toBe(false);
expect(instance.verifyToken(instance.issueToken())).toBe(false);
});
it('verifies its own tokens until they expire', () => {
const { instance, advance } = auth();
const token = instance.issueToken();
expect(instance.verifyToken(token)).toBe(true);
advance(60_001);
expect(instance.verifyToken(token)).toBe(false);
});
it('rejects tampered tokens and tokens signed with another secret', () => {
const { instance } = auth();
const [expiresAt, signature] = instance.issueToken().split('.');
expect(instance.verifyToken(`${Number(expiresAt) + 1}.${signature}`)).toBe(false);
expect(instance.verifyToken('garbage')).toBe(false);
expect(instance.verifyToken(undefined)).toBe(false);
const other = auth({ sessionSecret: 'different' }).instance;
expect(other.verifyToken(instance.issueToken())).toBe(false);
});
});

View File

@@ -0,0 +1,57 @@
import { createHash, createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
import type { Clock } from '#lib/server/sync/core/ports.ts';
export const SESSION_COOKIE = 'admin_session';
export interface AdminAuthConfig {
username: string | undefined;
password: string | undefined;
/** Signs session tokens. If omitted, a random one is used and sessions end on restart. */
sessionSecret: string | undefined;
sessionTtlMs: number;
clock: Clock;
}
/** The single admin account from env, with stateless signed session tokens. */
export class AdminAuth {
private readonly secret: string;
constructor(private readonly config: AdminAuthConfig) {
this.secret = config.sessionSecret || randomBytes(32).toString('hex');
}
get enabled(): boolean {
return Boolean(this.config.username && this.config.password);
}
checkCredentials(username: string, password: string): boolean {
if (!this.enabled) return false;
// Evaluate both so timing doesn't reveal which one was wrong.
const userOk = safeEqual(username, this.config.username!);
const passwordOk = safeEqual(password, this.config.password!);
return userOk && passwordOk;
}
/** A token of the form `<expiresAtMs>.<signature>`. */
issueToken(): string {
const expiresAt = String(this.config.clock().getTime() + this.config.sessionTtlMs);
return `${expiresAt}.${this.sign(expiresAt)}`;
}
verifyToken(token: string | undefined): boolean {
if (!this.enabled || !token) return false;
const [expiresAt, signature] = token.split('.');
if (!expiresAt || !signature || !safeEqual(signature, this.sign(expiresAt))) return false;
return Number(expiresAt) > this.config.clock().getTime();
}
private sign(value: string): string {
return createHmac('sha256', this.secret).update(value).digest('base64url');
}
}
/** Constant-time comparison that also hides length differences. */
function safeEqual(a: string, b: string): boolean {
const digest = (value: string) => createHash('sha256').update(value).digest();
return timingSafeEqual(digest(a), digest(b));
}

16
src/lib/server/logger.ts Normal file
View File

@@ -0,0 +1,16 @@
import type { Logger } from '#lib/server/sync/core/ports.ts';
type Level = 'info' | 'warn' | 'error';
/** One JSON object per line on stdout/stderr, which Docker and Portainer show as-is. */
export const consoleLogger: Logger = {
info: (message, meta) => write('info', message, meta),
warn: (message, meta) => write('warn', message, meta),
error: (message, meta) => write('error', message, meta)
};
function write(level: Level, message: string, meta?: Record<string, unknown>): void {
const line = JSON.stringify({ time: new Date().toISOString(), level, message, ...meta });
if (level === 'info') console.log(line);
else console.error(line);
}

View File

@@ -0,0 +1,149 @@
import type {
Project,
RunEvent,
RunRecord,
StatusCounts
} from '#lib/server/sync/core/model.ts';
import type {
Clock,
ItemRepository,
ProjectRepository,
RunRepository
} from '#lib/server/sync/core/ports.ts';
import { ItemTree } from '#lib/server/sync/core/tree.ts';
import { buildTree, type TreeNode } from './tree-view.ts';
export type ProjectHealth =
| 'disabled'
| 'paused'
| 'running'
| 'issues'
| 'pending'
| 'never-run'
| 'healthy';
export interface ProjectSummary {
project: Project;
health: ProjectHealth;
running: boolean;
counts: StatusCounts;
lastRun: RunRecord | null;
issueCount: number;
}
export interface Issue {
/** Null for project-level issues. */
sourceId: string | null;
path: string;
message: string;
attempts: number;
lastAttemptAt: Date | null;
}
export interface ProjectDetail extends ProjectSummary {
issues: Issue[];
tree: TreeNode[];
}
export interface StatusServiceDeps {
projects: ProjectRepository;
items: ItemRepository;
runs: RunRepository;
activity: { isRunning(projectId: string): boolean };
clock: Clock;
retentionDays: number;
}
/** Read-only views for the status pages. Never calls SharePoint or ACC. */
export class StatusService {
constructor(private readonly deps: StatusServiceDeps) {}
async listProjects(): Promise<ProjectSummary[]> {
const projects = await this.deps.projects.list();
return Promise.all(projects.map((project) => this.summarize(project)));
}
async getProject(id: string): Promise<ProjectDetail | null> {
const project = await this.deps.projects.get(id);
if (!project) return null;
const [summary, items] = await Promise.all([
this.summarize(project),
this.deps.items.listByProject(id)
]);
const rootId = project.source.rootItemId;
const paths = new ItemTree(rootId, new Map(items.map((item) => [item.sourceId, item])));
const itemIssues: Issue[] = items
.filter((item) => item.status === 'failed')
.map((item) => ({
sourceId: item.sourceId,
path: paths.path(item.sourceId),
message: `${item.desired.deleted ? 'Delete failed: ' : ''}${item.lastError ?? 'Unknown error'}`,
attempts: item.attemptCount,
lastAttemptAt: item.lastAttemptAt
}))
.sort((a, b) => a.path.localeCompare(b.path));
return {
...summary,
issues: [...projectIssues(project, summary.lastRun), ...itemIssues],
tree: buildTree(rootId, items)
};
}
async history(projectId: string): Promise<RunRecord[]> {
const since = new Date(this.deps.clock().getTime() - this.deps.retentionDays * 86_400_000);
return this.deps.runs.listSince(projectId, since);
}
async runEvents(projectId: string, runId: string): Promise<RunEvent[]> {
const events = await this.deps.runs.listEvents(runId);
return events.filter((event) => event.projectId === projectId);
}
private async summarize(project: Project): Promise<ProjectSummary> {
const [counts, lastRun] = await Promise.all([
this.deps.items.countByStatus(project.id),
this.deps.runs.latest(project.id)
]);
const running = this.deps.activity.isRunning(project.id);
return {
project,
running,
counts,
lastRun,
health: healthOf(project, running, counts, lastRun),
issueCount: projectIssues(project, lastRun).length + counts.failed
};
}
}
function healthOf(
project: Project,
running: boolean,
counts: StatusCounts,
lastRun: RunRecord | null
): ProjectHealth {
if (!project.enabled) return 'disabled';
if (project.pausedReason) return 'paused';
if (running) return 'running';
if (counts.failed > 0 || lastRun?.outcome === 'failed') return 'issues';
if (counts.pending > 0) return 'pending';
if (!lastRun) return 'never-run';
return 'healthy';
}
function projectIssues(project: Project, lastRun: RunRecord | null): Issue[] {
const issue = (message: string): Issue => ({
sourceId: null,
path: 'Project',
message,
attempts: 0,
lastAttemptAt: lastRun?.finishedAt ?? null
});
if (project.pausedReason) return [issue(`Paused: ${project.pausedReason}`)];
if (lastRun?.outcome === 'failed' && lastRun.error) {
return [issue(`Last run failed: ${lastRun.error}`)];
}
return [];
}

View File

@@ -0,0 +1,66 @@
import type { ItemState, ItemStatus } from '#lib/server/sync/core/model.ts';
import { WAITING_FOR_PARENT } from '#lib/server/sync/core/reconciler.ts';
export interface TreeNode {
sourceId: string;
name: string;
isFolder: boolean;
status: ItemStatus;
/** Pending only because its parent folder hasn't synced yet. */
waiting: boolean;
error: string | null;
modifiedAt: Date;
syncedAt: Date | null;
/** Failed items anywhere inside this folder (0 for files). */
problems: number;
children: TreeNode[];
}
const collator = new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' });
/** Nests the project's live items under the root, folders first, then by name. */
export function buildTree(rootId: string, items: Iterable<ItemState>): TreeNode[] {
const byParent = new Map<string, ItemState[]>();
for (const item of items) {
if (item.desired.deleted) continue;
const siblings = byParent.get(item.desired.parentSourceId) ?? [];
siblings.push(item);
byParent.set(item.desired.parentSourceId, siblings);
}
const seen = new Set<string>(); // guards against parent cycles
const unseen = (item: ItemState) => {
if (seen.has(item.sourceId)) return false;
seen.add(item.sourceId);
return true;
};
const build = (parentId: string): TreeNode[] =>
(byParent.get(parentId) ?? [])
.filter(unseen)
.sort(
(a, b) =>
Number(b.desired.isFolder) - Number(a.desired.isFolder) ||
collator.compare(a.desired.name, b.desired.name)
)
.map((item) => toNode(item, item.desired.isFolder ? build(item.sourceId) : []));
return build(rootId);
}
function toNode(item: ItemState, children: TreeNode[]): TreeNode {
return {
sourceId: item.sourceId,
name: item.desired.name,
isFolder: item.desired.isFolder,
status: item.status,
waiting: item.status === 'pending' && item.lastError === WAITING_FOR_PARENT,
error: item.lastError,
modifiedAt: item.desired.modifiedAt,
syncedAt: item.synced.at,
problems: children.reduce(
(sum, child) => sum + child.problems + (child.status === 'failed' ? 1 : 0),
0
),
children
};
}

View File

@@ -1,9 +1,9 @@
import { describe, expect, it } from 'vitest';
import type { ProjectSyncRunner } from './core/engine';
import { emptyCounts, type Project, type RunTrigger } from './core/model';
import type { ProjectRepository } from './core/ports';
import { SyncCoordinator } from './coordinator';
import { silentLogger } from './testing/harness';
import type { ProjectSyncRunner } from './core/engine.ts';
import { emptyCounts, type Project, type RunTrigger } from './core/model.ts';
import type { ProjectRepository } from './core/ports.ts';
import { SyncCoordinator } from './coordinator.ts';
import { silentLogger } from './testing/harness.ts';
function project(id: string, overrides: Partial<Project> = {}): Project {
return { id, enabled: true, pausedReason: null, ...overrides } as Project;

View File

@@ -1,7 +1,7 @@
import type { ProjectSyncRunner } from './core/engine';
import { errorMessage } from './core/errors';
import type { RunTrigger } from './core/model';
import type { Clock, Logger, ProjectRepository } from './core/ports';
import type { ProjectSyncRunner } from './core/engine.ts';
import { errorMessage } from './core/errors.ts';
import type { RunTrigger } from './core/model.ts';
import type { Clock, Logger, ProjectRepository } from './core/ports.ts';
export type RunRequestResult =
| { status: 'started' }

View File

@@ -1,8 +1,8 @@
import { describe, expect, it } from 'vitest';
import { createHarness } from '../testing/harness';
import { ServiceUnavailableError } from './errors';
import { ROOT_DELETED } from './ingest';
import { WAITING_FOR_PARENT } from './reconciler';
import { createHarness } from '#lib/server/sync/testing/harness.ts';
import { ServiceUnavailableError } from './errors.ts';
import { ROOT_DELETED } from './ingest.ts';
import { WAITING_FOR_PARENT } from './reconciler.ts';
describe('SyncEngine', () => {
it('mirrors the initial tree and ignores temp files', async () => {

View File

@@ -1,5 +1,5 @@
import { errorMessage } from './errors';
import { ingestChanges } from './ingest';
import { errorMessage } from './errors.ts';
import { ingestChanges } from './ingest.ts';
import {
emptyCounts,
type ItemState,
@@ -7,7 +7,7 @@ import {
type RunOutcome,
type RunResult,
type RunTrigger
} from './model';
} from './model.ts';
import type {
ChangeSource,
Clock,
@@ -16,8 +16,8 @@ import type {
Logger,
ProjectRepository,
RunRepository
} from './ports';
import type { Reconciler } from './reconciler';
} from './ports.ts';
import type { Reconciler } from './reconciler.ts';
export interface ProjectSyncRunner {
runProject(projectId: string, trigger: RunTrigger): Promise<RunResult>;

View File

@@ -10,6 +10,14 @@ export class ServiceUnavailableError extends Error {
}
}
/** A request that can't be carried out as asked; the message is safe to show to the user. */
export class InvalidRequestError extends Error {
constructor(message: string) {
super(message);
this.name = 'InvalidRequestError';
}
}
export function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}

View File

@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
import { PatternIgnorePolicy } from './ignore';
import { PatternIgnorePolicy } from './ignore.ts';
describe('PatternIgnorePolicy', () => {
const policy = new PatternIgnorePolicy();

View File

@@ -1,4 +1,4 @@
import type { IgnorePolicy } from './ports';
import type { IgnorePolicy } from './ports.ts';
/** Office lock files, temp/partial downloads and OS metadata. */
export const DEFAULT_IGNORE_PATTERNS = [

View File

@@ -1,7 +1,7 @@
import { describe, expect, it } from 'vitest';
import { PatternIgnorePolicy } from './ignore';
import { ingestChanges, ROOT_MISSING, type IngestInput } from './ingest';
import type { ItemState, SourceChange, SourceItem } from './model';
import { PatternIgnorePolicy } from './ignore.ts';
import { ingestChanges, ROOT_MISSING, type IngestInput } from './ingest.ts';
import type { ItemState, SourceChange, SourceItem } from './model.ts';
const ROOT = 'root';

View File

@@ -1,7 +1,7 @@
import type { DesiredState, ItemState, SourceChange, SourceItem } from './model';
import { withStatus } from './plan';
import type { IgnorePolicy } from './ports';
import { ItemTree } from './tree';
import type { DesiredState, ItemState, SourceChange, SourceItem } from './model.ts';
import { withStatus } from './plan.ts';
import type { IgnorePolicy } from './ports.ts';
import { ItemTree } from './tree.ts';
export const ROOT_DELETED = 'The SharePoint root folder was deleted.';
export const ROOT_MISSING = 'The SharePoint root folder was not found in a full listing.';

View File

@@ -34,6 +34,13 @@ export type NewProject = Pick<
'name' | 'source' | 'sourceDisplayPath' | 'target' | 'targetDisplayPath'
>;
export type ProjectChanges = Partial<
Pick<
Project,
'name' | 'enabled' | 'source' | 'sourceDisplayPath' | 'target' | 'targetDisplayPath'
>
>;
/** An item as SharePoint currently describes it. */
export interface SourceItem {
sourceId: string;
@@ -80,7 +87,8 @@ export interface SyncedState {
at: Date | null;
}
export type ItemStatus = 'pending' | 'synced' | 'failed' | 'deleted';
export const ITEM_STATUSES = ['pending', 'synced', 'failed', 'deleted'] as const;
export type ItemStatus = (typeof ITEM_STATUSES)[number];
export interface ItemState {
sourceId: string;
@@ -114,8 +122,18 @@ export type EventAction = (typeof EVENT_ACTIONS)[number];
export type RunCounts = Record<EventAction, number>;
export type StatusCounts = Record<ItemStatus, number>;
function zeroed<K extends string>(keys: readonly K[]): Record<K, number> {
return Object.fromEntries(keys.map((key) => [key, 0])) as Record<K, number>;
}
export function emptyCounts(): RunCounts {
return Object.fromEntries(EVENT_ACTIONS.map((a) => [a, 0])) as RunCounts;
return zeroed(EVENT_ACTIONS);
}
export function emptyStatusCounts(): StatusCounts {
return zeroed(ITEM_STATUSES);
}
export interface RunResult {

View File

@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest';
import type { DesiredState, ItemState, SyncedState } from './model';
import { computeStatus, planAction } from './plan';
import type { DesiredState, ItemState, SyncedState } from './model.ts';
import { computeStatus, planAction } from './plan.ts';
const desired: DesiredState = {
parentSourceId: 'p',

View File

@@ -1,5 +1,5 @@
import type { ItemState, ItemStatus, SyncAction } from './model';
import type { ItemTree } from './tree';
import type { ItemState, ItemStatus, SyncAction } from './model.ts';
import type { ItemTree } from './tree.ts';
/** The ACC operation needed to make the synced state match the desired state, if any. */
export function planAction({ desired, synced }: ItemState): SyncAction | null {

View File

@@ -3,6 +3,7 @@ import type {
ItemState,
NewProject,
Project,
ProjectChanges,
RunEvent,
RunOutcome,
RunCounts,
@@ -10,8 +11,10 @@ import type {
RunTrigger,
SourceChange,
SourceRoot,
TargetRef
} from './model';
StatusCounts,
TargetRef,
TargetRoot
} from './model.ts';
export interface ChangeBatch {
changes: SourceChange[];
@@ -52,10 +55,30 @@ export interface IgnorePolicy {
isIgnored(name: string, isFolder: boolean): boolean;
}
export interface ResolvedRoot<T> {
root: T;
displayPath: string;
}
/**
* Turn what an admin pastes (e.g. a folder URL) into stable IDs. Throw
* `InvalidRequestError` when the input can't be resolved.
*/
export interface SourceRootResolver {
resolveSourceRoot(input: string): Promise<ResolvedRoot<SourceRoot>>;
}
export interface TargetRootResolver {
resolveTargetRoot(input: string): Promise<ResolvedRoot<TargetRoot>>;
}
export interface ProjectRepository {
get(id: string): Promise<Project | null>;
list(): Promise<Project[]>;
create(project: NewProject): Promise<Project>;
update(id: string, changes: ProjectChanges): Promise<void>;
/** Removes the project with all its items and history. */
delete(id: string): Promise<void>;
pause(id: string, reason: string): Promise<void>;
resume(id: string): Promise<void>;
markFirstSynced(id: string, at: Date): Promise<void>;
@@ -65,7 +88,9 @@ export interface ProjectRepository {
}
export interface ItemRepository {
get(projectId: string, sourceId: string): Promise<ItemState | null>;
listByProject(projectId: string): Promise<ItemState[]>;
countByStatus(projectId: string): Promise<StatusCounts>;
/** Atomically store ingested changes and the cursor they were read up to. */
commitIngest(projectId: string, items: ItemState[], cursor: string): Promise<void>;
save(projectId: string, items: ItemState[]): Promise<void>;
@@ -78,6 +103,7 @@ export interface RunRepository {
result: { finishedAt: Date; outcome: RunOutcome; error: string | null; counts: RunCounts }
): Promise<void>;
addEvent(event: RunEvent): Promise<void>;
latest(projectId: string): Promise<RunRecord | null>;
listSince(projectId: string, since: Date): Promise<RunRecord[]>;
listEvents(runId: string): Promise<RunEvent[]>;
/** Delete runs (and their events) started before `cutoff`; returns how many runs were removed. */

View File

@@ -1,4 +1,4 @@
import { errorMessage, ServiceUnavailableError } from './errors';
import { errorMessage, ServiceUnavailableError } from './errors.ts';
import {
emptyCounts,
type EventAction,
@@ -8,10 +8,16 @@ import {
type SyncAction,
type SyncedState,
type TargetRef
} from './model';
import { planAction, withStatus, workQueue } from './plan';
import type { ChangeSource, Clock, DocumentTarget, ItemRepository, RunRepository } from './ports';
import { ItemTree } from './tree';
} from './model.ts';
import { planAction, withStatus, workQueue } from './plan.ts';
import type {
ChangeSource,
Clock,
DocumentTarget,
ItemRepository,
RunRepository
} from './ports.ts';
import { ItemTree } from './tree.ts';
export const WAITING_FOR_PARENT = 'Waiting for its parent folder to sync.';

View File

@@ -1,4 +1,4 @@
import type { ItemState } from './model';
import type { ItemState } from './model.ts';
type ParentOf = (item: ItemState) => string | null;

View File

@@ -0,0 +1,125 @@
import { InvalidRequestError } from './core/errors.ts';
import type { Project, ProjectChanges } from './core/model.ts';
import { withStatus } from './core/plan.ts';
import type {
ItemRepository,
ProjectRepository,
SourceRootResolver,
TargetRootResolver
} from './core/ports.ts';
export const ROOTS_LOCKED =
'Roots are locked after the first sync. Reset the project to change them.';
export const BUSY = 'A sync is running for this project. Try again when it finishes.';
export interface NewProjectInput {
name: string;
sourceInput: string;
targetInput: string;
}
export interface ProjectEdit {
name: string;
enabled: boolean;
/** Blank or omitted leaves the root unchanged. */
sourceInput?: string;
targetInput?: string;
}
export interface ProjectServiceDeps {
projects: ProjectRepository;
items: ItemRepository;
sourceResolver: SourceRootResolver;
targetResolver: TargetRootResolver;
activity: { isRunning(projectId: string): boolean };
maxAttempts: number;
}
/** Admin operations on projects, enforcing the rules that keep sync state consistent. */
export class ProjectService {
constructor(private readonly deps: ProjectServiceDeps) {}
async create(input: NewProjectInput): Promise<Project> {
const [source, target] = await Promise.all([
this.deps.sourceResolver.resolveSourceRoot(required(input.sourceInput, 'SharePoint folder')),
this.deps.targetResolver.resolveTargetRoot(required(input.targetInput, 'ACC folder'))
]);
return this.deps.projects.create({
name: validName(input.name),
source: source.root,
sourceDisplayPath: source.displayPath,
target: target.root,
targetDisplayPath: target.displayPath
});
}
async update(id: string, edit: ProjectEdit): Promise<void> {
const project = await this.require(id);
const changes: ProjectChanges = { name: validName(edit.name), enabled: edit.enabled };
const sourceInput = edit.sourceInput?.trim();
const targetInput = edit.targetInput?.trim();
if ((sourceInput || targetInput) && project.firstSyncedAt) {
throw new InvalidRequestError(ROOTS_LOCKED);
}
if (sourceInput) {
const source = await this.deps.sourceResolver.resolveSourceRoot(sourceInput);
Object.assign(changes, { source: source.root, sourceDisplayPath: source.displayPath });
}
if (targetInput) {
const target = await this.deps.targetResolver.resolveTargetRoot(targetInput);
Object.assign(changes, { target: target.root, targetDisplayPath: target.displayPath });
}
await this.deps.projects.update(id, changes);
}
async resume(id: string): Promise<void> {
await this.require(id);
await this.deps.projects.resume(id);
}
/** Forget all sync state; the next run starts from a full listing and roots unlock. */
async reset(id: string): Promise<void> {
await this.requireIdle(id);
await this.deps.projects.reset(id);
}
async delete(id: string): Promise<void> {
await this.requireIdle(id);
await this.deps.projects.delete(id);
}
/** Give a failed item a fresh set of attempts on the next run. */
async retryItem(projectId: string, sourceId: string): Promise<void> {
await this.requireIdle(projectId);
const item = await this.deps.items.get(projectId, sourceId);
if (!item) throw new InvalidRequestError('That file is no longer tracked.');
if (item.status !== 'failed') return;
const fresh = withStatus({ ...item, attemptCount: 0, lastError: null }, this.deps.maxAttempts);
await this.deps.items.save(projectId, [fresh]);
}
private async require(id: string): Promise<Project> {
const project = await this.deps.projects.get(id);
if (!project) throw new InvalidRequestError('Project not found.');
return project;
}
private async requireIdle(id: string): Promise<Project> {
const project = await this.require(id);
if (this.deps.activity.isRunning(id)) throw new InvalidRequestError(BUSY);
return project;
}
}
function required(value: string, label: string): string {
const trimmed = value.trim();
if (!trimmed) throw new InvalidRequestError(`${label} is required.`);
return trimmed;
}
function validName(name: string): string {
const trimmed = required(name, 'Name');
if (trimmed.length > 100) throw new InvalidRequestError('Name must be 100 characters or fewer.');
return trimmed;
}

View File

@@ -1,6 +1,6 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { IntervalScheduler } from './scheduler';
import { silentLogger } from './testing/harness';
import { IntervalScheduler } from './scheduler.ts';
import { silentLogger } from './testing/harness.ts';
describe('IntervalScheduler', () => {
beforeEach(() => vi.useFakeTimers());
@@ -23,6 +23,18 @@ describe('IntervalScheduler', () => {
scheduler.stop();
});
it('can run the first time immediately', async () => {
const scheduler = new IntervalScheduler(silentLogger);
const task = vi.fn().mockResolvedValue(undefined);
scheduler.every('eager', 1000, task, { immediate: true });
await vi.advanceTimersByTimeAsync(0);
expect(task).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(1000);
expect(task).toHaveBeenCalledTimes(2);
scheduler.stop();
});
it('keeps going after a task throws, and stops cleanly', async () => {
const scheduler = new IntervalScheduler(silentLogger);
const task = vi.fn().mockRejectedValueOnce(new Error('boom')).mockResolvedValue(undefined);

View File

@@ -1,5 +1,5 @@
import { errorMessage } from './core/errors';
import type { Logger } from './core/ports';
import { errorMessage } from './core/errors.ts';
import type { Logger } from './core/ports.ts';
/**
* Runs async tasks on fixed intervals. The next run of a task is scheduled only
@@ -11,7 +11,13 @@ export class IntervalScheduler {
constructor(private readonly logger: Logger) {}
every(name: string, intervalMs: number, task: () => Promise<unknown>): void {
/** With `immediate`, the first run starts right away instead of after one interval. */
every(
name: string,
intervalMs: number,
task: () => Promise<unknown>,
{ immediate = false } = {}
): void {
const tick = async () => {
try {
await task();
@@ -20,15 +26,15 @@ export class IntervalScheduler {
}
schedule();
};
const schedule = () => {
const schedule = (delayMs = intervalMs) => {
if (this.stopped) return;
const timer = setTimeout(() => {
this.timers.delete(timer);
void tick();
}, intervalMs);
}, delayMs);
this.timers.add(timer);
};
schedule();
schedule(immediate ? 0 : intervalMs);
}
stop(): void {

View File

@@ -1,13 +1,13 @@
import { openDatabase } from '../../adapters/sqlite/database';
import { SqliteItemRepository } from '../../adapters/sqlite/item-repository';
import { SqliteProjectRepository } from '../../adapters/sqlite/project-repository';
import { SqliteRunRepository } from '../../adapters/sqlite/run-repository';
import { SyncEngine } from '../core/engine';
import { PatternIgnorePolicy } from '../core/ignore';
import type { Logger } from '../core/ports';
import { Reconciler } from '../core/reconciler';
import { FakeAcc } from './fake-acc';
import { FakeDrive } from './fake-drive';
import { openDatabase } from '#lib/server/adapters/sqlite/database.ts';
import { SqliteItemRepository } from '#lib/server/adapters/sqlite/item-repository.ts';
import { SqliteProjectRepository } from '#lib/server/adapters/sqlite/project-repository.ts';
import { SqliteRunRepository } from '#lib/server/adapters/sqlite/run-repository.ts';
import { SyncEngine } from '#lib/server/sync/core/engine.ts';
import { PatternIgnorePolicy } from '#lib/server/sync/core/ignore.ts';
import type { Logger } from '#lib/server/sync/core/ports.ts';
import { Reconciler } from '#lib/server/sync/core/reconciler.ts';
import { FakeAcc } from '#lib/server/adapters/simulated/fake-acc.ts';
import { FakeDrive } from '#lib/server/adapters/simulated/fake-drive.ts';
export const silentLogger: Logger = { info() {}, warn() {}, error() {} };