first draft
This commit is contained in:
16
src/lib/server/adapters/sqlite/database.ts
Normal file
16
src/lib/server/adapters/sqlite/database.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import { drizzle, type BetterSQLite3Database } from 'drizzle-orm/better-sqlite3';
|
||||
import { migrate } from 'drizzle-orm/better-sqlite3/migrator';
|
||||
import * as schema from './schema';
|
||||
|
||||
export type Db = BetterSQLite3Database<typeof schema>;
|
||||
|
||||
/** Opens (or creates) the database and brings its schema up to date. */
|
||||
export function openDatabase(path: string, migrationsFolder = 'drizzle'): Db {
|
||||
const client = new Database(path);
|
||||
client.pragma('journal_mode = WAL');
|
||||
client.pragma('foreign_keys = ON');
|
||||
const db = drizzle(client, { schema });
|
||||
migrate(db, { migrationsFolder });
|
||||
return db;
|
||||
}
|
||||
88
src/lib/server/adapters/sqlite/item-repository.ts
Normal file
88
src/lib/server/adapters/sqlite/item-repository.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
import { eq } from 'drizzle-orm';
|
||||
import type { ItemState } from '../../sync/core/model';
|
||||
import type { ItemRepository } from '../../sync/core/ports';
|
||||
import type { Db } from './database';
|
||||
import { items, projects } from './schema';
|
||||
|
||||
type ItemRow = typeof items.$inferSelect;
|
||||
type Tx = Parameters<Parameters<Db['transaction']>[0]>[0];
|
||||
|
||||
export class SqliteItemRepository implements ItemRepository {
|
||||
constructor(private readonly db: Db) {}
|
||||
|
||||
async listByProject(projectId: string): Promise<ItemState[]> {
|
||||
return this.db.select().from(items).where(eq(items.projectId, projectId)).all().map(toItem);
|
||||
}
|
||||
|
||||
async commitIngest(projectId: string, states: ItemState[], cursor: string): Promise<void> {
|
||||
this.db.transaction((tx) => {
|
||||
upsert(tx, projectId, states);
|
||||
tx.update(projects).set({ deltaCursor: cursor }).where(eq(projects.id, projectId)).run();
|
||||
});
|
||||
}
|
||||
|
||||
async save(projectId: string, states: ItemState[]): Promise<void> {
|
||||
this.db.transaction((tx) => upsert(tx, projectId, states));
|
||||
}
|
||||
}
|
||||
|
||||
function upsert(tx: Tx, projectId: string, states: ItemState[]): void {
|
||||
for (const state of states) {
|
||||
const row = toRow(projectId, state);
|
||||
tx.insert(items)
|
||||
.values(row)
|
||||
.onConflictDoUpdate({ target: [items.projectId, items.sourceId], set: row })
|
||||
.run();
|
||||
}
|
||||
}
|
||||
|
||||
function toRow(projectId: string, { sourceId, desired, synced, ...rest }: ItemState): ItemRow {
|
||||
return {
|
||||
projectId,
|
||||
sourceId,
|
||||
parentSourceId: desired.parentSourceId,
|
||||
name: desired.name,
|
||||
isFolder: desired.isFolder,
|
||||
contentTag: desired.contentTag,
|
||||
sourceModifiedAt: desired.modifiedAt,
|
||||
size: desired.size,
|
||||
deleted: desired.deleted,
|
||||
targetId: synced.targetId,
|
||||
syncedParentSourceId: synced.parentSourceId,
|
||||
syncedName: synced.name,
|
||||
syncedContentTag: synced.contentTag,
|
||||
syncedDeleted: synced.deleted,
|
||||
syncedAt: synced.at,
|
||||
status: rest.status,
|
||||
attemptCount: rest.attemptCount,
|
||||
lastError: rest.lastError,
|
||||
lastAttemptAt: rest.lastAttemptAt
|
||||
};
|
||||
}
|
||||
|
||||
function toItem(row: ItemRow): ItemState {
|
||||
return {
|
||||
sourceId: row.sourceId,
|
||||
desired: {
|
||||
parentSourceId: row.parentSourceId,
|
||||
name: row.name,
|
||||
isFolder: row.isFolder,
|
||||
contentTag: row.contentTag,
|
||||
modifiedAt: row.sourceModifiedAt,
|
||||
size: row.size,
|
||||
deleted: row.deleted
|
||||
},
|
||||
synced: {
|
||||
targetId: row.targetId,
|
||||
parentSourceId: row.syncedParentSourceId,
|
||||
name: row.syncedName,
|
||||
contentTag: row.syncedContentTag,
|
||||
deleted: row.syncedDeleted,
|
||||
at: row.syncedAt
|
||||
},
|
||||
status: row.status,
|
||||
attemptCount: row.attemptCount,
|
||||
lastError: row.lastError,
|
||||
lastAttemptAt: row.lastAttemptAt
|
||||
};
|
||||
}
|
||||
95
src/lib/server/adapters/sqlite/project-repository.ts
Normal file
95
src/lib/server/adapters/sqlite/project-repository.ts
Normal file
@@ -0,0 +1,95 @@
|
||||
import { eq } from 'drizzle-orm';
|
||||
import type { NewProject, Project } from '../../sync/core/model';
|
||||
import type { Clock, ProjectRepository } from '../../sync/core/ports';
|
||||
import type { Db } from './database';
|
||||
import { items, projects } from './schema';
|
||||
|
||||
type ProjectRow = typeof projects.$inferSelect;
|
||||
|
||||
export class SqliteProjectRepository implements ProjectRepository {
|
||||
constructor(
|
||||
private readonly db: Db,
|
||||
private readonly clock: Clock
|
||||
) {}
|
||||
|
||||
async get(id: string): Promise<Project | null> {
|
||||
const row = this.db.select().from(projects).where(eq(projects.id, id)).get();
|
||||
return row ? toProject(row) : null;
|
||||
}
|
||||
|
||||
async list(): Promise<Project[]> {
|
||||
return this.db.select().from(projects).orderBy(projects.name).all().map(toProject);
|
||||
}
|
||||
|
||||
async create(project: NewProject): Promise<Project> {
|
||||
const row = this.db
|
||||
.insert(projects)
|
||||
.values({
|
||||
id: crypto.randomUUID(),
|
||||
name: project.name,
|
||||
spSiteId: project.source.siteId,
|
||||
spDriveId: project.source.driveId,
|
||||
spRootItemId: project.source.rootItemId,
|
||||
spDisplayPath: project.sourceDisplayPath,
|
||||
accHubId: project.target.hubId,
|
||||
accProjectId: project.target.projectId,
|
||||
accRootFolderId: project.target.rootFolderId,
|
||||
accDisplayPath: project.targetDisplayPath,
|
||||
createdAt: this.clock()
|
||||
})
|
||||
.returning()
|
||||
.get();
|
||||
return toProject(row);
|
||||
}
|
||||
|
||||
async pause(id: string, reason: string): Promise<void> {
|
||||
this.update(id, { pausedReason: reason });
|
||||
}
|
||||
|
||||
async resume(id: string): Promise<void> {
|
||||
this.update(id, { pausedReason: null });
|
||||
}
|
||||
|
||||
async markFirstSynced(id: string, at: Date): Promise<void> {
|
||||
this.update(id, { firstSyncedAt: at });
|
||||
}
|
||||
|
||||
async setLastRunAt(id: string, at: Date): Promise<void> {
|
||||
this.update(id, { lastRunAt: at });
|
||||
}
|
||||
|
||||
async reset(id: string): Promise<void> {
|
||||
this.db.transaction((tx) => {
|
||||
tx.delete(items).where(eq(items.projectId, id)).run();
|
||||
tx.update(projects)
|
||||
.set({ deltaCursor: null, firstSyncedAt: null, pausedReason: null })
|
||||
.where(eq(projects.id, id))
|
||||
.run();
|
||||
});
|
||||
}
|
||||
|
||||
private update(id: string, values: Partial<ProjectRow>): void {
|
||||
this.db.update(projects).set(values).where(eq(projects.id, id)).run();
|
||||
}
|
||||
}
|
||||
|
||||
function toProject(row: ProjectRow): Project {
|
||||
return {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
enabled: row.enabled,
|
||||
pausedReason: row.pausedReason,
|
||||
source: { siteId: row.spSiteId, driveId: row.spDriveId, rootItemId: row.spRootItemId },
|
||||
sourceDisplayPath: row.spDisplayPath,
|
||||
target: {
|
||||
hubId: row.accHubId,
|
||||
projectId: row.accProjectId,
|
||||
rootFolderId: row.accRootFolderId
|
||||
},
|
||||
targetDisplayPath: row.accDisplayPath,
|
||||
deltaCursor: row.deltaCursor,
|
||||
firstSyncedAt: row.firstSyncedAt,
|
||||
lastRunAt: row.lastRunAt,
|
||||
createdAt: row.createdAt
|
||||
};
|
||||
}
|
||||
28
src/lib/server/adapters/sqlite/run-repository.spec.ts
Normal file
28
src/lib/server/adapters/sqlite/run-repository.spec.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { createHarness } from '../../sync/testing/harness';
|
||||
|
||||
describe('SqliteRunRepository', () => {
|
||||
it('prunes old runs together with their events', async () => {
|
||||
const h = await createHarness();
|
||||
h.drive.addFile(h.rootId, 'a.txt');
|
||||
const old = await h.sync();
|
||||
const cutoff = new Date(Date.UTC(2026, 9, 2));
|
||||
const recent = await h.runs.start(h.project.id, 'manual', new Date(Date.UTC(2026, 9, 3)));
|
||||
|
||||
expect(await h.runs.pruneBefore(cutoff)).toBe(1);
|
||||
|
||||
expect((await h.runs.listSince(h.project.id, new Date(0))).map((r) => r.runId)).toEqual([
|
||||
recent
|
||||
]);
|
||||
expect(await h.runs.listEvents(old.runId)).toEqual([]);
|
||||
});
|
||||
|
||||
it('reports an unfinished run with no outcome', async () => {
|
||||
const h = await createHarness();
|
||||
const runId = await h.runs.start(h.project.id, 'manual', new Date());
|
||||
|
||||
const [run] = await h.runs.listSince(h.project.id, new Date(0));
|
||||
|
||||
expect(run).toMatchObject({ runId, outcome: null, finishedAt: null });
|
||||
});
|
||||
});
|
||||
71
src/lib/server/adapters/sqlite/run-repository.ts
Normal file
71
src/lib/server/adapters/sqlite/run-repository.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
import { and, asc, desc, eq, gte, lt } from 'drizzle-orm';
|
||||
import type { RunCounts, RunEvent, RunOutcome, RunRecord, RunTrigger } from '../../sync/core/model';
|
||||
import type { RunRepository } from '../../sync/core/ports';
|
||||
import type { Db } from './database';
|
||||
import { runEvents, runs } from './schema';
|
||||
|
||||
type RunRow = typeof runs.$inferSelect;
|
||||
|
||||
export class SqliteRunRepository implements RunRepository {
|
||||
constructor(private readonly db: Db) {}
|
||||
|
||||
async start(projectId: string, trigger: RunTrigger, at: Date): Promise<string> {
|
||||
const id = crypto.randomUUID();
|
||||
this.db.insert(runs).values({ id, projectId, trigger, startedAt: at }).run();
|
||||
return id;
|
||||
}
|
||||
|
||||
async finish(
|
||||
runId: string,
|
||||
result: { finishedAt: Date; outcome: RunOutcome; error: string | null; counts: RunCounts }
|
||||
): Promise<void> {
|
||||
const { finishedAt, outcome, error, counts } = result;
|
||||
this.db
|
||||
.update(runs)
|
||||
.set({ finishedAt, outcome, error, ...counts })
|
||||
.where(eq(runs.id, runId))
|
||||
.run();
|
||||
}
|
||||
|
||||
async addEvent(event: RunEvent): Promise<void> {
|
||||
this.db.insert(runEvents).values(event).run();
|
||||
}
|
||||
|
||||
async listSince(projectId: string, since: Date): Promise<RunRecord[]> {
|
||||
return this.db
|
||||
.select()
|
||||
.from(runs)
|
||||
.where(and(eq(runs.projectId, projectId), gte(runs.startedAt, since)))
|
||||
.orderBy(desc(runs.startedAt))
|
||||
.all()
|
||||
.map(toRunRecord);
|
||||
}
|
||||
|
||||
async listEvents(runId: string): Promise<RunEvent[]> {
|
||||
return this.db
|
||||
.select()
|
||||
.from(runEvents)
|
||||
.where(eq(runEvents.runId, runId))
|
||||
.orderBy(asc(runEvents.id))
|
||||
.all()
|
||||
.map(({ id, ...event }) => event);
|
||||
}
|
||||
|
||||
async pruneBefore(cutoff: Date): Promise<number> {
|
||||
return this.db.delete(runs).where(lt(runs.startedAt, cutoff)).run().changes;
|
||||
}
|
||||
}
|
||||
|
||||
function toRunRecord(row: RunRow): RunRecord {
|
||||
const { id, projectId, trigger, startedAt, finishedAt, outcome, error, ...counts } = row;
|
||||
return {
|
||||
runId: id,
|
||||
projectId,
|
||||
trigger,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
outcome,
|
||||
error,
|
||||
counts
|
||||
};
|
||||
}
|
||||
99
src/lib/server/adapters/sqlite/schema.ts
Normal file
99
src/lib/server/adapters/sqlite/schema.ts
Normal file
@@ -0,0 +1,99 @@
|
||||
import { index, integer, primaryKey, sqliteTable, text } from 'drizzle-orm/sqlite-core';
|
||||
import { EVENT_ACTIONS } from '../../sync/core/model';
|
||||
|
||||
const timestamp = (name: string) => integer(name, { mode: 'timestamp_ms' });
|
||||
const flag = (name: string) => integer(name, { mode: 'boolean' });
|
||||
const count = (name: string) => integer(name).notNull().default(0);
|
||||
|
||||
export const projects = sqliteTable('projects', {
|
||||
id: text('id').primaryKey(),
|
||||
name: text('name').notNull(),
|
||||
enabled: flag('enabled').notNull().default(true),
|
||||
pausedReason: text('paused_reason'),
|
||||
spSiteId: text('sp_site_id').notNull(),
|
||||
spDriveId: text('sp_drive_id').notNull(),
|
||||
spRootItemId: text('sp_root_item_id').notNull(),
|
||||
spDisplayPath: text('sp_display_path').notNull(),
|
||||
accHubId: text('acc_hub_id').notNull(),
|
||||
accProjectId: text('acc_project_id').notNull(),
|
||||
accRootFolderId: text('acc_root_folder_id').notNull(),
|
||||
accDisplayPath: text('acc_display_path').notNull(),
|
||||
deltaCursor: text('delta_cursor'),
|
||||
firstSyncedAt: timestamp('first_synced_at'),
|
||||
lastRunAt: timestamp('last_run_at'),
|
||||
createdAt: timestamp('created_at').notNull()
|
||||
});
|
||||
|
||||
export const items = sqliteTable(
|
||||
'items',
|
||||
{
|
||||
projectId: text('project_id')
|
||||
.notNull()
|
||||
.references(() => projects.id, { onDelete: 'cascade' }),
|
||||
sourceId: text('source_id').notNull(),
|
||||
// desired (from SharePoint)
|
||||
parentSourceId: text('parent_source_id').notNull(),
|
||||
name: text('name').notNull(),
|
||||
isFolder: flag('is_folder').notNull(),
|
||||
contentTag: text('content_tag'),
|
||||
sourceModifiedAt: timestamp('source_modified_at').notNull(),
|
||||
size: integer('size'),
|
||||
deleted: flag('deleted').notNull(),
|
||||
// synced (what ACC has)
|
||||
targetId: text('target_id'),
|
||||
syncedParentSourceId: text('synced_parent_source_id'),
|
||||
syncedName: text('synced_name'),
|
||||
syncedContentTag: text('synced_content_tag'),
|
||||
syncedDeleted: flag('synced_deleted').notNull(),
|
||||
syncedAt: timestamp('synced_at'),
|
||||
// bookkeeping
|
||||
status: text('status', { enum: ['pending', 'synced', 'failed', 'deleted'] }).notNull(),
|
||||
attemptCount: count('attempt_count'),
|
||||
lastError: text('last_error'),
|
||||
lastAttemptAt: timestamp('last_attempt_at')
|
||||
},
|
||||
(t) => [
|
||||
primaryKey({ columns: [t.projectId, t.sourceId] }),
|
||||
index('items_project_status').on(t.projectId, t.status)
|
||||
]
|
||||
);
|
||||
|
||||
export const runs = sqliteTable(
|
||||
'runs',
|
||||
{
|
||||
id: text('id').primaryKey(),
|
||||
projectId: text('project_id')
|
||||
.notNull()
|
||||
.references(() => projects.id, { onDelete: 'cascade' }),
|
||||
trigger: text('trigger', { enum: ['schedule', 'manual'] }).notNull(),
|
||||
startedAt: timestamp('started_at').notNull(),
|
||||
finishedAt: timestamp('finished_at'),
|
||||
outcome: text('outcome', { enum: ['success', 'partial', 'failed'] }),
|
||||
error: text('error'),
|
||||
created: count('created'),
|
||||
updated: count('updated'),
|
||||
renamed: count('renamed'),
|
||||
moved: count('moved'),
|
||||
deleted: count('deleted'),
|
||||
restored: count('restored'),
|
||||
failed: count('failed')
|
||||
},
|
||||
(t) => [index('runs_project_started').on(t.projectId, t.startedAt)]
|
||||
);
|
||||
|
||||
export const runEvents = sqliteTable(
|
||||
'run_events',
|
||||
{
|
||||
id: integer('id').primaryKey({ autoIncrement: true }),
|
||||
runId: text('run_id')
|
||||
.notNull()
|
||||
.references(() => runs.id, { onDelete: 'cascade' }),
|
||||
projectId: text('project_id').notNull(),
|
||||
sourceId: text('source_id').notNull(),
|
||||
path: text('path').notNull(),
|
||||
action: text('action', { enum: EVENT_ACTIONS }).notNull(),
|
||||
message: text('message'),
|
||||
at: timestamp('at').notNull()
|
||||
},
|
||||
(t) => [index('run_events_run').on(t.runId)]
|
||||
);
|
||||
115
src/lib/server/sync/coordinator.spec.ts
Normal file
115
src/lib/server/sync/coordinator.spec.ts
Normal file
@@ -0,0 +1,115 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type { ProjectSyncRunner } from './core/engine';
|
||||
import { emptyCounts, type Project, type RunTrigger } from './core/model';
|
||||
import type { ProjectRepository } from './core/ports';
|
||||
import { SyncCoordinator } from './coordinator';
|
||||
import { silentLogger } from './testing/harness';
|
||||
|
||||
function project(id: string, overrides: Partial<Project> = {}): Project {
|
||||
return { id, enabled: true, pausedReason: null, ...overrides } as Project;
|
||||
}
|
||||
|
||||
/** A runner whose runs stay in flight until `finish` is called. */
|
||||
class ControlledRunner implements ProjectSyncRunner {
|
||||
readonly started: Array<[string, RunTrigger]> = [];
|
||||
private readonly pending: Array<() => void> = [];
|
||||
|
||||
runProject(projectId: string, trigger: RunTrigger) {
|
||||
this.started.push([projectId, trigger]);
|
||||
return new Promise<never>((resolve) => {
|
||||
this.pending.push(() =>
|
||||
resolve({ runId: 'r', outcome: 'success', error: null, counts: emptyCounts() } as never)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
finishAll() {
|
||||
this.pending.splice(0).forEach((finish) => finish());
|
||||
}
|
||||
}
|
||||
|
||||
function setup(projects: Project[]) {
|
||||
let now = 0;
|
||||
const runner = new ControlledRunner();
|
||||
const repo = {
|
||||
get: async (id: string) => projects.find((p) => p.id === id) ?? null,
|
||||
list: async () => projects
|
||||
} as ProjectRepository;
|
||||
const coordinator = new SyncCoordinator({
|
||||
runner,
|
||||
projects: repo,
|
||||
clock: () => new Date(now),
|
||||
logger: silentLogger,
|
||||
manualCooldownMs: 30_000
|
||||
});
|
||||
return { runner, coordinator, advance: (ms: number) => (now += ms) };
|
||||
}
|
||||
|
||||
describe('SyncCoordinator', () => {
|
||||
it('starts a manual run and reports it as already running on a second click', async () => {
|
||||
const { runner, coordinator } = setup([project('p1')]);
|
||||
|
||||
expect(await coordinator.requestManualRun('p1')).toEqual({ status: 'started' });
|
||||
expect(await coordinator.requestManualRun('p1')).toEqual({ status: 'already-running' });
|
||||
expect(runner.started).toEqual([['p1', 'manual']]);
|
||||
});
|
||||
|
||||
it('enforces the manual cooldown after a run finishes', async () => {
|
||||
const { runner, coordinator, advance } = setup([project('p1')]);
|
||||
await coordinator.requestManualRun('p1');
|
||||
runner.finishAll();
|
||||
await coordinator.drain();
|
||||
|
||||
advance(10_000);
|
||||
expect(await coordinator.requestManualRun('p1')).toEqual({
|
||||
status: 'cooldown',
|
||||
retryAfterSeconds: 20
|
||||
});
|
||||
|
||||
advance(20_000);
|
||||
expect(await coordinator.requestManualRun('p1')).toEqual({ status: 'started' });
|
||||
});
|
||||
|
||||
it('keeps cooldowns per project', async () => {
|
||||
const { coordinator } = setup([project('p1'), project('p2')]);
|
||||
await coordinator.requestManualRun('p1');
|
||||
expect(await coordinator.requestManualRun('p2')).toEqual({ status: 'started' });
|
||||
});
|
||||
|
||||
it('refuses manual runs for disabled, paused or unknown projects', async () => {
|
||||
const { coordinator } = setup([
|
||||
project('off', { enabled: false }),
|
||||
project('paused', { pausedReason: 'Root deleted' })
|
||||
]);
|
||||
|
||||
expect(await coordinator.requestManualRun('off')).toMatchObject({ status: 'unavailable' });
|
||||
expect(await coordinator.requestManualRun('paused')).toEqual({
|
||||
status: 'unavailable',
|
||||
reason: 'Root deleted'
|
||||
});
|
||||
expect(await coordinator.requestManualRun('nope')).toMatchObject({ status: 'unavailable' });
|
||||
});
|
||||
|
||||
it('runs scheduled passes sequentially, skipping busy, disabled and paused projects', async () => {
|
||||
const { runner, coordinator } = setup([
|
||||
project('busy'),
|
||||
project('off', { enabled: false }),
|
||||
project('paused', { pausedReason: 'x' }),
|
||||
project('a'),
|
||||
project('b')
|
||||
]);
|
||||
await coordinator.requestManualRun('busy');
|
||||
|
||||
const pass = coordinator.runScheduledPass();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
expect(runner.started.map(([id]) => id)).toEqual(['busy', 'a']);
|
||||
|
||||
runner.finishAll();
|
||||
await new Promise((r) => setTimeout(r));
|
||||
expect(runner.started.map(([id]) => id)).toEqual(['busy', 'a', 'b']);
|
||||
|
||||
runner.finishAll();
|
||||
await pass;
|
||||
});
|
||||
});
|
||||
79
src/lib/server/sync/coordinator.ts
Normal file
79
src/lib/server/sync/coordinator.ts
Normal file
@@ -0,0 +1,79 @@
|
||||
import type { ProjectSyncRunner } from './core/engine';
|
||||
import { errorMessage } from './core/errors';
|
||||
import type { RunTrigger } from './core/model';
|
||||
import type { Clock, Logger, ProjectRepository } from './core/ports';
|
||||
|
||||
export type RunRequestResult =
|
||||
| { status: 'started' }
|
||||
| { status: 'already-running' }
|
||||
| { status: 'cooldown'; retryAfterSeconds: number }
|
||||
| { status: 'unavailable'; reason: string };
|
||||
|
||||
export interface SyncCoordinatorDeps {
|
||||
runner: ProjectSyncRunner;
|
||||
projects: ProjectRepository;
|
||||
clock: Clock;
|
||||
logger: Logger;
|
||||
manualCooldownMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The only way a sync run starts. Guarantees at most one run per project at a
|
||||
* time and rate-limits manual "Run now" requests per project.
|
||||
*/
|
||||
export class SyncCoordinator {
|
||||
private readonly active = new Map<string, Promise<void>>();
|
||||
private readonly lastManualRequest = new Map<string, number>();
|
||||
|
||||
constructor(private readonly deps: SyncCoordinatorDeps) {}
|
||||
|
||||
isRunning(projectId: string): boolean {
|
||||
return this.active.has(projectId);
|
||||
}
|
||||
|
||||
async requestManualRun(projectId: string): Promise<RunRequestResult> {
|
||||
const project = await this.deps.projects.get(projectId);
|
||||
if (!project) return { status: 'unavailable', reason: 'Project not found.' };
|
||||
if (!project.enabled) return { status: 'unavailable', reason: 'Project is disabled.' };
|
||||
if (project.pausedReason) return { status: 'unavailable', reason: project.pausedReason };
|
||||
if (this.isRunning(projectId)) return { status: 'already-running' };
|
||||
|
||||
const now = this.deps.clock().getTime();
|
||||
const elapsed = now - (this.lastManualRequest.get(projectId) ?? -Infinity);
|
||||
if (elapsed < this.deps.manualCooldownMs) {
|
||||
return {
|
||||
status: 'cooldown',
|
||||
retryAfterSeconds: Math.ceil((this.deps.manualCooldownMs - elapsed) / 1000)
|
||||
};
|
||||
}
|
||||
|
||||
this.lastManualRequest.set(projectId, now);
|
||||
void this.launch(projectId, 'manual');
|
||||
return { status: 'started' };
|
||||
}
|
||||
|
||||
/** Syncs every enabled, unpaused, idle project, one after another. */
|
||||
async runScheduledPass(): Promise<void> {
|
||||
for (const project of await this.deps.projects.list()) {
|
||||
if (!project.enabled || project.pausedReason || this.isRunning(project.id)) continue;
|
||||
await this.launch(project.id, 'schedule');
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolves once every in-flight run has finished. */
|
||||
async drain(): Promise<void> {
|
||||
await Promise.all(this.active.values());
|
||||
}
|
||||
|
||||
private launch(projectId: string, trigger: RunTrigger): Promise<void> {
|
||||
const run = this.deps.runner
|
||||
.runProject(projectId, trigger)
|
||||
.then(() => undefined)
|
||||
.catch((error) => {
|
||||
this.deps.logger.error('Sync run crashed', { projectId, error: errorMessage(error) });
|
||||
})
|
||||
.finally(() => this.active.delete(projectId));
|
||||
this.active.set(projectId, run);
|
||||
return run;
|
||||
}
|
||||
}
|
||||
243
src/lib/server/sync/core/engine.spec.ts
Normal file
243
src/lib/server/sync/core/engine.spec.ts
Normal file
@@ -0,0 +1,243 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { createHarness } from '../testing/harness';
|
||||
import { ServiceUnavailableError } from './errors';
|
||||
import { ROOT_DELETED } from './ingest';
|
||||
import { WAITING_FOR_PARENT } from './reconciler';
|
||||
|
||||
describe('SyncEngine', () => {
|
||||
it('mirrors the initial tree and ignores temp files', async () => {
|
||||
const h = await createHarness();
|
||||
const drawings = h.drive.addFolder(h.rootId, 'Drawings');
|
||||
h.drive.addFile(drawings, 'plan.pdf');
|
||||
h.drive.addFile(drawings, '~$plan.docx');
|
||||
h.drive.addFile(h.rootId, 'notes.txt');
|
||||
h.drive.addFile(h.drive.driveRootId, 'outside-root.txt');
|
||||
|
||||
const result = await h.sync();
|
||||
|
||||
expect(result.outcome).toBe('success');
|
||||
expect(result.counts.created).toBe(3);
|
||||
expect(h.acc.tree()).toEqual(['Drawings/', 'Drawings/plan.pdf@1', 'notes.txt@1']);
|
||||
});
|
||||
|
||||
it('uploads edits as new versions', async () => {
|
||||
const h = await createHarness();
|
||||
const file = h.drive.addFile(h.rootId, 'spec.docx');
|
||||
await h.sync();
|
||||
|
||||
h.drive.edit(file, 'spec v2');
|
||||
const result = await h.sync();
|
||||
|
||||
expect(result.counts.updated).toBe(1);
|
||||
expect(h.acc.tree()).toEqual(['spec.docx@2']);
|
||||
expect(h.acc.latestContent('spec.docx')).toBe('spec v2');
|
||||
});
|
||||
|
||||
it('applies a move and rename of the same file', async () => {
|
||||
const h = await createHarness();
|
||||
const a = h.drive.addFolder(h.rootId, 'A');
|
||||
const b = h.drive.addFolder(h.rootId, 'B');
|
||||
const file = h.drive.addFile(a, 'old.txt');
|
||||
await h.sync();
|
||||
|
||||
h.drive.move(file, b);
|
||||
h.drive.rename(file, 'new.txt');
|
||||
const result = await h.sync();
|
||||
|
||||
expect(result.counts).toMatchObject({ moved: 1, renamed: 1, created: 0 });
|
||||
expect(h.acc.tree()).toEqual(['A/', 'B/', 'B/new.txt@1']);
|
||||
});
|
||||
|
||||
it('creates a folder before files moved into it', async () => {
|
||||
const h = await createHarness();
|
||||
const file = h.drive.addFile(h.rootId, 'report.pdf');
|
||||
await h.sync();
|
||||
|
||||
const archive = h.drive.addFolder(h.rootId, 'Archive');
|
||||
h.drive.move(file, archive);
|
||||
await h.sync();
|
||||
|
||||
expect(h.acc.tree()).toEqual(['Archive/', 'Archive/report.pdf@1']);
|
||||
});
|
||||
|
||||
it('deletes a folder with a single ACC call that covers its contents', async () => {
|
||||
const h = await createHarness();
|
||||
const folder = h.drive.addFolder(h.rootId, 'Old');
|
||||
h.drive.addFile(folder, 'a.txt');
|
||||
h.drive.addFile(folder, 'b.txt');
|
||||
await h.sync();
|
||||
|
||||
h.drive.remove(folder);
|
||||
const result = await h.sync();
|
||||
|
||||
expect(result.counts.deleted).toBe(1);
|
||||
expect(h.acc.calls.filter((c) => c.startsWith('delete'))).toEqual(['delete Old']);
|
||||
expect(h.acc.tree()).toEqual([]);
|
||||
expect((await h.itemNamed('a.txt')).status).toBe('deleted');
|
||||
});
|
||||
|
||||
it('treats items moved out of the root as deleted', async () => {
|
||||
const h = await createHarness();
|
||||
const file = h.drive.addFile(h.rootId, 'leaving.txt');
|
||||
await h.sync();
|
||||
|
||||
h.drive.move(file, h.drive.driveRootId);
|
||||
await h.sync();
|
||||
|
||||
expect(h.acc.tree()).toEqual([]);
|
||||
});
|
||||
|
||||
it('treats a rename to an ignored name as a delete', async () => {
|
||||
const h = await createHarness();
|
||||
const file = h.drive.addFile(h.rootId, 'draft.docx');
|
||||
await h.sync();
|
||||
|
||||
h.drive.rename(file, 'draft.tmp');
|
||||
await h.sync();
|
||||
|
||||
expect(h.acc.tree()).toEqual([]);
|
||||
});
|
||||
|
||||
it('revives the same ACC items when a deleted folder is restored', async () => {
|
||||
const h = await createHarness();
|
||||
const folder = h.drive.addFolder(h.rootId, 'Specs');
|
||||
h.drive.addFile(folder, 'spec.docx');
|
||||
await h.sync();
|
||||
const before = await h.itemNamed('spec.docx');
|
||||
|
||||
h.drive.remove(folder);
|
||||
await h.sync();
|
||||
h.drive.restore(folder);
|
||||
const result = await h.sync();
|
||||
|
||||
expect(result.counts.restored).toBe(2);
|
||||
expect(result.counts.created).toBe(0);
|
||||
expect(h.acc.tree()).toEqual(['Specs/', 'Specs/spec.docx@1']);
|
||||
expect((await h.itemNamed('spec.docx')).synced.targetId).toBe(before.synced.targetId);
|
||||
});
|
||||
|
||||
it('never touches files added to ACC by hand', async () => {
|
||||
const h = await createHarness();
|
||||
const folder = h.drive.addFolder(h.rootId, 'Shared');
|
||||
await h.sync();
|
||||
const accFolder = (await h.itemNamed('Shared')).synced.targetId!;
|
||||
h.acc.addByHand(h.acc.rootFolderId, 'manual.pdf');
|
||||
h.acc.addByHand(accFolder, 'inside.pdf');
|
||||
|
||||
h.drive.addFile(h.rootId, 'synced.txt');
|
||||
h.drive.rename(folder, 'Shared Renamed');
|
||||
await h.sync();
|
||||
|
||||
expect(h.acc.tree()).toEqual([
|
||||
'Shared Renamed/',
|
||||
'Shared Renamed/inside.pdf@1',
|
||||
'manual.pdf@1',
|
||||
'synced.txt@1'
|
||||
]);
|
||||
});
|
||||
|
||||
describe('failures', () => {
|
||||
it('retries a failing item, then reports it after the last attempt', async () => {
|
||||
const h = await createHarness({ maxAttempts: 3 });
|
||||
h.drive.addFile(h.rootId, 'bad.dwg');
|
||||
h.drive.addFile(h.rootId, 'good.txt');
|
||||
h.acc.failOn('bad.dwg', new Error('Upload rejected'));
|
||||
|
||||
const first = await h.sync();
|
||||
expect(first.outcome).toBe('partial');
|
||||
expect((await h.itemNamed('bad.dwg')).status).toBe('pending');
|
||||
|
||||
await h.sync();
|
||||
await h.sync();
|
||||
const failed = await h.itemNamed('bad.dwg');
|
||||
expect(failed).toMatchObject({
|
||||
status: 'failed',
|
||||
attemptCount: 3,
|
||||
lastError: 'Upload rejected'
|
||||
});
|
||||
|
||||
const uploadsBefore = h.acc.calls.length;
|
||||
await h.sync();
|
||||
expect(h.acc.calls.length).toBe(uploadsBefore);
|
||||
expect(h.acc.tree()).toEqual(['good.txt@1']);
|
||||
});
|
||||
|
||||
it('gives a failed item fresh attempts when it changes in SharePoint', async () => {
|
||||
const h = await createHarness({ maxAttempts: 1 });
|
||||
const file = h.drive.addFile(h.rootId, 'bad.dwg');
|
||||
h.acc.failOn('bad.dwg', new Error('Upload rejected'), 1);
|
||||
await h.sync();
|
||||
expect((await h.itemNamed('bad.dwg')).status).toBe('failed');
|
||||
|
||||
h.drive.edit(file, 'fixed');
|
||||
await h.sync();
|
||||
|
||||
expect((await h.itemNamed('bad.dwg')).status).toBe('synced');
|
||||
expect(h.acc.latestContent('bad.dwg')).toBe('fixed');
|
||||
});
|
||||
|
||||
it('holds children without charging attempts while their folder fails', async () => {
|
||||
const h = await createHarness();
|
||||
const folder = h.drive.addFolder(h.rootId, 'Broken');
|
||||
h.drive.addFile(folder, 'child.txt');
|
||||
h.acc.failOn('Broken', new Error('Folder create failed'), 1);
|
||||
|
||||
await h.sync();
|
||||
expect(await h.itemNamed('child.txt')).toMatchObject({
|
||||
status: 'pending',
|
||||
attemptCount: 0,
|
||||
lastError: WAITING_FOR_PARENT
|
||||
});
|
||||
|
||||
await h.sync();
|
||||
expect(h.acc.tree()).toEqual(['Broken/', 'Broken/child.txt@1']);
|
||||
});
|
||||
|
||||
it('stops the run on an outage without charging attempts', async () => {
|
||||
const h = await createHarness();
|
||||
h.drive.addFile(h.rootId, 'a.txt');
|
||||
h.drive.addFile(h.rootId, 'b.txt');
|
||||
h.acc.failOn('a.txt', new ServiceUnavailableError('ACC is down'), 1);
|
||||
|
||||
const result = await h.sync();
|
||||
|
||||
expect(result).toMatchObject({ outcome: 'failed', error: 'ACC is down' });
|
||||
expect((await h.itemNamed('a.txt')).attemptCount).toBe(0);
|
||||
expect(h.acc.calls).toEqual(['uploadFile a.txt']);
|
||||
|
||||
await h.sync();
|
||||
expect(h.acc.tree()).toEqual(['a.txt@1', 'b.txt@1']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('root safety', () => {
|
||||
it('pauses instead of deleting when the root folder is deleted', async () => {
|
||||
const h = await createHarness();
|
||||
h.drive.addFile(h.rootId, 'keep.txt');
|
||||
await h.sync();
|
||||
|
||||
h.drive.remove(h.rootId);
|
||||
const result = await h.sync();
|
||||
|
||||
expect(result).toMatchObject({ outcome: 'failed', error: ROOT_DELETED });
|
||||
expect((await h.projects.get(h.project.id))!.pausedReason).toBe(ROOT_DELETED);
|
||||
expect(h.acc.tree()).toEqual(['keep.txt@1']);
|
||||
});
|
||||
});
|
||||
|
||||
it('records run history and per-file events', async () => {
|
||||
const h = await createHarness();
|
||||
const folder = h.drive.addFolder(h.rootId, 'Docs');
|
||||
h.drive.addFile(folder, 'a.txt');
|
||||
|
||||
const result = await h.sync();
|
||||
|
||||
const [run] = await h.runs.listSince(h.project.id, new Date(0));
|
||||
expect(run).toMatchObject({ runId: result.runId, trigger: 'schedule', outcome: 'success' });
|
||||
const events = await h.runs.listEvents(result.runId);
|
||||
expect(events.map((e) => `${e.action} ${e.path}`)).toEqual([
|
||||
'created Docs',
|
||||
'created Docs/a.txt'
|
||||
]);
|
||||
});
|
||||
});
|
||||
104
src/lib/server/sync/core/engine.ts
Normal file
104
src/lib/server/sync/core/engine.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import { errorMessage } from './errors';
|
||||
import { ingestChanges } from './ingest';
|
||||
import {
|
||||
emptyCounts,
|
||||
type ItemState,
|
||||
type Project,
|
||||
type RunOutcome,
|
||||
type RunResult,
|
||||
type RunTrigger
|
||||
} from './model';
|
||||
import type {
|
||||
ChangeSource,
|
||||
Clock,
|
||||
IgnorePolicy,
|
||||
ItemRepository,
|
||||
Logger,
|
||||
ProjectRepository,
|
||||
RunRepository
|
||||
} from './ports';
|
||||
import type { Reconciler } from './reconciler';
|
||||
|
||||
export interface ProjectSyncRunner {
|
||||
runProject(projectId: string, trigger: RunTrigger): Promise<RunResult>;
|
||||
}
|
||||
|
||||
export interface SyncEngineDeps {
|
||||
source: ChangeSource;
|
||||
projects: ProjectRepository;
|
||||
items: ItemRepository;
|
||||
runs: RunRepository;
|
||||
reconciler: Reconciler;
|
||||
ignore: IgnorePolicy;
|
||||
clock: Clock;
|
||||
logger: Logger;
|
||||
maxAttempts: number;
|
||||
}
|
||||
|
||||
/** Runs one sync of one project: ingest SharePoint changes, then reconcile ACC. */
|
||||
export class SyncEngine implements ProjectSyncRunner {
|
||||
constructor(private readonly deps: SyncEngineDeps) {}
|
||||
|
||||
async runProject(projectId: string, trigger: RunTrigger): Promise<RunResult> {
|
||||
const { projects, items, runs, clock, logger } = this.deps;
|
||||
const project = await projects.get(projectId);
|
||||
if (!project) throw new Error(`Project ${projectId} not found.`);
|
||||
|
||||
const runId = await runs.start(projectId, trigger, clock());
|
||||
let counts = emptyCounts();
|
||||
let outcome: RunOutcome = 'failed';
|
||||
let error: string | null;
|
||||
|
||||
try {
|
||||
const stored = await items.listByProject(projectId);
|
||||
const current = new Map(stored.map((item) => [item.sourceId, item]));
|
||||
const pausedReason = await this.ingest(project, current);
|
||||
|
||||
if (pausedReason) {
|
||||
await projects.pause(projectId, pausedReason);
|
||||
error = pausedReason;
|
||||
logger.warn('Project paused', { projectId, reason: pausedReason });
|
||||
} else {
|
||||
if (!project.firstSyncedAt) await projects.markFirstSynced(projectId, clock());
|
||||
const result = await this.deps.reconciler.reconcile(project, runId, current);
|
||||
counts = result.counts;
|
||||
error = result.abortedWith;
|
||||
outcome = result.abortedWith ? 'failed' : counts.failed > 0 ? 'partial' : 'success';
|
||||
}
|
||||
} catch (caught) {
|
||||
error = errorMessage(caught);
|
||||
logger.error('Sync run failed', { projectId, runId, error });
|
||||
}
|
||||
|
||||
const finishedAt = clock();
|
||||
await runs.finish(runId, { finishedAt, outcome, error, counts });
|
||||
await projects.setLastRunAt(projectId, finishedAt);
|
||||
return { runId, outcome, error, counts };
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads SharePoint changes into `current` and commits them with their cursor.
|
||||
* Follows up with one full listing when ingest asks for it. Returns a pause
|
||||
* reason if the batch was rejected.
|
||||
*/
|
||||
private async ingest(project: Project, current: Map<string, ItemState>): Promise<string | null> {
|
||||
let cursor = project.deltaCursor;
|
||||
for (;;) {
|
||||
const batch = await this.deps.source.getChanges(project.source, cursor);
|
||||
const result = ingestChanges({
|
||||
rootId: project.source.rootItemId,
|
||||
items: current,
|
||||
changes: batch.changes,
|
||||
fullSync: batch.fullSync,
|
||||
ignore: this.deps.ignore,
|
||||
maxAttempts: this.deps.maxAttempts
|
||||
});
|
||||
if (result.kind === 'paused') return result.reason;
|
||||
|
||||
await this.deps.items.commitIngest(project.id, result.changed, batch.cursor);
|
||||
for (const item of result.changed) current.set(item.sourceId, item);
|
||||
if (!result.needsFullListing) return null;
|
||||
cursor = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
15
src/lib/server/sync/core/errors.ts
Normal file
15
src/lib/server/sync/core/errors.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
/**
|
||||
* A remote service can't be used right now (outage, network, auth, throttling
|
||||
* that outlasted retries). Adapters throw this so the run stops early instead of
|
||||
* charging a failed attempt to every item it would otherwise touch.
|
||||
*/
|
||||
export class ServiceUnavailableError extends Error {
|
||||
constructor(message: string, options?: ErrorOptions) {
|
||||
super(message, options);
|
||||
this.name = 'ServiceUnavailableError';
|
||||
}
|
||||
}
|
||||
|
||||
export function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
26
src/lib/server/sync/core/ignore.spec.ts
Normal file
26
src/lib/server/sync/core/ignore.spec.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { PatternIgnorePolicy } from './ignore';
|
||||
|
||||
describe('PatternIgnorePolicy', () => {
|
||||
const policy = new PatternIgnorePolicy();
|
||||
|
||||
it.each([
|
||||
'~$Report.docx',
|
||||
'~WRL0003.tmp',
|
||||
'upload.TMP',
|
||||
'.~lock.plan.odt#',
|
||||
'thumbs.db',
|
||||
'.DS_Store'
|
||||
])('ignores %s', (name) => expect(policy.isIgnored(name)).toBe(true));
|
||||
|
||||
it.each(['Report.docx', 'tmp-notes.txt', 'plan.dwg', 'desktop.ini.bak'])('keeps %s', (name) =>
|
||||
expect(policy.isIgnored(name)).toBe(false)
|
||||
);
|
||||
|
||||
it('accepts custom patterns', () => {
|
||||
const custom = new PatternIgnorePolicy(['*.bak', 'draft?.txt']);
|
||||
expect(custom.isIgnored('a.bak')).toBe(true);
|
||||
expect(custom.isIgnored('draft1.txt')).toBe(true);
|
||||
expect(custom.isIgnored('draft10.txt')).toBe(false);
|
||||
});
|
||||
});
|
||||
35
src/lib/server/sync/core/ignore.ts
Normal file
35
src/lib/server/sync/core/ignore.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
import type { IgnorePolicy } from './ports';
|
||||
|
||||
/** Office lock files, temp/partial downloads and OS metadata. */
|
||||
export const DEFAULT_IGNORE_PATTERNS = [
|
||||
'~$*',
|
||||
'~*.tmp',
|
||||
'*.tmp',
|
||||
'.~lock.*#',
|
||||
'*.crdownload',
|
||||
'*.partial',
|
||||
'Thumbs.db',
|
||||
'desktop.ini',
|
||||
'.DS_Store'
|
||||
];
|
||||
|
||||
/** Matches names against case-insensitive `*` / `?` wildcard patterns. */
|
||||
export class PatternIgnorePolicy implements IgnorePolicy {
|
||||
private readonly patterns: RegExp[];
|
||||
|
||||
constructor(patterns: readonly string[] = DEFAULT_IGNORE_PATTERNS) {
|
||||
this.patterns = patterns.map(toRegExp);
|
||||
}
|
||||
|
||||
isIgnored(name: string): boolean {
|
||||
return this.patterns.some((pattern) => pattern.test(name));
|
||||
}
|
||||
}
|
||||
|
||||
function toRegExp(pattern: string): RegExp {
|
||||
const body = pattern
|
||||
.split('')
|
||||
.map((ch) => (ch === '*' ? '.*' : ch === '?' ? '.' : ch.replace(/[.+^${}()|[\]\\#]/g, '\\$&')))
|
||||
.join('');
|
||||
return new RegExp(`^${body}$`, 'i');
|
||||
}
|
||||
123
src/lib/server/sync/core/ingest.spec.ts
Normal file
123
src/lib/server/sync/core/ingest.spec.ts
Normal file
@@ -0,0 +1,123 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { PatternIgnorePolicy } from './ignore';
|
||||
import { ingestChanges, ROOT_MISSING, type IngestInput } from './ingest';
|
||||
import type { ItemState, SourceChange, SourceItem } from './model';
|
||||
|
||||
const ROOT = 'root';
|
||||
|
||||
function source(sourceId: string, parentSourceId: string, overrides: Partial<SourceItem> = {}) {
|
||||
return {
|
||||
sourceId,
|
||||
parentSourceId,
|
||||
name: sourceId,
|
||||
isFolder: false,
|
||||
contentTag: 'c1',
|
||||
modifiedAt: new Date(0),
|
||||
size: 1,
|
||||
...overrides
|
||||
} satisfies SourceItem;
|
||||
}
|
||||
|
||||
const upsert = (item: SourceItem): SourceChange => ({ kind: 'upsert', item });
|
||||
const folder = (id: string, parent: string) =>
|
||||
source(id, parent, { isFolder: true, contentTag: null });
|
||||
|
||||
function run(
|
||||
changes: SourceChange[],
|
||||
items: ItemState[] = [],
|
||||
overrides: Partial<IngestInput> = {}
|
||||
) {
|
||||
const result = ingestChanges({
|
||||
rootId: ROOT,
|
||||
items: new Map(items.map((i) => [i.sourceId, i])),
|
||||
changes,
|
||||
fullSync: false,
|
||||
ignore: new PatternIgnorePolicy(),
|
||||
maxAttempts: 3,
|
||||
...overrides
|
||||
});
|
||||
if (result.kind !== 'ok') throw new Error(result.reason);
|
||||
return result;
|
||||
}
|
||||
|
||||
const byId = (items: ItemState[]) => Object.fromEntries(items.map((i) => [i.sourceId, i]));
|
||||
|
||||
describe('ingestChanges', () => {
|
||||
it('accepts children reported before their parent', () => {
|
||||
const { changed } = run([upsert(source('file', 'dir')), upsert(folder('dir', ROOT))]);
|
||||
expect(changed.map((i) => i.sourceId).sort()).toEqual(['dir', 'file']);
|
||||
});
|
||||
|
||||
it('skips untracked items outside the root', () => {
|
||||
const { changed } = run([upsert(source('elsewhere', 'other-folder'))]);
|
||||
expect(changed).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps the last change when an item appears twice', () => {
|
||||
const { changed } = run([
|
||||
upsert(source('f', ROOT, { name: 'first' })),
|
||||
upsert(source('f', ROOT, { name: 'second' }))
|
||||
]);
|
||||
expect(changed[0].desired.name).toBe('second');
|
||||
});
|
||||
|
||||
it('skips everything under an ignored folder', () => {
|
||||
const ignore = { isIgnored: (name: string) => name === 'skip' };
|
||||
const { changed } = run([upsert(folder('skip', ROOT)), upsert(source('f', 'skip'))], [], {
|
||||
ignore
|
||||
});
|
||||
expect(changed).toEqual([]);
|
||||
});
|
||||
|
||||
it('cascades a folder delete to everything tracked beneath it', () => {
|
||||
const first = run([
|
||||
upsert(folder('a', ROOT)),
|
||||
upsert(folder('b', 'a')),
|
||||
upsert(source('f', 'b'))
|
||||
]).changed;
|
||||
const { changed } = run([{ kind: 'delete', sourceId: 'a' }], first);
|
||||
expect(changed.map((i) => [i.sourceId, i.desired.deleted]).sort()).toEqual([
|
||||
['a', true],
|
||||
['b', true],
|
||||
['f', true]
|
||||
]);
|
||||
});
|
||||
|
||||
it('deletes tracked items missing from a full listing', () => {
|
||||
const first = run([upsert(source('gone', ROOT)), upsert(source('kept', ROOT))]).changed;
|
||||
const { changed } = run([upsert(folder(ROOT, 'drive')), upsert(source('kept', ROOT))], first, {
|
||||
fullSync: true
|
||||
});
|
||||
expect(byId(changed).gone.desired.deleted).toBe(true);
|
||||
expect(byId(changed).kept).toBeUndefined();
|
||||
});
|
||||
|
||||
it('pauses when a full listing does not include the root', () => {
|
||||
const result = ingestChanges({
|
||||
rootId: ROOT,
|
||||
items: new Map(),
|
||||
changes: [],
|
||||
fullSync: true,
|
||||
ignore: new PatternIgnorePolicy(),
|
||||
maxAttempts: 3
|
||||
});
|
||||
expect(result).toEqual({ kind: 'paused', reason: ROOT_MISSING });
|
||||
});
|
||||
|
||||
it('keeps attempts for metadata-only changes but resets them for real ones', () => {
|
||||
const [tracked] = run([upsert(source('f', ROOT))]).changed;
|
||||
const failing = { ...tracked, attemptCount: 3, lastError: 'boom' };
|
||||
|
||||
const touched = run([upsert(source('f', ROOT, { modifiedAt: new Date(5) }))], [failing]);
|
||||
expect(touched.changed[0].attemptCount).toBe(3);
|
||||
|
||||
const edited = run([upsert(source('f', ROOT, { contentTag: 'c2' }))], [failing]);
|
||||
expect(edited.changed[0]).toMatchObject({ attemptCount: 0, lastError: null });
|
||||
});
|
||||
|
||||
it('asks for a full listing when a deleted folder comes back', () => {
|
||||
const first = run([upsert(folder('a', ROOT))]).changed;
|
||||
const deleted = run([{ kind: 'delete', sourceId: 'a' }], first).changed;
|
||||
expect(run([upsert(folder('a', ROOT))], deleted).needsFullListing).toBe(true);
|
||||
});
|
||||
});
|
||||
200
src/lib/server/sync/core/ingest.ts
Normal file
200
src/lib/server/sync/core/ingest.ts
Normal file
@@ -0,0 +1,200 @@
|
||||
import type { DesiredState, ItemState, SourceChange, SourceItem } from './model';
|
||||
import { withStatus } from './plan';
|
||||
import type { IgnorePolicy } from './ports';
|
||||
import { ItemTree } from './tree';
|
||||
|
||||
export const ROOT_DELETED = 'The SharePoint root folder was deleted.';
|
||||
export const ROOT_MISSING = 'The SharePoint root folder was not found in a full listing.';
|
||||
|
||||
export interface IngestInput {
|
||||
rootId: string;
|
||||
items: ReadonlyMap<string, ItemState>;
|
||||
changes: readonly SourceChange[];
|
||||
fullSync: boolean;
|
||||
ignore: IgnorePolicy;
|
||||
maxAttempts: number;
|
||||
}
|
||||
|
||||
export type IngestResult =
|
||||
| {
|
||||
kind: 'ok';
|
||||
changed: ItemState[];
|
||||
/**
|
||||
* A deleted folder came back. Its contents may not be reported individually,
|
||||
* so the caller should follow up with a full listing to revive them.
|
||||
*/
|
||||
needsFullListing: boolean;
|
||||
}
|
||||
| { kind: 'paused'; reason: string };
|
||||
|
||||
/**
|
||||
* Folds a batch of SharePoint changes into the desired state of the project's
|
||||
* items. Pure: returns only the items that changed and never touches ACC.
|
||||
*
|
||||
* Anything that leaves the root's scope (moved out, under an ignored or deleted
|
||||
* folder, renamed to an ignored name, or absent from a full listing) is treated
|
||||
* as deleted. If the root itself disappears the batch is rejected instead, so a
|
||||
* misconfiguration can never cascade into deleting everything in ACC.
|
||||
*/
|
||||
export function ingestChanges(input: IngestInput): IngestResult {
|
||||
const { rootId, changes, fullSync, ignore, maxAttempts } = input;
|
||||
|
||||
const latest = new Map<string, SourceChange>();
|
||||
for (const change of changes) latest.set(changeId(change), change);
|
||||
|
||||
const rootChange = latest.get(rootId);
|
||||
if (rootChange?.kind === 'delete') return { kind: 'paused', reason: ROOT_DELETED };
|
||||
if (fullSync && !rootChange) return { kind: 'paused', reason: ROOT_MISSING };
|
||||
latest.delete(rootId);
|
||||
|
||||
const working = new Map(input.items);
|
||||
const touched = new Set<string>();
|
||||
let revivedFolder = false;
|
||||
|
||||
const setDesired = (sourceId: string, desired: DesiredState) => {
|
||||
const current = working.get(sourceId);
|
||||
if (current && sameDesired(current.desired, desired)) return;
|
||||
const base = current ?? newItem(sourceId, desired);
|
||||
const resetAttempts = !current || actionableChange(current.desired, desired);
|
||||
if (current?.desired.deleted && !desired.deleted && desired.isFolder) revivedFolder = true;
|
||||
working.set(sourceId, {
|
||||
...base,
|
||||
desired,
|
||||
...(resetAttempts && { attemptCount: 0, lastError: null })
|
||||
});
|
||||
touched.add(sourceId);
|
||||
};
|
||||
|
||||
const markDeleted = (sourceId: string) => {
|
||||
const current = working.get(sourceId);
|
||||
if (current && !current.desired.deleted) {
|
||||
setDesired(sourceId, { ...current.desired, deleted: true });
|
||||
}
|
||||
};
|
||||
|
||||
const upserts = new Map<string, SourceItem>();
|
||||
for (const change of latest.values()) {
|
||||
if (change.kind === 'upsert') upserts.set(change.item.sourceId, change.item);
|
||||
else markDeleted(change.sourceId);
|
||||
}
|
||||
|
||||
if (fullSync) {
|
||||
for (const sourceId of input.items.keys()) if (!latest.has(sourceId)) markDeleted(sourceId);
|
||||
}
|
||||
|
||||
const inScope = scopeResolver(rootId, upserts, working, ignore);
|
||||
for (const item of upserts.values()) {
|
||||
if (inScope(item.sourceId)) setDesired(item.sourceId, toDesired(item));
|
||||
else markDeleted(item.sourceId);
|
||||
}
|
||||
|
||||
const tree = new ItemTree(rootId, working);
|
||||
for (const sourceId of [...touched]) {
|
||||
const item = working.get(sourceId)!;
|
||||
if (item.desired.deleted && item.desired.isFolder) {
|
||||
for (const descendant of tree.desiredDescendants(sourceId)) markDeleted(descendant.sourceId);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
kind: 'ok',
|
||||
changed: [...touched].map((id) => withStatus(working.get(id)!, maxAttempts)),
|
||||
needsFullListing: revivedFolder && !fullSync
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* An item is in scope when every ancestor up to the root is a live, non-ignored
|
||||
* folder. Newer information from this batch wins over stored state.
|
||||
*/
|
||||
function scopeResolver(
|
||||
rootId: string,
|
||||
upserts: ReadonlyMap<string, SourceItem>,
|
||||
known: ReadonlyMap<string, ItemState>,
|
||||
ignore: IgnorePolicy
|
||||
): (sourceId: string) => boolean {
|
||||
const memo = new Map<string, boolean>();
|
||||
|
||||
const resolve = (sourceId: string): boolean => {
|
||||
if (sourceId === rootId) return true;
|
||||
const cached = memo.get(sourceId);
|
||||
if (cached !== undefined) return cached;
|
||||
memo.set(sourceId, false); // guards against parent cycles
|
||||
|
||||
const fresh = upserts.get(sourceId);
|
||||
const stored = known.get(sourceId);
|
||||
let result: boolean;
|
||||
if (fresh) {
|
||||
result =
|
||||
!ignore.isIgnored(fresh.name, fresh.isFolder) &&
|
||||
fresh.parentSourceId !== null &&
|
||||
resolveParent(fresh.parentSourceId);
|
||||
} else {
|
||||
result = !!stored && !stored.desired.deleted && resolveParent(stored.desired.parentSourceId);
|
||||
}
|
||||
memo.set(sourceId, result);
|
||||
return result;
|
||||
};
|
||||
|
||||
const resolveParent = (parentId: string): boolean => {
|
||||
if (parentId === rootId) return true;
|
||||
const isFolder = upserts.get(parentId)?.isFolder ?? known.get(parentId)?.desired.isFolder;
|
||||
return isFolder === true && resolve(parentId);
|
||||
};
|
||||
|
||||
return resolve;
|
||||
}
|
||||
|
||||
function changeId(change: SourceChange): string {
|
||||
return change.kind === 'upsert' ? change.item.sourceId : change.sourceId;
|
||||
}
|
||||
|
||||
function toDesired(item: SourceItem): DesiredState {
|
||||
return {
|
||||
parentSourceId: item.parentSourceId!,
|
||||
name: item.name,
|
||||
isFolder: item.isFolder,
|
||||
contentTag: item.contentTag,
|
||||
modifiedAt: item.modifiedAt,
|
||||
size: item.size,
|
||||
deleted: false
|
||||
};
|
||||
}
|
||||
|
||||
function newItem(sourceId: string, desired: DesiredState): ItemState {
|
||||
return {
|
||||
sourceId,
|
||||
desired,
|
||||
synced: {
|
||||
targetId: null,
|
||||
parentSourceId: null,
|
||||
name: null,
|
||||
contentTag: null,
|
||||
deleted: false,
|
||||
at: null
|
||||
},
|
||||
status: 'pending',
|
||||
attemptCount: 0,
|
||||
lastError: null,
|
||||
lastAttemptAt: null
|
||||
};
|
||||
}
|
||||
|
||||
/** Changes that alter what needs doing in ACC (as opposed to display metadata). */
|
||||
function actionableChange(a: DesiredState, b: DesiredState): boolean {
|
||||
return (
|
||||
a.parentSourceId !== b.parentSourceId ||
|
||||
a.name !== b.name ||
|
||||
a.contentTag !== b.contentTag ||
|
||||
a.deleted !== b.deleted
|
||||
);
|
||||
}
|
||||
|
||||
function sameDesired(a: DesiredState, b: DesiredState): boolean {
|
||||
return (
|
||||
!actionableChange(a, b) &&
|
||||
a.isFolder === b.isFolder &&
|
||||
a.size === b.size &&
|
||||
a.modifiedAt.getTime() === b.modifiedAt.getTime()
|
||||
);
|
||||
}
|
||||
145
src/lib/server/sync/core/model.ts
Normal file
145
src/lib/server/sync/core/model.ts
Normal file
@@ -0,0 +1,145 @@
|
||||
/** Where a project's files come from in SharePoint (all Graph IDs). */
|
||||
export interface SourceRoot {
|
||||
siteId: string;
|
||||
driveId: string;
|
||||
/** The folder whose contents are mirrored. Never stored as an item itself. */
|
||||
rootItemId: string;
|
||||
}
|
||||
|
||||
/** Where a project's files go in ACC (all APS IDs). */
|
||||
export interface TargetRoot {
|
||||
hubId: string;
|
||||
projectId: string;
|
||||
rootFolderId: string;
|
||||
}
|
||||
|
||||
export interface Project {
|
||||
id: string;
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
/** Set when the sync refuses to continue (e.g. the SharePoint root vanished). */
|
||||
pausedReason: string | null;
|
||||
source: SourceRoot;
|
||||
sourceDisplayPath: string;
|
||||
target: TargetRoot;
|
||||
targetDisplayPath: string;
|
||||
deltaCursor: string | null;
|
||||
firstSyncedAt: Date | null;
|
||||
lastRunAt: Date | null;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export type NewProject = Pick<
|
||||
Project,
|
||||
'name' | 'source' | 'sourceDisplayPath' | 'target' | 'targetDisplayPath'
|
||||
>;
|
||||
|
||||
/** An item as SharePoint currently describes it. */
|
||||
export interface SourceItem {
|
||||
sourceId: string;
|
||||
parentSourceId: string | null;
|
||||
name: string;
|
||||
isFolder: boolean;
|
||||
/** Changes only when file content changes (Graph `cTag`). Null for folders. */
|
||||
contentTag: string | null;
|
||||
modifiedAt: Date;
|
||||
size: number | null;
|
||||
}
|
||||
|
||||
export type SourceChange =
|
||||
{ kind: 'upsert'; item: SourceItem } | { kind: 'delete'; sourceId: string };
|
||||
|
||||
export interface FileContent {
|
||||
size: number;
|
||||
stream: ReadableStream<Uint8Array>;
|
||||
}
|
||||
|
||||
export interface TargetRef {
|
||||
id: string;
|
||||
isFolder: boolean;
|
||||
}
|
||||
|
||||
/** What SharePoint says the item should look like. */
|
||||
export interface DesiredState {
|
||||
parentSourceId: string;
|
||||
name: string;
|
||||
isFolder: boolean;
|
||||
contentTag: string | null;
|
||||
modifiedAt: Date;
|
||||
size: number | null;
|
||||
deleted: boolean;
|
||||
}
|
||||
|
||||
/** What we last successfully wrote to ACC. */
|
||||
export interface SyncedState {
|
||||
targetId: string | null;
|
||||
parentSourceId: string | null;
|
||||
name: string | null;
|
||||
contentTag: string | null;
|
||||
deleted: boolean;
|
||||
at: Date | null;
|
||||
}
|
||||
|
||||
export type ItemStatus = 'pending' | 'synced' | 'failed' | 'deleted';
|
||||
|
||||
export interface ItemState {
|
||||
sourceId: string;
|
||||
desired: DesiredState;
|
||||
synced: SyncedState;
|
||||
status: ItemStatus;
|
||||
attemptCount: number;
|
||||
lastError: string | null;
|
||||
lastAttemptAt: Date | null;
|
||||
}
|
||||
|
||||
export type SyncAction =
|
||||
| { kind: 'create' }
|
||||
| { kind: 'restore' }
|
||||
| { kind: 'delete' }
|
||||
| { kind: 'update'; move: boolean; rename: boolean; content: boolean };
|
||||
|
||||
export type RunTrigger = 'schedule' | 'manual';
|
||||
export type RunOutcome = 'success' | 'partial' | 'failed';
|
||||
|
||||
export const EVENT_ACTIONS = [
|
||||
'created',
|
||||
'updated',
|
||||
'renamed',
|
||||
'moved',
|
||||
'deleted',
|
||||
'restored',
|
||||
'failed'
|
||||
] as const;
|
||||
export type EventAction = (typeof EVENT_ACTIONS)[number];
|
||||
|
||||
export type RunCounts = Record<EventAction, number>;
|
||||
|
||||
export function emptyCounts(): RunCounts {
|
||||
return Object.fromEntries(EVENT_ACTIONS.map((a) => [a, 0])) as RunCounts;
|
||||
}
|
||||
|
||||
export interface RunResult {
|
||||
runId: string;
|
||||
outcome: RunOutcome;
|
||||
error: string | null;
|
||||
counts: RunCounts;
|
||||
}
|
||||
|
||||
export interface RunRecord extends Omit<RunResult, 'outcome'> {
|
||||
/** Null while the run is still in progress. */
|
||||
outcome: RunOutcome | null;
|
||||
projectId: string;
|
||||
trigger: RunTrigger;
|
||||
startedAt: Date;
|
||||
finishedAt: Date | null;
|
||||
}
|
||||
|
||||
export interface RunEvent {
|
||||
runId: string;
|
||||
projectId: string;
|
||||
sourceId: string;
|
||||
path: string;
|
||||
action: EventAction;
|
||||
message: string | null;
|
||||
at: Date;
|
||||
}
|
||||
67
src/lib/server/sync/core/plan.spec.ts
Normal file
67
src/lib/server/sync/core/plan.spec.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type { DesiredState, ItemState, SyncedState } from './model';
|
||||
import { computeStatus, planAction } from './plan';
|
||||
|
||||
const desired: DesiredState = {
|
||||
parentSourceId: 'p',
|
||||
name: 'a.txt',
|
||||
isFolder: false,
|
||||
contentTag: 'c1',
|
||||
modifiedAt: new Date(0),
|
||||
size: 1,
|
||||
deleted: false
|
||||
};
|
||||
const synced: SyncedState = {
|
||||
targetId: 't',
|
||||
parentSourceId: 'p',
|
||||
name: 'a.txt',
|
||||
contentTag: 'c1',
|
||||
deleted: false,
|
||||
at: new Date(0)
|
||||
};
|
||||
|
||||
function item(d: Partial<DesiredState> = {}, s: Partial<SyncedState> = {}, attemptCount = 0) {
|
||||
return {
|
||||
sourceId: 'x',
|
||||
desired: { ...desired, ...d },
|
||||
synced: { ...synced, ...s },
|
||||
status: 'pending',
|
||||
attemptCount,
|
||||
lastError: null,
|
||||
lastAttemptAt: null
|
||||
} satisfies ItemState;
|
||||
}
|
||||
|
||||
describe('planAction', () => {
|
||||
it.each([
|
||||
['in sync', item(), null],
|
||||
['never synced', item({}, { targetId: null }), { kind: 'create' }],
|
||||
['deleted in ACC, alive in SharePoint', item({}, { deleted: true }), { kind: 'restore' }],
|
||||
['deleted in SharePoint', item({ deleted: true }), { kind: 'delete' }],
|
||||
['deleted before it was ever synced', item({ deleted: true }, { targetId: null }), null],
|
||||
['already deleted in both', item({ deleted: true }, { deleted: true }), null],
|
||||
[
|
||||
'moved, renamed and edited',
|
||||
item({ parentSourceId: 'q', name: 'b.txt', contentTag: 'c2' }),
|
||||
{ kind: 'update', move: true, rename: true, content: true }
|
||||
],
|
||||
[
|
||||
'folder (content tags ignored)',
|
||||
item({ isFolder: true, contentTag: null }, { contentTag: null }),
|
||||
null
|
||||
]
|
||||
])('%s', (_label, state, expected) => {
|
||||
expect(planAction(state)).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeStatus', () => {
|
||||
it.each([
|
||||
[item(), 'synced'],
|
||||
[item({ deleted: true }, { deleted: true }), 'deleted'],
|
||||
[item({ name: 'new' }, {}, 2), 'pending'],
|
||||
[item({ name: 'new' }, {}, 3), 'failed']
|
||||
] as const)('%#: %s', (state, expected) => {
|
||||
expect(computeStatus(state, 3)).toBe(expected);
|
||||
});
|
||||
});
|
||||
39
src/lib/server/sync/core/plan.ts
Normal file
39
src/lib/server/sync/core/plan.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import type { ItemState, ItemStatus, SyncAction } from './model';
|
||||
import type { ItemTree } from './tree';
|
||||
|
||||
/** The ACC operation needed to make the synced state match the desired state, if any. */
|
||||
export function planAction({ desired, synced }: ItemState): SyncAction | null {
|
||||
if (desired.deleted) return synced.targetId && !synced.deleted ? { kind: 'delete' } : null;
|
||||
if (!synced.targetId) return { kind: 'create' };
|
||||
if (synced.deleted) return { kind: 'restore' };
|
||||
|
||||
const move = desired.parentSourceId !== synced.parentSourceId;
|
||||
const rename = desired.name !== synced.name;
|
||||
const content = !desired.isFolder && desired.contentTag !== synced.contentTag;
|
||||
return move || rename || content ? { kind: 'update', move, rename, content } : null;
|
||||
}
|
||||
|
||||
export function computeStatus(item: ItemState, maxAttempts: number): ItemStatus {
|
||||
if (!planAction(item)) return item.desired.deleted ? 'deleted' : 'synced';
|
||||
return item.attemptCount >= maxAttempts ? 'failed' : 'pending';
|
||||
}
|
||||
|
||||
export function withStatus(item: ItemState, maxAttempts: number): ItemState {
|
||||
return { ...item, status: computeStatus(item, maxAttempts) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Items with outstanding work and attempts left, ordered so parents exist before
|
||||
* children: folders (shallow first), then files, then deletes (shallow first, so
|
||||
* a folder delete covers its contents).
|
||||
*/
|
||||
export function workQueue(
|
||||
items: Iterable<ItemState>,
|
||||
tree: ItemTree,
|
||||
maxAttempts: number
|
||||
): ItemState[] {
|
||||
const rank = (item: ItemState) => (item.desired.deleted ? 2 : item.desired.isFolder ? 0 : 1);
|
||||
return [...items]
|
||||
.filter((item) => computeStatus(item, maxAttempts) === 'pending')
|
||||
.sort((a, b) => rank(a) - rank(b) || tree.depth(a.sourceId) - tree.depth(b.sourceId));
|
||||
}
|
||||
93
src/lib/server/sync/core/ports.ts
Normal file
93
src/lib/server/sync/core/ports.ts
Normal file
@@ -0,0 +1,93 @@
|
||||
import type {
|
||||
FileContent,
|
||||
ItemState,
|
||||
NewProject,
|
||||
Project,
|
||||
RunEvent,
|
||||
RunOutcome,
|
||||
RunCounts,
|
||||
RunRecord,
|
||||
RunTrigger,
|
||||
SourceChange,
|
||||
SourceRoot,
|
||||
TargetRef
|
||||
} from './model';
|
||||
|
||||
export interface ChangeBatch {
|
||||
changes: SourceChange[];
|
||||
/** Cursor to pass next time. */
|
||||
cursor: string;
|
||||
/** True when this is a complete listing: anything tracked but absent has been removed. */
|
||||
fullSync: boolean;
|
||||
}
|
||||
|
||||
export interface ChangeSource {
|
||||
/** Changes since `cursor`; a null cursor (or an expired one) yields a full listing. */
|
||||
getChanges(root: SourceRoot, cursor: string | null): Promise<ChangeBatch>;
|
||||
download(root: SourceRoot, sourceId: string): Promise<FileContent>;
|
||||
}
|
||||
|
||||
/**
|
||||
* The destination document store. Implementations own any fallbacks they need
|
||||
* (e.g. move by copy + delete), so callers never branch on target quirks.
|
||||
*/
|
||||
export interface DocumentTarget {
|
||||
createFolder(parentId: string, name: string): Promise<TargetRef>;
|
||||
uploadFile(parentId: string, name: string, content: FileContent): Promise<TargetRef>;
|
||||
uploadVersion(ref: TargetRef, name: string, content: FileContent): Promise<void>;
|
||||
rename(ref: TargetRef, name: string): Promise<void>;
|
||||
/** May return a different ref if the move had to be done by copy + delete. */
|
||||
move(ref: TargetRef, newParentId: string): Promise<TargetRef>;
|
||||
delete(ref: TargetRef): Promise<void>;
|
||||
/** Bring back a deleted item, keeping its history where possible. */
|
||||
restore(
|
||||
ref: TargetRef,
|
||||
parentId: string,
|
||||
name: string,
|
||||
content?: FileContent
|
||||
): Promise<TargetRef>;
|
||||
}
|
||||
|
||||
export interface IgnorePolicy {
|
||||
isIgnored(name: string, isFolder: boolean): boolean;
|
||||
}
|
||||
|
||||
export interface ProjectRepository {
|
||||
get(id: string): Promise<Project | null>;
|
||||
list(): Promise<Project[]>;
|
||||
create(project: NewProject): Promise<Project>;
|
||||
pause(id: string, reason: string): Promise<void>;
|
||||
resume(id: string): Promise<void>;
|
||||
markFirstSynced(id: string, at: Date): Promise<void>;
|
||||
setLastRunAt(id: string, at: Date): Promise<void>;
|
||||
/** Forget all sync state so the project starts over from a full listing. */
|
||||
reset(id: string): Promise<void>;
|
||||
}
|
||||
|
||||
export interface ItemRepository {
|
||||
listByProject(projectId: string): Promise<ItemState[]>;
|
||||
/** Atomically store ingested changes and the cursor they were read up to. */
|
||||
commitIngest(projectId: string, items: ItemState[], cursor: string): Promise<void>;
|
||||
save(projectId: string, items: ItemState[]): Promise<void>;
|
||||
}
|
||||
|
||||
export interface RunRepository {
|
||||
start(projectId: string, trigger: RunTrigger, at: Date): Promise<string>;
|
||||
finish(
|
||||
runId: string,
|
||||
result: { finishedAt: Date; outcome: RunOutcome; error: string | null; counts: RunCounts }
|
||||
): Promise<void>;
|
||||
addEvent(event: RunEvent): Promise<void>;
|
||||
listSince(projectId: string, since: Date): Promise<RunRecord[]>;
|
||||
listEvents(runId: string): Promise<RunEvent[]>;
|
||||
/** Delete runs (and their events) started before `cutoff`; returns how many runs were removed. */
|
||||
pruneBefore(cutoff: Date): Promise<number>;
|
||||
}
|
||||
|
||||
export type Clock = () => Date;
|
||||
|
||||
export interface Logger {
|
||||
info(message: string, meta?: Record<string, unknown>): void;
|
||||
warn(message: string, meta?: Record<string, unknown>): void;
|
||||
error(message: string, meta?: Record<string, unknown>): void;
|
||||
}
|
||||
237
src/lib/server/sync/core/reconciler.ts
Normal file
237
src/lib/server/sync/core/reconciler.ts
Normal file
@@ -0,0 +1,237 @@
|
||||
import { errorMessage, ServiceUnavailableError } from './errors';
|
||||
import {
|
||||
emptyCounts,
|
||||
type EventAction,
|
||||
type ItemState,
|
||||
type Project,
|
||||
type RunCounts,
|
||||
type SyncAction,
|
||||
type SyncedState,
|
||||
type TargetRef
|
||||
} from './model';
|
||||
import { planAction, withStatus, workQueue } from './plan';
|
||||
import type { ChangeSource, Clock, DocumentTarget, ItemRepository, RunRepository } from './ports';
|
||||
import { ItemTree } from './tree';
|
||||
|
||||
export const WAITING_FOR_PARENT = 'Waiting for its parent folder to sync.';
|
||||
|
||||
export interface ReconcilerDeps {
|
||||
source: ChangeSource;
|
||||
target: DocumentTarget;
|
||||
items: ItemRepository;
|
||||
runs: RunRepository;
|
||||
clock: Clock;
|
||||
maxAttempts: number;
|
||||
}
|
||||
|
||||
export interface ReconcileResult {
|
||||
counts: RunCounts;
|
||||
/** Set when a service outage stopped the run early. */
|
||||
abortedWith: string | null;
|
||||
}
|
||||
|
||||
type RecordEvent = (action: EventAction, message?: string) => Promise<void>;
|
||||
|
||||
/** Pushes every pending item's desired state to ACC, one item at a time. */
|
||||
export class Reconciler {
|
||||
constructor(private readonly deps: ReconcilerDeps) {}
|
||||
|
||||
async reconcile(
|
||||
project: Project,
|
||||
runId: string,
|
||||
items: Map<string, ItemState>
|
||||
): Promise<ReconcileResult> {
|
||||
const { maxAttempts } = this.deps;
|
||||
const tree = new ItemTree(project.source.rootItemId, items);
|
||||
const counts = emptyCounts();
|
||||
|
||||
for (const { sourceId } of workQueue(items.values(), tree, maxAttempts)) {
|
||||
// Re-read: an earlier folder delete may already have covered this item.
|
||||
const item = items.get(sourceId)!;
|
||||
const action = planAction(item);
|
||||
if (!action) continue;
|
||||
|
||||
const record: RecordEvent = async (eventAction, message) => {
|
||||
counts[eventAction]++;
|
||||
await this.deps.runs.addEvent({
|
||||
runId,
|
||||
projectId: project.id,
|
||||
sourceId,
|
||||
path: tree.path(sourceId),
|
||||
action: eventAction,
|
||||
message: message ?? null,
|
||||
at: this.deps.clock()
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
await this.apply(project, items, tree, item, action, record);
|
||||
} catch (error) {
|
||||
if (error instanceof ServiceUnavailableError) {
|
||||
return { counts, abortedWith: error.message };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return { counts, abortedWith: null };
|
||||
}
|
||||
|
||||
private async apply(
|
||||
project: Project,
|
||||
items: Map<string, ItemState>,
|
||||
tree: ItemTree,
|
||||
item: ItemState,
|
||||
action: SyncAction,
|
||||
record: RecordEvent
|
||||
): Promise<void> {
|
||||
const parentTargetId = this.targetParentId(project, items, item);
|
||||
const needsParent = action.kind !== 'delete' && (action.kind !== 'update' || action.move);
|
||||
if (needsParent && !parentTargetId) {
|
||||
await this.store(project, items, [{ ...item, lastError: WAITING_FOR_PARENT }]);
|
||||
return;
|
||||
}
|
||||
|
||||
const synced: SyncedState = { ...item.synced };
|
||||
const now = this.deps.clock();
|
||||
try {
|
||||
await this.execute(project, item, action, parentTargetId!, synced, record);
|
||||
const done: ItemState = {
|
||||
...item,
|
||||
synced: { ...synced, at: now },
|
||||
attemptCount: 0,
|
||||
lastError: null,
|
||||
lastAttemptAt: now
|
||||
};
|
||||
await this.store(project, items, [done]);
|
||||
if (action.kind === 'delete' && item.desired.isFolder) {
|
||||
await this.coverDeletedContents(project, items, tree, item.sourceId);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof ServiceUnavailableError) {
|
||||
// Keep partial progress but don't charge an attempt for an outage.
|
||||
await this.store(project, items, [{ ...item, synced }]);
|
||||
throw error;
|
||||
}
|
||||
const message = errorMessage(error);
|
||||
await this.store(project, items, [
|
||||
{
|
||||
...item,
|
||||
synced,
|
||||
attemptCount: item.attemptCount + 1,
|
||||
lastError: message,
|
||||
lastAttemptAt: now
|
||||
}
|
||||
]);
|
||||
await record('failed', message);
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs the target calls for one action, recording progress into `synced` as each succeeds. */
|
||||
private async execute(
|
||||
project: Project,
|
||||
item: ItemState,
|
||||
action: SyncAction,
|
||||
parentTargetId: string,
|
||||
synced: SyncedState,
|
||||
record: RecordEvent
|
||||
): Promise<void> {
|
||||
const { target } = this.deps;
|
||||
const { desired } = item;
|
||||
const adopt = (ref: TargetRef) =>
|
||||
Object.assign(synced, {
|
||||
targetId: ref.id,
|
||||
parentSourceId: desired.parentSourceId,
|
||||
name: desired.name,
|
||||
contentTag: desired.contentTag,
|
||||
deleted: false
|
||||
});
|
||||
|
||||
switch (action.kind) {
|
||||
case 'create': {
|
||||
const ref = desired.isFolder
|
||||
? await target.createFolder(parentTargetId, desired.name)
|
||||
: await target.uploadFile(
|
||||
parentTargetId,
|
||||
desired.name,
|
||||
await this.download(project, item)
|
||||
);
|
||||
adopt(ref);
|
||||
return record('created');
|
||||
}
|
||||
case 'restore': {
|
||||
const stale = !desired.isFolder && desired.contentTag !== synced.contentTag;
|
||||
const content = stale ? await this.download(project, item) : undefined;
|
||||
adopt(await target.restore(refOf(item), parentTargetId, desired.name, content));
|
||||
return record('restored');
|
||||
}
|
||||
case 'delete': {
|
||||
await target.delete(refOf(item));
|
||||
synced.deleted = true;
|
||||
return record('deleted');
|
||||
}
|
||||
case 'update': {
|
||||
let ref = refOf(item);
|
||||
if (action.move) {
|
||||
ref = await target.move(ref, parentTargetId);
|
||||
Object.assign(synced, { targetId: ref.id, parentSourceId: desired.parentSourceId });
|
||||
await record('moved');
|
||||
}
|
||||
if (action.rename) {
|
||||
await target.rename(ref, desired.name);
|
||||
synced.name = desired.name;
|
||||
await record('renamed');
|
||||
}
|
||||
if (action.content) {
|
||||
await target.uploadVersion(ref, desired.name, await this.download(project, item));
|
||||
synced.contentTag = desired.contentTag;
|
||||
await record('updated');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Deleting a folder in ACC takes its contents with it; reflect that in their synced state. */
|
||||
private async coverDeletedContents(
|
||||
project: Project,
|
||||
items: Map<string, ItemState>,
|
||||
tree: ItemTree,
|
||||
folderId: string
|
||||
): Promise<void> {
|
||||
const covered = tree
|
||||
.syncedDescendants(folderId)
|
||||
.filter((d) => d.synced.targetId && !d.synced.deleted)
|
||||
.map((d) => ({ ...d, synced: { ...d.synced, deleted: true } }));
|
||||
await this.store(project, items, covered);
|
||||
}
|
||||
|
||||
private targetParentId(
|
||||
project: Project,
|
||||
items: ReadonlyMap<string, ItemState>,
|
||||
item: ItemState
|
||||
): string | null {
|
||||
const parentId = item.desired.parentSourceId;
|
||||
if (parentId === project.source.rootItemId) return project.target.rootFolderId;
|
||||
const parent = items.get(parentId);
|
||||
return parent?.synced.targetId && !parent.synced.deleted ? parent.synced.targetId : null;
|
||||
}
|
||||
|
||||
private download(project: Project, item: ItemState) {
|
||||
return this.deps.source.download(project.source, item.sourceId);
|
||||
}
|
||||
|
||||
/** Persists items with fresh statuses and mirrors them into the in-memory map. */
|
||||
private async store(
|
||||
project: Project,
|
||||
items: Map<string, ItemState>,
|
||||
updated: ItemState[]
|
||||
): Promise<void> {
|
||||
if (updated.length === 0) return;
|
||||
const withStatuses = updated.map((item) => withStatus(item, this.deps.maxAttempts));
|
||||
await this.deps.items.save(project.id, withStatuses);
|
||||
for (const item of withStatuses) items.set(item.sourceId, item);
|
||||
}
|
||||
}
|
||||
|
||||
function refOf(item: ItemState): TargetRef {
|
||||
return { id: item.synced.targetId!, isFolder: item.desired.isFolder };
|
||||
}
|
||||
73
src/lib/server/sync/core/tree.ts
Normal file
73
src/lib/server/sync/core/tree.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
import type { ItemState } from './model';
|
||||
|
||||
type ParentOf = (item: ItemState) => string | null;
|
||||
|
||||
const desiredParent: ParentOf = (item) => item.desired.parentSourceId;
|
||||
const syncedParent: ParentOf = (item) => item.synced.parentSourceId;
|
||||
|
||||
/**
|
||||
* Read-only view over a project's items. Reads the map live, so it reflects
|
||||
* entries replaced after construction.
|
||||
*/
|
||||
export class ItemTree {
|
||||
constructor(
|
||||
private readonly rootId: string,
|
||||
private readonly items: ReadonlyMap<string, ItemState>
|
||||
) {}
|
||||
|
||||
/** Desired ancestors, nearest first, stopping before the root. Cycle-safe. */
|
||||
ancestors(sourceId: string): ItemState[] {
|
||||
const result: ItemState[] = [];
|
||||
const seen = new Set([sourceId]);
|
||||
let parentId = this.items.get(sourceId)?.desired.parentSourceId;
|
||||
while (parentId && parentId !== this.rootId && !seen.has(parentId)) {
|
||||
const parent = this.items.get(parentId);
|
||||
if (!parent) break;
|
||||
result.push(parent);
|
||||
seen.add(parentId);
|
||||
parentId = parent.desired.parentSourceId;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
depth(sourceId: string): number {
|
||||
return this.ancestors(sourceId).length;
|
||||
}
|
||||
|
||||
/** Display path relative to the project root, e.g. `Drawings/Level 1/plan.pdf`. */
|
||||
path(sourceId: string): string {
|
||||
const item = this.items.get(sourceId);
|
||||
if (!item) return sourceId;
|
||||
return [...this.ancestors(sourceId).reverse(), item].map((i) => i.desired.name).join('/');
|
||||
}
|
||||
|
||||
/** Items under `folderId` according to SharePoint. */
|
||||
desiredDescendants(folderId: string): ItemState[] {
|
||||
return this.descendants(folderId, desiredParent);
|
||||
}
|
||||
|
||||
/** Items under `folderId` as they currently sit in ACC. */
|
||||
syncedDescendants(folderId: string): ItemState[] {
|
||||
return this.descendants(folderId, syncedParent);
|
||||
}
|
||||
|
||||
private descendants(folderId: string, parentOf: ParentOf): ItemState[] {
|
||||
const children = new Map<string, ItemState[]>();
|
||||
for (const item of this.items.values()) {
|
||||
const parentId = parentOf(item);
|
||||
if (parentId) children.set(parentId, [...(children.get(parentId) ?? []), item]);
|
||||
}
|
||||
const result: ItemState[] = [];
|
||||
const seen = new Set([folderId]);
|
||||
const queue = [folderId];
|
||||
while (queue.length > 0) {
|
||||
for (const child of children.get(queue.shift()!) ?? []) {
|
||||
if (seen.has(child.sourceId)) continue;
|
||||
seen.add(child.sourceId);
|
||||
result.push(child);
|
||||
queue.push(child.sourceId);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
38
src/lib/server/sync/scheduler.spec.ts
Normal file
38
src/lib/server/sync/scheduler.spec.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { IntervalScheduler } from './scheduler';
|
||||
import { silentLogger } from './testing/harness';
|
||||
|
||||
describe('IntervalScheduler', () => {
|
||||
beforeEach(() => vi.useFakeTimers());
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
it('waits for a slow task to finish before scheduling the next run', async () => {
|
||||
const scheduler = new IntervalScheduler(silentLogger);
|
||||
let runs = 0;
|
||||
scheduler.every('slow', 1000, async () => {
|
||||
runs++;
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(runs).toBe(1);
|
||||
await vi.advanceTimersByTimeAsync(4000);
|
||||
expect(runs).toBe(1);
|
||||
await vi.advanceTimersByTimeAsync(2000);
|
||||
expect(runs).toBe(2);
|
||||
scheduler.stop();
|
||||
});
|
||||
|
||||
it('keeps going after a task throws, and stops cleanly', async () => {
|
||||
const scheduler = new IntervalScheduler(silentLogger);
|
||||
const task = vi.fn().mockRejectedValueOnce(new Error('boom')).mockResolvedValue(undefined);
|
||||
scheduler.every('flaky', 1000, task);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(2000);
|
||||
expect(task).toHaveBeenCalledTimes(2);
|
||||
|
||||
scheduler.stop();
|
||||
await vi.advanceTimersByTimeAsync(5000);
|
||||
expect(task).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
39
src/lib/server/sync/scheduler.ts
Normal file
39
src/lib/server/sync/scheduler.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import { errorMessage } from './core/errors';
|
||||
import type { Logger } from './core/ports';
|
||||
|
||||
/**
|
||||
* Runs async tasks on fixed intervals. The next run of a task is scheduled only
|
||||
* after the previous one settles, so a slow task never overlaps itself.
|
||||
*/
|
||||
export class IntervalScheduler {
|
||||
private readonly timers = new Set<ReturnType<typeof setTimeout>>();
|
||||
private stopped = false;
|
||||
|
||||
constructor(private readonly logger: Logger) {}
|
||||
|
||||
every(name: string, intervalMs: number, task: () => Promise<unknown>): void {
|
||||
const tick = async () => {
|
||||
try {
|
||||
await task();
|
||||
} catch (error) {
|
||||
this.logger.error('Scheduled task failed', { task: name, error: errorMessage(error) });
|
||||
}
|
||||
schedule();
|
||||
};
|
||||
const schedule = () => {
|
||||
if (this.stopped) return;
|
||||
const timer = setTimeout(() => {
|
||||
this.timers.delete(timer);
|
||||
void tick();
|
||||
}, intervalMs);
|
||||
this.timers.add(timer);
|
||||
};
|
||||
schedule();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
for (const timer of this.timers) clearTimeout(timer);
|
||||
this.timers.clear();
|
||||
}
|
||||
}
|
||||
145
src/lib/server/sync/testing/fake-acc.ts
Normal file
145
src/lib/server/sync/testing/fake-acc.ts
Normal file
@@ -0,0 +1,145 @@
|
||||
import type { FileContent, TargetRef } from '../core/model';
|
||||
import type { DocumentTarget } from '../core/ports';
|
||||
|
||||
interface AccNode {
|
||||
id: string;
|
||||
parentId: string | null;
|
||||
name: string;
|
||||
isFolder: boolean;
|
||||
deleted: boolean;
|
||||
versions: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* In-memory stand-in for ACC Docs. Deleting a folder hides its contents (as in
|
||||
* ACC); restoring a folder does not bring its contents back.
|
||||
*/
|
||||
export class FakeAcc implements DocumentTarget {
|
||||
readonly rootFolderId = 'acc-root';
|
||||
readonly calls: string[] = [];
|
||||
private readonly nodes = new Map<string, AccNode>();
|
||||
private readonly failures = new Map<string, { error: Error; remaining: number }>();
|
||||
private nextId = 1;
|
||||
|
||||
constructor() {
|
||||
this.nodes.set(this.rootFolderId, {
|
||||
id: this.rootFolderId,
|
||||
parentId: null,
|
||||
name: 'Project Files',
|
||||
isFolder: true,
|
||||
deleted: false,
|
||||
versions: []
|
||||
});
|
||||
}
|
||||
|
||||
/** Make the next `times` operations on an item with this name throw `error`. */
|
||||
failOn(name: string, error: Error, times = Infinity): void {
|
||||
this.failures.set(name, { error, remaining: times });
|
||||
}
|
||||
|
||||
/** Adds a file directly, as if someone uploaded it by hand in ACC. */
|
||||
addByHand(parentId: string, name: string): string {
|
||||
return this.insert(parentId, name, false, ['by hand']);
|
||||
}
|
||||
|
||||
/** Live paths, e.g. `Drawings/`, `Drawings/plan.pdf@2` (file@version count). */
|
||||
tree(): string[] {
|
||||
return [...this.nodes.values()]
|
||||
.filter((node) => node.id !== this.rootFolderId && this.isAlive(node))
|
||||
.map((node) => this.path(node) + (node.isFolder ? '/' : `@${node.versions.length}`))
|
||||
.sort();
|
||||
}
|
||||
|
||||
latestContent(path: string): string | undefined {
|
||||
const node = [...this.nodes.values()].find((n) => this.isAlive(n) && this.path(n) === path);
|
||||
return node?.versions.at(-1);
|
||||
}
|
||||
|
||||
async createFolder(parentId: string, name: string): Promise<TargetRef> {
|
||||
this.enter('createFolder', name);
|
||||
return { id: this.insert(parentId, name, true, []), isFolder: true };
|
||||
}
|
||||
|
||||
async uploadFile(parentId: string, name: string, content: FileContent): Promise<TargetRef> {
|
||||
this.enter('uploadFile', name);
|
||||
return { id: this.insert(parentId, name, false, [await read(content)]), isFolder: false };
|
||||
}
|
||||
|
||||
async uploadVersion(ref: TargetRef, name: string, content: FileContent): Promise<void> {
|
||||
const node = this.node(ref, 'uploadVersion');
|
||||
node.versions.push(await read(content));
|
||||
node.name = name;
|
||||
}
|
||||
|
||||
async rename(ref: TargetRef, name: string): Promise<void> {
|
||||
this.node(ref, 'rename').name = name;
|
||||
}
|
||||
|
||||
async move(ref: TargetRef, newParentId: string): Promise<TargetRef> {
|
||||
this.node(ref, 'move').parentId = newParentId;
|
||||
return ref;
|
||||
}
|
||||
|
||||
async delete(ref: TargetRef): Promise<void> {
|
||||
this.node(ref, 'delete').deleted = true;
|
||||
}
|
||||
|
||||
async restore(
|
||||
ref: TargetRef,
|
||||
parentId: string,
|
||||
name: string,
|
||||
content?: FileContent
|
||||
): Promise<TargetRef> {
|
||||
const node = this.node(ref, 'restore');
|
||||
Object.assign(node, { deleted: false, parentId, name });
|
||||
if (content) node.versions.push(await read(content));
|
||||
return ref;
|
||||
}
|
||||
|
||||
private insert(parentId: string, name: string, isFolder: boolean, versions: string[]): string {
|
||||
if (!this.nodes.has(parentId)) throw new Error(`No ACC folder ${parentId}`);
|
||||
const id = `acc-${this.nextId++}`;
|
||||
this.nodes.set(id, { id, parentId, name, isFolder, deleted: false, versions });
|
||||
return id;
|
||||
}
|
||||
|
||||
private node(ref: TargetRef, operation: string): AccNode {
|
||||
const node = this.nodes.get(ref.id);
|
||||
if (!node) throw new Error(`No ACC item ${ref.id}`);
|
||||
this.enter(operation, node.name);
|
||||
return node;
|
||||
}
|
||||
|
||||
private enter(operation: string, name: string): void {
|
||||
this.calls.push(`${operation} ${name}`);
|
||||
const failure = this.failures.get(name);
|
||||
if (failure && failure.remaining > 0) {
|
||||
failure.remaining--;
|
||||
throw failure.error;
|
||||
}
|
||||
}
|
||||
|
||||
private isAlive(node: AccNode): boolean {
|
||||
for (
|
||||
let n: AccNode | undefined = node;
|
||||
n;
|
||||
n = n.parentId ? this.nodes.get(n.parentId) : undefined
|
||||
) {
|
||||
if (n.deleted) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private path(node: AccNode): string {
|
||||
const names: string[] = [];
|
||||
for (let n: AccNode | undefined = node; n && n.id !== this.rootFolderId;) {
|
||||
names.unshift(n.name);
|
||||
n = n.parentId ? this.nodes.get(n.parentId) : undefined;
|
||||
}
|
||||
return names.join('/');
|
||||
}
|
||||
}
|
||||
|
||||
async function read(content: FileContent): Promise<string> {
|
||||
return new Response(content.stream).text();
|
||||
}
|
||||
127
src/lib/server/sync/testing/fake-drive.ts
Normal file
127
src/lib/server/sync/testing/fake-drive.ts
Normal file
@@ -0,0 +1,127 @@
|
||||
import type { FileContent, SourceChange, SourceItem, SourceRoot } from '../core/model';
|
||||
import type { ChangeBatch, ChangeSource } from '../core/ports';
|
||||
|
||||
interface DriveNode {
|
||||
id: string;
|
||||
parentId: string | null;
|
||||
name: string;
|
||||
isFolder: boolean;
|
||||
content: string;
|
||||
version: number;
|
||||
deleted: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* In-memory stand-in for a SharePoint document library that behaves like Graph
|
||||
* delta: a null cursor lists everything; otherwise each item changed since the
|
||||
* cursor is reported once, in its latest state.
|
||||
*/
|
||||
export class FakeDrive implements ChangeSource {
|
||||
readonly driveRootId = 'drive-root';
|
||||
private readonly nodes = new Map<string, DriveNode>();
|
||||
private readonly log: string[] = [];
|
||||
private nextId = 1;
|
||||
|
||||
constructor() {
|
||||
this.nodes.set(this.driveRootId, {
|
||||
id: this.driveRootId,
|
||||
parentId: null,
|
||||
name: 'root',
|
||||
isFolder: true,
|
||||
content: '',
|
||||
version: 0,
|
||||
deleted: false
|
||||
});
|
||||
}
|
||||
|
||||
addFolder(parentId: string, name: string): string {
|
||||
return this.add(parentId, name, true, '');
|
||||
}
|
||||
|
||||
addFile(parentId: string, name: string, content = `${name} v1`): string {
|
||||
return this.add(parentId, name, false, content);
|
||||
}
|
||||
|
||||
edit(id: string, content: string): void {
|
||||
this.change(id, (node) => {
|
||||
node.content = content;
|
||||
node.version++;
|
||||
});
|
||||
}
|
||||
|
||||
rename(id: string, name: string): void {
|
||||
this.change(id, (node) => (node.name = name));
|
||||
}
|
||||
|
||||
move(id: string, parentId: string): void {
|
||||
this.change(id, (node) => (node.parentId = parentId));
|
||||
}
|
||||
|
||||
/** Reports only the removed item, not its contents: the harder case for the engine. */
|
||||
remove(id: string): void {
|
||||
this.change(id, (node) => (node.deleted = true));
|
||||
}
|
||||
|
||||
/** Reports only the restored item, not its contents. */
|
||||
restore(id: string): void {
|
||||
this.change(id, (node) => (node.deleted = false));
|
||||
}
|
||||
|
||||
async getChanges(_root: SourceRoot, cursor: string | null): Promise<ChangeBatch> {
|
||||
const end = String(this.log.length);
|
||||
if (cursor === null) {
|
||||
const changes = [...this.nodes.values()]
|
||||
.filter((node) => this.isAlive(node))
|
||||
.map((node) => this.toChange(node));
|
||||
return { changes, cursor: end, fullSync: true };
|
||||
}
|
||||
const ids = [...new Set(this.log.slice(Number(cursor)).reverse())].reverse();
|
||||
return {
|
||||
changes: ids.map((id) => this.toChange(this.nodes.get(id)!)),
|
||||
cursor: end,
|
||||
fullSync: false
|
||||
};
|
||||
}
|
||||
|
||||
async download(_root: SourceRoot, sourceId: string): Promise<FileContent> {
|
||||
const bytes = new TextEncoder().encode(this.nodes.get(sourceId)!.content);
|
||||
return { size: bytes.length, stream: new Response(bytes).body! };
|
||||
}
|
||||
|
||||
private add(parentId: string, name: string, isFolder: boolean, content: string): string {
|
||||
const id = `sp-${this.nextId++}`;
|
||||
this.nodes.set(id, { id, parentId, name, isFolder, content, version: 1, deleted: false });
|
||||
this.log.push(id);
|
||||
return id;
|
||||
}
|
||||
|
||||
private change(id: string, mutate: (node: DriveNode) => void): void {
|
||||
mutate(this.nodes.get(id)!);
|
||||
this.log.push(id);
|
||||
}
|
||||
|
||||
private isAlive(node: DriveNode): boolean {
|
||||
for (
|
||||
let n: DriveNode | undefined = node;
|
||||
n;
|
||||
n = n.parentId ? this.nodes.get(n.parentId) : undefined
|
||||
) {
|
||||
if (n.deleted) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private toChange(node: DriveNode): SourceChange {
|
||||
if (node.deleted) return { kind: 'delete', sourceId: node.id };
|
||||
const item: SourceItem = {
|
||||
sourceId: node.id,
|
||||
parentSourceId: node.parentId,
|
||||
name: node.name,
|
||||
isFolder: node.isFolder,
|
||||
contentTag: node.isFolder ? null : `c${node.version}`,
|
||||
modifiedAt: new Date(Date.UTC(2026, 0, 1, 0, 0, node.version)),
|
||||
size: node.isFolder ? null : node.content.length
|
||||
};
|
||||
return { kind: 'upsert', item };
|
||||
}
|
||||
}
|
||||
70
src/lib/server/sync/testing/harness.ts
Normal file
70
src/lib/server/sync/testing/harness.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
import { openDatabase } from '../../adapters/sqlite/database';
|
||||
import { SqliteItemRepository } from '../../adapters/sqlite/item-repository';
|
||||
import { SqliteProjectRepository } from '../../adapters/sqlite/project-repository';
|
||||
import { SqliteRunRepository } from '../../adapters/sqlite/run-repository';
|
||||
import { SyncEngine } from '../core/engine';
|
||||
import { PatternIgnorePolicy } from '../core/ignore';
|
||||
import type { Logger } from '../core/ports';
|
||||
import { Reconciler } from '../core/reconciler';
|
||||
import { FakeAcc } from './fake-acc';
|
||||
import { FakeDrive } from './fake-drive';
|
||||
|
||||
export const silentLogger: Logger = { info() {}, warn() {}, error() {} };
|
||||
|
||||
/** A real engine over in-memory SQLite, a fake SharePoint drive and a fake ACC. */
|
||||
export async function createHarness({ maxAttempts = 3 } = {}) {
|
||||
let now = Date.UTC(2026, 9, 1);
|
||||
const clock = () => new Date((now += 1000));
|
||||
|
||||
const db = openDatabase(':memory:');
|
||||
const projects = new SqliteProjectRepository(db, clock);
|
||||
const items = new SqliteItemRepository(db);
|
||||
const runs = new SqliteRunRepository(db);
|
||||
|
||||
const drive = new FakeDrive();
|
||||
const acc = new FakeAcc();
|
||||
const rootId = drive.addFolder(drive.driveRootId, 'Project X');
|
||||
|
||||
const reconciler = new Reconciler({
|
||||
source: drive,
|
||||
target: acc,
|
||||
items,
|
||||
runs,
|
||||
clock,
|
||||
maxAttempts
|
||||
});
|
||||
const engine = new SyncEngine({
|
||||
source: drive,
|
||||
projects,
|
||||
items,
|
||||
runs,
|
||||
reconciler,
|
||||
ignore: new PatternIgnorePolicy(),
|
||||
clock,
|
||||
logger: silentLogger,
|
||||
maxAttempts
|
||||
});
|
||||
|
||||
const project = await projects.create({
|
||||
name: 'Project X',
|
||||
source: { siteId: 'site', driveId: 'drive', rootItemId: rootId },
|
||||
sourceDisplayPath: '/sites/ProjectX/Documents/Project X',
|
||||
target: { hubId: 'hub', projectId: 'acc-project', rootFolderId: acc.rootFolderId },
|
||||
targetDisplayPath: 'Project X/Project Files'
|
||||
});
|
||||
|
||||
return {
|
||||
db,
|
||||
drive,
|
||||
acc,
|
||||
rootId,
|
||||
projects,
|
||||
items,
|
||||
runs,
|
||||
engine,
|
||||
project,
|
||||
sync: () => engine.runProject(project.id, 'schedule'),
|
||||
itemNamed: async (name: string) =>
|
||||
(await items.listByProject(project.id)).find((i) => i.desired.name === name)!
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user