| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272 |
- import test from 'node:test';
- import assert from 'node:assert/strict';
- import { mkdtemp, rm, readFile, readdir } from 'node:fs/promises';
- import { existsSync } from 'node:fs';
- import { tmpdir } from 'node:os';
- import { join, resolve, sep } from 'node:path';
- import { createStore } from '../src/store.js';
- import { createMediaService, imageCredential } from '../src/media-service.js';
- import { createAiService } from '../src/ai-service.js';
- const tenant = '11111111-1111-4111-8111-111111111111', user = '22222222-2222-4222-8222-222222222222';
- const accountId = `web-api:${tenant}:${user}`;
- const png = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a3ioAAAAASUVORK5CYII=', 'base64');
- const token = (overrides = {}) => 'Bearer eyJhbGciOiJub25lIn0.' + Buffer.from(JSON.stringify({ aud: 'https://ic3.teams.office.com', tid: tenant, oid: user, exp: Math.floor(Date.now()/1000) + 3600, ...overrides })).toString('base64url') + '.fictional';
- const har = value => ({ log: { entries: [{ request: { url: 'https://teams.cloud.microsoft/api/fictional', headers: [{ name: 'Authorization', value: value || token() }] } }] } });
- const response = () => new Response(png, { headers: { 'content-type': 'image/png' } });
- const record = (overrides = {}) => ({ source: 'web-api', accountId, kind: 'message', externalId: '1', parentId: 'fictional-chat', title: '图片', text: '图片', raw: { images: [{ objectId: '0-ea-fictional', label: '图片' }] }, ...overrides });
- const deferred = () => { let resolve; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; };
- async function fixture(t, options = {}) {
- const directory = await mkdtemp(join(tmpdir(), 'teams-media-test-'));
- const store = createStore(join(directory, 'teams.sqlite'));
- store.createInstance({ id: 'default', name: '测试' });
- store.bindInstanceAccounts('default', [{ source: 'web-api', accountId }]);
- store.setState('web-api:service', { enabled: true });
- const requests = [];
- const media = createMediaService({ store, config: { dataDir: directory, historyRequestMinDelayMs: 0, historyRequestMaxDelayMs: 0 }, readSession: () => har(), fetchImpl: async (url, options) => { requests.push({ url, options }); return response(); }, ...options });
- t.after(async () => {
- await media.stop(); store.close();
- assert.ok(resolve(directory).startsWith(resolve(tmpdir()) + sep) && directory.split(sep).at(-1).startsWith('teams-media-test-'));
- await rm(directory, { recursive: true, force: true });
- });
- return { store, media, requests, directory, save: (value = record(), ingestion = 'live') => {
- const saved = store.upsertRecord(value, { ingestion }); return { messageId: saved.id, imageId: store.listMessageImages(saved.id)[0]?.id };
- } };
- }
- test('live images download automatically once, survive reopening, and appear in AI messages', async t => {
- const f = await fixture(t), { messageId, imageId } = f.save();
- assert.equal(f.media.describe(imageId).status, 'pending');
- await f.media.runOnce();
- const saved = f.media.describe(imageId);
- assert.equal(saved.status, 'saved'); assert.equal(saved.bytes, png.length);
- assert.match(saved.localPath, /\.png$/);
- assert.deepEqual(await readFile(saved.localPath), png);
- assert.equal(f.requests[0].url, 'https://as-prod.asyncgw.teams.microsoft.com/v1/objects/0-ea-fictional/views/imgo');
- assert.equal(f.requests[0].options.redirect, 'manual');
- assert.equal(f.requests[0].options.headers.Cookie, undefined);
- await f.media.downloadImage({ id: imageId }); await f.media.runOnce(); assert.equal(f.requests.length, 1);
- const query = createAiService({ store: f.store, media: f.media });
- assert.equal(query.getMessage({ id: messageId }).message.images[0].id, imageId);
- assert.doesNotMatch(JSON.stringify(query.getMessage({ id: messageId })), /Bearer|eyJhbG|objectId|asyncgw/);
- const second = createStore(join(f.directory, 'teams.sqlite'));
- try { assert.equal(second.listMessageImages(messageId)[0].status, 'saved'); } finally { second.close(); }
- });
- test('history stays on demand and later live observation can promote the same image', async t => {
- const f = await fixture(t), { imageId } = f.save(record(), 'history');
- await f.media.runOnce(); assert.equal(f.requests.length, 0); assert.equal(f.media.describe(imageId).status, 'available');
- f.save(record(), 'live'); await f.media.runOnce(); assert.equal(f.requests.length, 1);
- const old = f.save(record({ externalId: '2' }), 'history');
- await f.media.downloadImage({ id: old.imageId }); assert.equal(f.media.describe(old.imageId).status, 'saved');
- });
- test('history backfill queues only the selected instance and remembers future discoveries and pause', async t => {
- const f = await fixture(t), secondId = '44444444-4444-4444-8444-444444444444';
- f.store.createInstance({ id: secondId, name: 'Other' });
- f.store.bindInstanceAccounts(secondId, [{ source: 'web-api', accountId: 'other-account' }]);
- const first = f.save(record(), 'history'), other = f.save(record({ accountId: 'other-account' }), 'history');
- const configured = f.media.configureHistoryDownloads({ instanceId: 'default', enabled: true });
- assert.equal(configured.queued, 1); assert.equal(configured.counts.pending, 1);
- assert.equal(f.media.describe(other.imageId).status, 'available');
- const future = f.save(record({ externalId: 'future-history' }), 'history');
- assert.equal(f.media.describe(future.imageId).status, 'pending');
- f.media.configureHistoryDownloads({ instanceId: 'default', enabled: false });
- await f.media.runOnce(); assert.equal(f.requests.length, 0);
- const live = f.save(record({ externalId: 'new-live' }));
- await f.media.runOnce(); assert.equal(f.media.describe(live.imageId).status, 'saved');
- assert.equal(f.media.describe(first.imageId).status, 'pending');
- f.media.configureHistoryDownloads({ instanceId: 'default', enabled: true });
- await f.media.runOnce(); assert.equal(f.media.describe(first.imageId).status, 'saved'); assert.equal(f.media.describe(future.imageId).status, 'saved');
- assert.equal(f.media.describe(other.imageId).status, 'available');
- const calls = f.requests.length;
- assert.equal(f.media.configureHistoryDownloads({ instanceId: 'default', enabled: true }).queued, 0);
- await f.media.runOnce(); assert.equal(f.requests.length, calls);
- });
- test('history preference and queued progress survive a separate service process', async t => {
- const f = await fixture(t), saved = f.save(record(), 'history');
- f.media.configureHistoryDownloads({ instanceId: 'default', enabled: true });
- const store = createStore(join(f.directory, 'teams.sqlite'));
- let calls = 0;
- const worker = createMediaService({ store, config: { dataDir: f.directory, historyRequestMinDelayMs: 0, historyRequestMaxDelayMs: 0 },
- readSession: () => har(), fetchImpl: async () => { calls++; return response(); } });
- try {
- assert.equal(worker.historyDownloadStatus({ instanceId: 'default' }).enabled, true);
- await worker.runOnce(); assert.equal(calls, 1); assert.equal(worker.describe(saved.imageId).status, 'saved');
- store.upsertRecord(record({ externalId: 'discovered-after-restart' }), { ingestion: 'history' });
- await worker.runOnce(); assert.equal(calls, 2);
- } finally { await worker.stop(); store.close(); }
- });
- test('0.7.0 image rows migrate without dropping saved files or existing retry state', async t => {
- const f = await fixture(t), history = f.save(record(), 'history'), live = f.save(record({ externalId: 'live' }));
- f.store.execute('ALTER TABLE message_images DROP COLUMN queueKind');
- const upgraded = createStore(join(f.directory, 'teams.sqlite'));
- try {
- assert.equal(upgraded.readOne('SELECT queueKind FROM message_images WHERE id=?', [history.imageId]).queueKind, 'history');
- assert.equal(upgraded.readOne('SELECT queueKind FROM message_images WHERE id=?', [live.imageId]).queueKind, 'live');
- assert.equal(upgraded.listMessageImages(history.messageId)[0].status, 'available');
- assert.equal(upgraded.listMessageImages(live.messageId)[0].status, 'pending');
- } finally { upgraded.close(); }
- });
- test('bulk retry excludes permanent failures, preserves cooldown and skips already saved images', async t => {
- const f = await fixture(t), one = f.save(record(), 'history'), two = f.save(record({ externalId: 'two' }), 'history'), three = f.save(record({ externalId: 'three' }), 'history');
- await f.media.downloadImage({ id: three.imageId });
- f.store.execute("UPDATE message_images SET status='failed',attempts=8,errorCode='download_failed' WHERE id=?", [one.imageId]);
- f.store.execute("UPDATE message_images SET status='failed',attempts=8,errorCode='image_unavailable' WHERE id=?", [two.imageId]);
- f.store.setState(`media:cooldown:${accountId}`, { until: Date.now() + 100000, reason: 'rate_limited' });
- const result = f.media.configureHistoryDownloads({ instanceId: 'default', enabled: true, retryFailed: true });
- assert.equal(result.retried, 1); assert.equal(result.counts.saved, 1); assert.equal(result.counts.failed, 1);
- assert.equal(f.media.describe(two.imageId).status, 'failed');
- await f.media.runOnce(); assert.equal(f.requests.length, 1);
- });
- test('per-instance pacing is persisted and live images take priority over historical backlog', async t => {
- let time = Date.now();
- const f = await fixture(t, { now: () => time, random: () => 0.5, config: { historyRequestMinDelayMs: 2000, historyRequestMaxDelayMs: 5000 } });
- f.media.configureHistoryDownloads({ instanceId: 'default', enabled: true });
- const old = f.save(record(), 'history'), live = f.save(record({ externalId: 'live' }));
- await f.media.runOnce(); assert.equal(f.requests.length, 1); assert.equal(f.media.describe(live.imageId).status, 'saved');
- assert.equal(f.store.getState('media:pacing:default').until, time + 3500);
- time += 3499; await f.media.runOnce(); assert.equal(f.requests.length, 1);
- time++; await f.media.runOnce(); assert.equal(f.requests.length, 2); assert.equal(f.media.describe(old.imageId).status, 'saved');
- });
- test('a large first-instance backlog gives another ready instance a turn', async t => {
- const secondId = '44444444-4444-4444-8444-444444444444', secondUser = '33333333-3333-4333-8333-333333333333';
- const f = await fixture(t, { readSession: id => har(token({ oid: id === 'default' ? user : secondUser })) });
- f.store.createInstance({ id: secondId, name: 'Other' }); f.store.bindInstanceAccounts(secondId, [{ source: 'web-api', accountId: `web-api:${tenant}:${secondUser}` }]);
- f.store.setState(`instance:${secondId}:web-api:service`, { enabled: true });
- for (const instanceId of ['default', secondId]) f.media.configureHistoryDownloads({ instanceId, enabled: true });
- f.store.transaction(() => { for (let i = 0; i < 120; i++) f.save(record({ externalId: `old-${i}` }), 'history'); });
- const other = f.save(record({ accountId: `web-api:${tenant}:${secondUser}` }), 'history');
- await f.media.runOnce(); assert.equal(f.requests.length, 3); assert.equal(f.media.describe(other.imageId).status, 'saved');
- });
- test('waiting for renewed credentials pauses the account without consuming per-image retry budgets', async t => {
- let time = Date.now(), session = har(token({ exp: 1 }));
- const f = await fixture(t, { now: () => time, readSession: () => session });
- f.media.configureHistoryDownloads({ instanceId: 'default', enabled: true });
- const first = f.save(record(), 'history'); f.save(record({ externalId: 'second' }), 'history');
- for (let i = 0; i < 10; i++) { await f.media.runOnce(); time += 60001; }
- assert.equal(f.requests.length, 0); assert.equal(f.store.readOne('SELECT SUM(attempts) n FROM message_images').n, 0);
- session = har(token({ exp: Math.floor(time / 1000) + 3600 }));
- await f.media.runOnce(); assert.equal(f.media.describe(first.imageId).status, 'saved');
- });
- test('editing a message retains its saved image as evidence from an earlier version', async t => {
- const f = await fixture(t), { imageId } = f.save(); await f.media.runOnce();
- const path = f.media.describe(imageId).localPath;
- f.save(record({ raw: {}, text: 'edited' }));
- assert.equal(f.media.describe(imageId).fromEarlierVersion, true); assert.equal(existsSync(path), true);
- f.save(); assert.equal(f.media.describe(imageId).fromEarlierVersion, false);
- });
- test('credential selection requires the exact instance identity, IC3 audience, trusted source and current expiry', async t => {
- const expected = token(); assert.equal(imageCredential(har(expected), accountId), expected);
- for (const claims of [{ oid: tenant }, { tid: user }, { aud: 'https://graph.microsoft.com' }, { exp: 1 }]) assert.throws(() => imageCredential(har(token(claims)), accountId), /session_expired/);
- const untrusted = har(); untrusted.log.entries[0].request.url = 'https://evil.invalid/';
- assert.throws(() => imageCredential(untrusted, accountId));
- const f = await fixture(t, { readSession: () => har(token({ oid: tenant })) });
- const { imageId } = f.save(); await f.media.runOnce();
- assert.equal(f.requests.length, 0); assert.equal(f.media.describe(imageId).errorCode, 'session_expired');
- });
- test('different instances use independent credentials and do not share cached image files', async t => {
- const secondUser = '33333333-3333-4333-8333-333333333333', instanceId = '44444444-4444-4444-8444-444444444444';
- const other = `web-api:${tenant}:${secondUser}`, calls = [];
- const f = await fixture(t, { readSession: id => har(token({ oid: id === 'default' ? user : secondUser })), fetchImpl: async (url, options) => { calls.push(options.headers.Authorization); return response(); } });
- f.store.createInstance({ id: instanceId, name: '第二实例' }); f.store.bindInstanceAccounts(instanceId, [{ source: 'web-api', accountId: other }]);
- f.store.setState(`instance:${instanceId}:web-api:service`, { enabled: true });
- const one = f.save(), two = f.save(record({ accountId: other }));
- await f.media.runOnce();
- assert.equal(calls.length, 2); assert.notEqual(calls[0], calls[1]);
- assert.notEqual(f.media.describe(one.imageId).localPath, f.media.describe(two.imageId).localPath);
- });
- test('automatic image downloads follow the instance collection switch', async t => {
- const f = await fixture(t); f.store.setState('web-api:service', { enabled: false });
- f.save(); await f.media.runOnce(); assert.equal(f.requests.length, 0);
- f.store.setState('web-api:service', { enabled: true }); await f.media.runOnce(); assert.equal(f.requests.length, 1);
- });
- test('a large cooled-down instance queue cannot starve another instance', async t => {
- const secondUser = '33333333-3333-4333-8333-333333333333', instanceId = '44444444-4444-4444-8444-444444444444';
- const other = `web-api:${tenant}:${secondUser}`;
- const f = await fixture(t, { readSession: id => har(token({ oid: id === 'default' ? user : secondUser })) });
- f.store.createInstance({ id: instanceId, name: '第二实例' }); f.store.bindInstanceAccounts(instanceId, [{ source: 'web-api', accountId: other }]);
- f.store.setState(`instance:${instanceId}:web-api:service`, { enabled: true });
- f.store.transaction(() => { for (let i = 0; i < 120; i++) f.save(record({ externalId: String(i) })); });
- f.store.setState(`media:cooldown:${accountId}`, { until: Date.now() + 120000 });
- const next = f.save(record({ accountId: other }));
- await f.media.runOnce(); assert.equal(f.requests.length, 1); assert.equal(f.media.describe(next.imageId).status, 'saved');
- });
- test('permanent failures never save HTML, SVG, redirects or oversized image responses', async t => {
- for (const [value, expected] of [
- [() => new Response('<html>private-error</html>', { headers: { 'content-type': 'image/png' } }), 'invalid_image'],
- [() => new Response('<svg/>', { headers: { 'content-type': 'image/svg+xml' } }), 'unsupported_image'],
- [() => new Response('', { status: 302, headers: { location: 'https://evil.invalid/?token=secret' } }), 'redirect_blocked'],
- [() => new Response(png, { headers: { 'content-type': 'image/png', 'content-length': '30000000' } }), 'image_too_large'],
- [() => new Response('', { status: 404 }), 'image_unavailable']]) {
- const f = await fixture(t, { fetchImpl: async () => value() }); const { imageId } = f.save();
- const result = await f.media.downloadImage({ id: imageId });
- assert.equal(result.status, 'failed'); assert.equal(result.errorCode, expected);
- assert.equal(existsSync(f.store.mediaPath(imageId)), false); assert.doesNotMatch(JSON.stringify(result), /private-error|evil|secret/);
- }
- const f = await fixture(t, { maxBytes: 10 }); const { imageId } = f.save();
- assert.equal((await f.media.downloadImage({ id: imageId })).errorCode, 'image_too_large');
- });
- test('429 persists cooldown across images and manual retries, then recovers', async t => {
- let time = Date.now(), calls = 0;
- const f = await fixture(t, { now: () => time, fetchImpl: async () => ++calls === 1 ? new Response('', { status: 429, headers: { 'retry-after': '120' } }) : response() });
- const { imageId } = f.save(); f.save(record({ externalId: '2' }));
- await f.media.runOnce(); assert.equal(calls, 1);
- assert.equal(f.media.describe(imageId).errorCode, 'rate_limited');
- await f.media.downloadImage({ id: imageId }); assert.equal(calls, 1);
- time += 120001; await f.media.runOnce(); assert.equal(calls, 3); assert.equal(f.media.describe(imageId).status, 'saved');
- });
- test('long server Retry-After values are respected without retrying before the requested time', async t => {
- const time = Date.now();
- const f = await fixture(t, { now: () => time, fetchImpl: async () => new Response('', { status: 429, headers: { 'retry-after': '7200' } }) });
- const { imageId } = f.save(); await f.media.runOnce();
- assert.equal(f.store.readOne('SELECT nextAttemptAt FROM message_images WHERE id=?', [imageId]).nextAttemptAt, time + 7200000);
- });
- test('download timeout releases its lease and reports retryable interruption', async t => {
- const f = await fixture(t, { timeoutMs: 15, fetchImpl: async (url, { signal }) => new Promise((resolve, reject) => signal.addEventListener('abort', () => reject(new Error('sensitive-url')), { once: true })) });
- const { imageId } = f.save();
- const keepAlive = setTimeout(() => {}, 1000); t.after(() => clearTimeout(keepAlive));
- const result = await f.media.downloadImage({ id: imageId });
- assert.equal(result.status, 'retrying'); assert.equal(result.errorCode, 'download_interrupted');
- assert.equal(f.store.readOne('SELECT leaseToken FROM message_images WHERE id=?', [imageId]).leaseToken, null);
- });
- test('an abandoned download lease and an expired session recover with the refreshed instance session', async t => {
- let time = Date.now(), current = token({ exp: 1 });
- const f = await fixture(t, { now: () => time, readSession: () => har(current) });
- const { imageId } = f.save();
- f.store.execute("UPDATE message_images SET status='downloading',leaseToken='abandoned',leaseUntil=? WHERE id=?", [time - 1, imageId]);
- await f.media.runOnce(); assert.equal(f.media.describe(imageId).errorCode, 'session_expired');
- assert.equal(f.requests.length, 0);
- time += 60001; current = token(); await f.media.runOnce();
- assert.equal(f.requests.length, 1); assert.equal(f.media.describe(imageId).status, 'saved');
- });
- test('deleting an account cleans saved files; a racing download cannot recreate them', async t => {
- const f = await fixture(t), saved = f.save(); await f.media.runOnce();
- const path = f.media.describe(saved.imageId).localPath;
- assert.equal(f.store.previewAccountDeletion({ source: 'web-api', accountId }).counts.images, 1);
- f.store.deleteAccountRecords({ source: 'web-api', accountId }); f.store.collectMediaGarbage();
- assert.equal(existsSync(path), false); assert.throws(() => f.media.describe(saved.imageId));
- const entered = deferred(), finish = deferred();
- const race = await fixture(t, { fetchImpl: async () => { entered.resolve(); await finish.promise; return response(); } });
- const item = race.save(), download = race.media.downloadImage({ id: item.imageId });
- const checked = assert.rejects(download, /image_not_found/);
- await entered.promise; race.store.deleteAccountRecords({ source: 'web-api', accountId }); finish.resolve(); await checked;
- race.store.collectMediaGarbage(); assert.equal(existsSync(race.store.mediaPath(item.imageId)), false);
- assert.deepEqual(await readdir(join(race.directory, 'media')), []);
- });
- test('inflight leases prevent duplicate downloads across MCP and collector processes', async t => {
- const entered = deferred(), finish = deferred(); let calls = 0;
- const f = await fixture(t, { fetchImpl: async () => { calls++; entered.resolve(); await finish.promise; return response(); } });
- const { imageId } = f.save(); const first = f.media.downloadImage({ id: imageId }); await entered.promise;
- const other = createMediaService({ store: f.store, config: { dataDir: f.directory }, readSession: () => har(), fetchImpl: async () => { calls++; return response(); } });
- try {
- assert.equal((await other.downloadImage({ id: imageId })).status, 'downloading');
- finish.resolve(); await first; assert.equal(calls, 1);
- } finally { finish.resolve(); await other.stop(); }
- });
|