var STORAGE_KEY_PREFIX = 'magnet-progress-state:'; var MAX_PROGRESS_ITEMS = 2000; var SESSION_MARKER_KEY = 'magnet-progress-session-marker-v1'; var CACHE_DB_NAME = 'magnet-thread-cache'; var CACHE_DB_VERSION = 3; var COVERAGE_LIMIT_PER_FORUM = 999; var CLOUD_CACHE_ENABLED = true; var CLOUD_CACHE_BASE_URL = 'https://s.52oai.com'; var CLOUD_CACHE_TIMEOUT = 6000; var CLOUD_SHARED_CACHE_WRITE_TOKEN = 's52oai-shared-cache-write-v1'; var CLOUD_AUTH_STORAGE_KEY = 'magnet-cloud-auth-v1'; var CLOUD_DEVICE_ID_KEY = 'magnet-cloud-device-id-v1'; var CLOUD_CACHE_BACKFILL_KEY = 'magnet-cloud-cache-backfill-v1'; var CLOUD_UPLOAD_META_KEY = 'magnet-cloud-upload-meta-v1'; var CLOUD_VAULT_KEY_VERSION = 1; var CLOUD_THREAD_UPLOAD_TTL_MS = 10 * 60 * 1000; var CLOUD_PAGE_UPLOAD_TTL_MS = 30 * 60 * 1000; var CLOUD_COVERAGE_UPLOAD_TTL_MS = 60 * 60 * 1000; var CLOUD_UPLOAD_META_THREAD_LIMIT = 50000; var CLOUD_UPLOAD_META_OTHER_LIMIT = 4000; var cloudAuthStateCache = null; var cloudUploadMetaCache = null; var cloudBackfillPromise = null; var isSessionReady = false; var isPreparingSession = false; var sessionReadyCallbacks = []; function requestToPromise(request) { return new Promise(function(resolve, reject) { request.onsuccess = function(event) { resolve(event.target.result); }; request.onerror = function(event) { reject(event.target.error || new Error('IndexedDB 请求失败')); }; }); } function transactionDone(transaction) { return new Promise(function(resolve, reject) { transaction.oncomplete = function() { resolve(); }; transaction.onerror = function() { reject(transaction.error || new Error('IndexedDB 事务失败')); }; transaction.onabort = function() { reject(transaction.error || new Error('IndexedDB 事务已中止')); }; }); } function normalizeCloudBaseUrl(url) { return String(url || '').replace(/\/+$/, ''); } function callCloudCache(path, payload, timeoutMs, extraHeaders) { if (!CLOUD_CACHE_ENABLED) { return Promise.resolve(null); } var controller = new AbortController(); var realTimeout = Math.max(2000, Math.min(Number(timeoutMs) || CLOUD_CACHE_TIMEOUT, 15000)); var timer = setTimeout(function() { controller.abort(); }, realTimeout); return fetch(normalizeCloudBaseUrl(CLOUD_CACHE_BASE_URL) + path, { method: 'POST', headers: Object.assign({ 'Content-Type': 'application/json' }, extraHeaders || {}), body: JSON.stringify(payload || {}), signal: controller.signal }).then(function(response) { if (!response.ok) { throw new Error('云缓存 HTTP ' + response.status); } return response.json(); }).catch(function(error) { if (error && error.name === 'AbortError') { throw new Error('云缓存请求超时(' + realTimeout + 'ms)'); } throw error; }).finally(function() { clearTimeout(timer); }); } function storageLocalGet(key) { return new Promise(function(resolve, reject) { chrome.storage.local.get(key, function(result) { if (chrome.runtime.lastError) { reject(new Error(chrome.runtime.lastError.message)); return; } resolve(result || {}); }); }); } function storageLocalSet(data) { return new Promise(function(resolve, reject) { chrome.storage.local.set(data, function() { if (chrome.runtime.lastError) { reject(new Error(chrome.runtime.lastError.message)); return; } resolve(); }); }); } function storageLocalRemove(key) { return new Promise(function(resolve, reject) { chrome.storage.local.remove(key, function() { if (chrome.runtime.lastError) { reject(new Error(chrome.runtime.lastError.message)); return; } resolve(); }); }); } function textEncoder() { return new TextEncoder(); } function textDecoder() { return new TextDecoder(); } function bytesToBase64(bytes) { var binary = ''; var chunkSize = 0x8000; var index = 0; for (index = 0; index < bytes.length; index += chunkSize) { binary += String.fromCharCode.apply(null, bytes.subarray(index, index + chunkSize)); } return btoa(binary); } function base64ToBytes(base64) { var binary = atob(String(base64 || '')); var bytes = new Uint8Array(binary.length); var index = 0; for (index = 0; index < binary.length; index++) { bytes[index] = binary.charCodeAt(index); } return bytes; } function arrayBufferToBase64(buffer) { return bytesToBase64(new Uint8Array(buffer)); } function randomBase64(size) { var bytes = new Uint8Array(size); crypto.getRandomValues(bytes); return bytesToBase64(bytes); } async function sha256Hex(value) { var digest = await crypto.subtle.digest('SHA-256', textEncoder().encode(String(value || ''))); return Array.from(new Uint8Array(digest)).map(function(byte) { return byte.toString(16).padStart(2, '0'); }).join(''); } async function derivePasswordKey(password, saltBase64, keyUsages) { var baseKey = await crypto.subtle.importKey( 'raw', textEncoder().encode(String(password || '')), 'PBKDF2', false, ['deriveKey'] ); return crypto.subtle.deriveKey({ name: 'PBKDF2', salt: base64ToBytes(saltBase64), iterations: 250000, hash: 'SHA-256' }, baseKey, { name: 'AES-GCM', length: 256 }, false, keyUsages || ['encrypt', 'decrypt']); } async function importVaultKey(vaultKeyBase64, keyUsages) { return crypto.subtle.importKey( 'raw', base64ToBytes(vaultKeyBase64), { name: 'AES-GCM', length: 256 }, false, keyUsages || ['encrypt', 'decrypt'] ); } function splitCipherAndTag(buffer) { var bytes = new Uint8Array(buffer); var tagLength = 16; return { ciphertext: bytesToBase64(bytes.slice(0, bytes.length - tagLength)), tag: bytesToBase64(bytes.slice(bytes.length - tagLength)) }; } function joinCipherAndTag(ciphertextBase64, tagBase64) { var ciphertext = base64ToBytes(ciphertextBase64); var tag = base64ToBytes(tagBase64); var merged = new Uint8Array(ciphertext.length + tag.length); merged.set(ciphertext, 0); merged.set(tag, ciphertext.length); return merged; } async function createVaultEnvelope(password) { var saltBase64 = randomBase64(16); var vaultKeyBase64 = randomBase64(32); var ivBase64 = randomBase64(12); var passwordKey = await derivePasswordKey(password, saltBase64, ['encrypt', 'decrypt']); var encrypted = await crypto.subtle.encrypt( { name: 'AES-GCM', iv: base64ToBytes(ivBase64) }, passwordKey, base64ToBytes(vaultKeyBase64) ); return { vaultKeyBase64: vaultKeyBase64, wrappedDek: JSON.stringify({ ciphertext: arrayBufferToBase64(encrypted), iv: ivBase64 }), kdfSalt: saltBase64, kdfParams: { algorithm: 'PBKDF2', hash: 'SHA-256', iterations: 250000 }, keyVersion: CLOUD_VAULT_KEY_VERSION }; } async function unlockVaultKey(password, keyring) { var envelope = keyring && keyring.wrappedDek ? JSON.parse(keyring.wrappedDek) : null; if (!envelope || !envelope.ciphertext || !envelope.iv || !keyring.kdfSalt) { throw new Error('保险柜密钥无效'); } var passwordKey = await derivePasswordKey(password, keyring.kdfSalt, ['decrypt']); var decrypted = await crypto.subtle.decrypt( { name: 'AES-GCM', iv: base64ToBytes(envelope.iv) }, passwordKey, base64ToBytes(envelope.ciphertext) ); return bytesToBase64(new Uint8Array(decrypted)); } async function encryptVaultPayload(vaultKeyBase64, payload) { var json = JSON.stringify(payload); var ivBase64 = randomBase64(12); var vaultKey = await importVaultKey(vaultKeyBase64, ['encrypt']); var encrypted = await crypto.subtle.encrypt( { name: 'AES-GCM', iv: base64ToBytes(ivBase64) }, vaultKey, textEncoder().encode(json) ); var split = splitCipherAndTag(encrypted); return { payloadCiphertext: split.ciphertext, payloadIv: ivBase64, payloadTag: split.tag, payloadHash: await sha256Hex(json), keyVersion: CLOUD_VAULT_KEY_VERSION }; } async function decryptVaultPayload(vaultKeyBase64, item) { var vaultKey = await importVaultKey(vaultKeyBase64, ['decrypt']); var decrypted = await crypto.subtle.decrypt( { name: 'AES-GCM', iv: base64ToBytes(item.payloadIv) }, vaultKey, joinCipherAndTag(item.payloadCiphertext, item.payloadTag) ); return JSON.parse(textDecoder().decode(decrypted)); } function normalizeCloudAuthState(state) { state = state && typeof state === 'object' ? state : {}; return { token: typeof state.token === 'string' ? state.token : '', vaultKeyBase64: typeof state.vaultKeyBase64 === 'string' ? state.vaultKeyBase64 : '', userId: Number(state.userId) || 0, email: typeof state.email === 'string' ? state.email : '', syncEnabled: !!state.syncEnabled, healthy: !!state.healthy, lastError: typeof state.lastError === 'string' ? state.lastError : '', lastSyncAt: Number(state.lastSyncAt) || 0 }; } async function loadCloudAuthState() { if (cloudAuthStateCache) { return cloudAuthStateCache; } var result = await storageLocalGet(CLOUD_AUTH_STORAGE_KEY); cloudAuthStateCache = normalizeCloudAuthState(result[CLOUD_AUTH_STORAGE_KEY]); return cloudAuthStateCache; } async function saveCloudAuthState(state) { cloudAuthStateCache = normalizeCloudAuthState(state); await storageLocalSet((function() { var data = {}; data[CLOUD_AUTH_STORAGE_KEY] = cloudAuthStateCache; return data; })()); return cloudAuthStateCache; } async function clearCloudAuthState() { cloudAuthStateCache = normalizeCloudAuthState(null); await storageLocalRemove(CLOUD_AUTH_STORAGE_KEY); return cloudAuthStateCache; } async function ensureCloudDeviceFingerprint() { var result = await storageLocalGet(CLOUD_DEVICE_ID_KEY); var existing = typeof result[CLOUD_DEVICE_ID_KEY] === 'string' ? result[CLOUD_DEVICE_ID_KEY] : ''; if (existing) { return existing; } existing = 'device-' + randomBase64(18).replace(/[^a-zA-Z0-9_-]/g, ''); await storageLocalSet((function() { var data = {}; data[CLOUD_DEVICE_ID_KEY] = existing; return data; })()); return existing; } function buildCloudStatus(state) { var normalized = normalizeCloudAuthState(state); var healthy = !!(normalized.token && normalized.syncEnabled && normalized.healthy); return { ok: true, syncEnabled: !!normalized.syncEnabled, authenticated: !!normalized.token, healthy: healthy, color: healthy ? 'green' : 'red', email: normalized.email, lastError: normalized.lastError, text: healthy ? '云同步正常' : (normalized.token ? '云同步需确认' : '云同步未登录'), accountText: healthy ? '账号状态正常' : (normalized.token ? '账号状态需确认' : '未登录') }; } function callCloudApi(path, options) { var method = options && options.method ? options.method : 'GET'; var payload = options && Object.prototype.hasOwnProperty.call(options, 'payload') ? options.payload : null; var token = options && options.token ? options.token : ''; var timeoutMs = options && options.timeoutMs ? options.timeoutMs : CLOUD_CACHE_TIMEOUT; var controller = new AbortController(); var realTimeout = Math.max(2000, Math.min(Number(timeoutMs) || CLOUD_CACHE_TIMEOUT, 15000)); var timer = setTimeout(function() { controller.abort(); }, realTimeout); var headers = { 'Content-Type': 'application/json' }; if (token) { headers.Authorization = 'Bearer ' + token; } return fetch(normalizeCloudBaseUrl(CLOUD_CACHE_BASE_URL) + path, { method: method, headers: headers, body: payload !== null ? JSON.stringify(payload) : undefined, signal: controller.signal }).then(function(response) { if (!response.ok) { return response.text().then(function(text) { var payload = null; try { payload = text ? JSON.parse(text) : null; } catch (e) { payload = null; } throw new Error(payload && payload.error ? payload.error : ('云接口 HTTP ' + response.status)); }); } return response.json(); }).finally(function() { clearTimeout(timer); }); } async function setCloudHealth(ok, errorMessage) { var state = await loadCloudAuthState(); state.healthy = !!ok; state.lastError = ok ? '' : String(errorMessage || '同步失败'); state.lastSyncAt = Date.now(); await saveCloudAuthState(state); return buildCloudStatus(state); } async function cloudRegisterAccount(payload) { var email = String(payload.email || '').trim().toLowerCase(); var password = String(payload.password || ''); var deviceFingerprint = await ensureCloudDeviceFingerprint(); var envelope = await createVaultEnvelope(password); var response = await callCloudApi('/api/auth/register', { method: 'POST', payload: { email: email, password: password, wrappedDek: envelope.wrappedDek, kdfSalt: envelope.kdfSalt, kdfParams: envelope.kdfParams, keyVersion: envelope.keyVersion, deviceName: 'Magnet Chrome Extension', deviceFingerprint: deviceFingerprint } }); if (!response || !response.ok || !response.token) { throw new Error(response && response.error ? response.error : '注册失败'); } await saveCloudAuthState({ token: response.token, vaultKeyBase64: envelope.vaultKeyBase64, userId: response.user && response.user.id, email: response.user && response.user.email ? response.user.email : email, syncEnabled: true, healthy: true, lastError: '', lastSyncAt: Date.now() }); backfillLocalCacheToCloud({ force: true }).catch(function(error) { console.warn('[MagnetPlugin][Cloud] register backfill failed:', error && error.message ? error.message : error); }); return { ok: true, status: buildCloudStatus(await loadCloudAuthState()), vaultItems: [] }; } async function cloudPullVaultItems(itemTypes) { var state = await loadCloudAuthState(); var response = null; var items = []; if (!state.token || !state.vaultKeyBase64) { return []; } response = await callCloudApi('/api/vault/pull', { method: 'POST', token: state.token, payload: { itemTypes: Array.isArray(itemTypes) ? itemTypes : [] } }); if (!response || !response.ok || !Array.isArray(response.items)) { throw new Error(response && response.error ? response.error : '读取保险柜失败'); } items = []; for (var index = 0; index < response.items.length; index++) { var item = response.items[index]; items.push({ itemType: item.itemType, itemKey: item.itemKey, data: await decryptVaultPayload(state.vaultKeyBase64, item), updatedAt: item.updatedAt, keyVersion: item.keyVersion }); } await setCloudHealth(true); return items; } async function cloudLoginAccount(payload) { var email = String(payload.email || '').trim().toLowerCase(); var password = String(payload.password || ''); var deviceFingerprint = await ensureCloudDeviceFingerprint(); var response = await callCloudApi('/api/auth/login', { method: 'POST', payload: { email: email, password: password, deviceName: 'Magnet Chrome Extension', deviceFingerprint: deviceFingerprint } }); var vaultKeyBase64 = null; var vaultItems = []; if (!response || !response.ok || !response.token || !response.keyring) { throw new Error(response && response.error ? response.error : '登录失败'); } vaultKeyBase64 = await unlockVaultKey(password, response.keyring); await saveCloudAuthState({ token: response.token, vaultKeyBase64: vaultKeyBase64, userId: response.user && response.user.id, email: response.user && response.user.email ? response.user.email : email, syncEnabled: true, healthy: true, lastError: '', lastSyncAt: Date.now() }); backfillLocalCacheToCloud({ force: true }).catch(function(error) { console.warn('[MagnetPlugin][Cloud] login backfill failed:', error && error.message ? error.message : error); }); cloudPullVaultItems(['favorites', 'search_history', 'ui_settings', 'progress_state']).catch(function(error) { console.warn('[MagnetPlugin][Cloud] login vault pull failed:', error && error.message ? error.message : error); return []; }); return { ok: true, status: buildCloudStatus(await loadCloudAuthState()), vaultItems: vaultItems }; } async function cloudLogoutAccount() { var state = await loadCloudAuthState(); if (state.token) { try { await callCloudApi('/api/auth/logout', { method: 'POST', token: state.token, payload: {} }); } catch (error) { console.warn('[MagnetPlugin][Cloud] logout failed:', error && error.message ? error.message : error); } } await clearCloudAuthState(); return { ok: true, status: buildCloudStatus(await loadCloudAuthState()) }; } async function cloudGetSyncStatus() { var state = await loadCloudAuthState(); if (!state.token) { return buildCloudStatus(state); } try { var response = await callCloudApi('/api/auth/me', { method: 'GET', token: state.token }); if (response && response.ok) { state.email = response.user && response.user.email ? response.user.email : state.email; state.userId = response.user && response.user.id ? response.user.id : state.userId; state.syncEnabled = true; state.healthy = true; state.lastError = ''; state.lastSyncAt = Date.now(); await saveCloudAuthState(state); backfillLocalCacheToCloud({ force: false }).catch(function(error) { console.warn('[MagnetPlugin][Cloud] background backfill failed:', error && error.message ? error.message : error); }); return buildCloudStatus(state); } throw new Error(response && response.error ? response.error : '状态检查失败'); } catch (error) { return { ok: true, syncEnabled: !!state.syncEnabled, authenticated: !!state.token, healthy: false, color: 'red', email: state.email, lastError: error && error.message ? error.message : String(error), text: state.token ? '云同步需确认' : '云同步未登录', accountText: state.token ? '账号状态需确认' : '未登录' }; } } async function cloudGetCacheStats() { return callCloudApi('/api/shared-cache/stats', { method: 'GET', timeoutMs: 12000 }); } async function getLocalCacheStats() { var db = await openCacheDb(); var privateState = null; var uploadMeta = null; var threadCount = 0; var magnetThreadCount = 0; var coverageCount = 0; var pageCount = 0; var storageEstimate = null; var allThreads = []; try { var tx = db.transaction(['threads', 'coverages', 'pageCoverages'], 'readonly'); threadCount = Number(await requestToPromise(tx.objectStore('threads').count()) || 0); coverageCount = Number(await requestToPromise(tx.objectStore('coverages').count()) || 0); pageCount = Number(await requestToPromise(tx.objectStore('pageCoverages').count()) || 0); await transactionDone(tx); allThreads = await readAllStoreRecords(db, 'threads'); magnetThreadCount = allThreads.filter(function(record) { return Array.isArray(record.magnets) && normalizeMagnets(record.magnets).length > 0; }).length; } finally { db.close(); } privateState = await storageLocalGet('magnet-private-state-v1'); uploadMeta = await loadCloudUploadMetaState(); storageEstimate = await getStorageEstimate(); var privateData = privateState['magnet-private-state-v1'] && typeof privateState['magnet-private-state-v1'] === 'object' ? privateState['magnet-private-state-v1'] : {}; return { ok: true, counts: { threads: threadCount, magnetThreads: magnetThreadCount, coverages: coverageCount, pages: pageCount, favorites: Array.isArray(privateData.favorites) ? privateData.favorites.length : 0, history: Array.isArray(privateData.searchHistory) ? privateData.searchHistory.length : 0, uploadMetaThreads: Object.keys(uploadMeta.threads || {}).length, uploadMetaCoverages: Object.keys(uploadMeta.coverages || {}).length, uploadMetaPages: Object.keys(uploadMeta.pages || {}).length }, storage: { usage: storageEstimate && Number(storageEstimate.usage) ? Number(storageEstimate.usage) : 0, quota: storageEstimate && Number(storageEstimate.quota) ? Number(storageEstimate.quota) : 0 } }; } async function cloudPushVaultItems(payload) { var state = await loadCloudAuthState(); var items = Array.isArray(payload.items) ? payload.items : []; var preparedItems = []; var index = 0; var item = null; if (!state.token || !state.vaultKeyBase64) { return buildCloudStatus(state); } for (index = 0; index < items.length; index++) { item = items[index]; if (!item || typeof item !== 'object') { continue; } var itemType = String(item.itemType || '').trim(); var itemKey = String(item.itemKey || '').trim() || itemType; if (!itemType) { continue; } var encrypted = await encryptVaultPayload(state.vaultKeyBase64, item.data); preparedItems.push({ itemType: itemType, itemKey: itemKey, payloadCiphertext: encrypted.payloadCiphertext, payloadIv: encrypted.payloadIv, payloadTag: encrypted.payloadTag, payloadHash: encrypted.payloadHash, keyVersion: encrypted.keyVersion }); } if (preparedItems.length === 0) { return buildCloudStatus(state); } try { await callCloudApi('/api/vault/push', { method: 'POST', token: state.token, payload: { items: preparedItems } }); return await setCloudHealth(true); } catch (error) { return await setCloudHealth(false, error && error.message ? error.message : error); } } function openCacheDb() { return new Promise(function(resolve, reject) { var request = indexedDB.open(CACHE_DB_NAME, CACHE_DB_VERSION); request.onupgradeneeded = function(event) { var db = event.target.result; if (!db.objectStoreNames.contains('threads')) { var threadStore = db.createObjectStore('threads', { keyPath: 'cacheKey' }); threadStore.createIndex('forumKey', 'forumKey', { unique: false }); threadStore.createIndex('lastSeenAt', 'lastSeenAt', { unique: false }); threadStore.createIndex('forumKeyLastSeenAt', ['forumKey', 'lastSeenAt'], { unique: false }); } else { var existingThreadStore = request.transaction.objectStore('threads'); if (!existingThreadStore.indexNames.contains('forumKeyLastSeenAt')) { existingThreadStore.createIndex('forumKeyLastSeenAt', ['forumKey', 'lastSeenAt'], { unique: false }); } } if (!db.objectStoreNames.contains('coverages')) { var coverageStore = db.createObjectStore('coverages', { keyPath: 'coverageKey' }); coverageStore.createIndex('forumKey', 'forumKey', { unique: false }); coverageStore.createIndex('crawledAt', 'crawledAt', { unique: false }); coverageStore.createIndex('forumKeyCrawledAt', ['forumKey', 'crawledAt'], { unique: false }); } else { var existingCoverageStore = request.transaction.objectStore('coverages'); if (!existingCoverageStore.indexNames.contains('forumKeyCrawledAt')) { existingCoverageStore.createIndex('forumKeyCrawledAt', ['forumKey', 'crawledAt'], { unique: false }); } } if (!db.objectStoreNames.contains('pageCoverages')) { var pageCoverageStore = db.createObjectStore('pageCoverages', { keyPath: 'coverageKey' }); pageCoverageStore.createIndex('forumKey', 'forumKey', { unique: false }); pageCoverageStore.createIndex('crawledAt', 'crawledAt', { unique: false }); pageCoverageStore.createIndex('forumKeyCrawledAt', ['forumKey', 'crawledAt'], { unique: false }); } else { var existingPageCoverageStore = request.transaction.objectStore('pageCoverages'); if (!existingPageCoverageStore.indexNames.contains('forumKeyCrawledAt')) { existingPageCoverageStore.createIndex('forumKeyCrawledAt', ['forumKey', 'crawledAt'], { unique: false }); } } }; request.onsuccess = function(event) { resolve(event.target.result); }; request.onerror = function(event) { reject(event.target.error || new Error('无法打开缓存数据库')); }; }); } function normalizeUrl(url) { if (typeof url !== 'string' || !url) { return ''; } try { var parsed = new URL(url); parsed.hash = ''; return parsed.href; } catch (e) { return url; } } function getThreadKeyFromUrl(url) { var normalized = normalizeUrl(url); if (!normalized) { return ''; } var match = normalized.match(/thread-(\d+)-/i) || normalized.match(/[?&]tid=(\d+)/i); return match ? match[1] : normalized; } function normalizeTitle(title) { return String(title || '').replace(/\s+/g, ' ').trim(); } function normalizeMagnets(magnets) { var seen = Object.create(null); var result = []; if (!Array.isArray(magnets)) { return result; } magnets.forEach(function(magnet) { if (typeof magnet !== 'string' || !magnet) { return; } if (seen[magnet]) { return; } seen[magnet] = true; result.push(magnet); }); return result; } function normalizeThreadEntry(thread, forumKey, crawledAt) { if (!thread || typeof thread !== 'object') { return null; } var normalizedUrl = normalizeUrl(thread.url); var normalizedTitle = normalizeTitle(thread.title); var threadKey = typeof thread.threadKey === 'string' && thread.threadKey ? thread.threadKey : getThreadKeyFromUrl(normalizedUrl); if (!forumKey || !normalizedUrl || !threadKey) { return null; } return { cacheKey: forumKey + '::' + threadKey, forumKey: forumKey, threadKey: threadKey, url: normalizedUrl, title: normalizedTitle, titleNorm: normalizedTitle.toLowerCase(), lastSeenAt: crawledAt, firstSeenAt: crawledAt, magnets: normalizeMagnets(thread.magnets), magnetCount: Array.isArray(thread.magnets) ? normalizeMagnets(thread.magnets).length : 0, lastMagnetSyncAt: Array.isArray(thread.magnets) && thread.magnets.length > 0 ? crawledAt : 0 }; } function getCoverageKey(forumKey, startPage, endPage) { return forumKey + '::' + startPage + '-' + endPage; } function getPageCoverageKey(forumKey, page) { return forumKey + '::page::' + page; } function normalizeThreadsForCoverage(rawThreads, forumKey, crawledAt) { var seen = Object.create(null); var normalizedThreads = []; (Array.isArray(rawThreads) ? rawThreads : []).forEach(function(thread) { var normalized = normalizeThreadEntry(thread, forumKey, crawledAt); if (!normalized || seen[normalized.cacheKey]) { return; } seen[normalized.cacheKey] = true; normalizedThreads.push({ threadKey: normalized.threadKey, url: normalized.url, title: normalized.title }); }); return normalizedThreads; } function mergeCoverageThreadLists(forumKey, crawledAt, primaryThreads, secondaryThreads) { var seen = Object.create(null); var merged = []; [primaryThreads, secondaryThreads].forEach(function(source) { normalizeThreadsForCoverage(source, forumKey, crawledAt).forEach(function(thread) { var key = String(thread.threadKey || ''); if (!key || seen[key]) { return; } seen[key] = true; merged.push({ threadKey: thread.threadKey, url: thread.url, title: thread.title }); }); }); return merged; } function normalizeThreadsForCloudMagnets(rawThreads, forumKey, crawledAt) { var seen = Object.create(null); var normalizedThreads = []; (Array.isArray(rawThreads) ? rawThreads : []).forEach(function(thread) { var normalized = normalizeThreadEntry(thread, forumKey, crawledAt); if (!normalized || seen[normalized.cacheKey]) { return; } var magnets = normalizeMagnets(thread.magnets); if (magnets.length === 0) { return; } seen[normalized.cacheKey] = true; normalizedThreads.push({ threadKey: normalized.threadKey, url: normalized.url, title: normalized.title, magnets: magnets, lastSeenAt: crawledAt }); }); return normalizedThreads; } function normalizeThreadsForCloudIndex(rawThreads, forumKey, crawledAt) { var seen = Object.create(null); var normalizedThreads = []; (Array.isArray(rawThreads) ? rawThreads : []).forEach(function(thread) { var normalized = normalizeThreadEntry(thread, forumKey, crawledAt); if (!normalized || seen[normalized.cacheKey]) { return; } seen[normalized.cacheKey] = true; normalizedThreads.push({ threadKey: normalized.threadKey, url: normalized.url, title: normalized.title, magnets: normalizeMagnets(thread.magnets), lastSeenAt: Number(thread.lastSeenAt) || crawledAt }); }); return normalizedThreads; } function normalizeCloudCoveragePayload(coverage) { if (!coverage || typeof coverage !== 'object') { return null; } var forumKey = typeof coverage.forumKey === 'string' ? coverage.forumKey : ''; var startPage = Math.max(1, Number(coverage.startPage) || 1); var endPage = Math.max(startPage, Number(coverage.endPage) || startPage); if (!forumKey) { return null; } return { forumKey: forumKey, startPage: startPage, endPage: endPage, crawledAt: Number(coverage.crawledAt) || Date.now(), strategy: typeof coverage.strategy === 'string' && coverage.strategy ? coverage.strategy : 'full_live', frontRefreshPages: Math.max(0, Number(coverage.frontRefreshPages) || 0), threads: normalizeThreadsForCoverage(coverage.threads, forumKey, Number(coverage.crawledAt) || Date.now()) }; } function normalizeCloudPageCoveragePayload(coverage) { if (!coverage || typeof coverage !== 'object') { return null; } var forumKey = typeof coverage.forumKey === 'string' ? coverage.forumKey : ''; var page = Math.max(1, Number(coverage.page) || 1); if (!forumKey) { return null; } return { forumKey: forumKey, page: page, crawledAt: Number(coverage.crawledAt) || Date.now(), threads: normalizeThreadsForCoverage(coverage.threads, forumKey, Number(coverage.crawledAt) || Date.now()) }; } function normalizeCloudThreadMagnetsPayload(result) { if (!result || typeof result !== 'object') { return null; } var forumKey = typeof result.forumKey === 'string' ? result.forumKey : ''; var threadKey = typeof result.threadKey === 'string' ? result.threadKey : getThreadKeyFromUrl(result.url); var url = normalizeUrl(result.url); var magnets = normalizeMagnets(result.magnets); if (!forumKey || !threadKey || !url || magnets.length === 0) { return null; } return { forumKey: forumKey, threadKey: threadKey, url: url, title: normalizeTitle(result.title), magnets: magnets, lastSeenAt: Number(result.lastSeenAt) || Date.now() }; } async function cloudLookupThreadMagnets(payload) { var response = await callCloudCache('/api/shared-cache/threads/lookup', { threads: normalizeThreadsForCoverage(payload.threads, payload.forumKey, Date.now()).map(function(item) { return { forumKey: payload.forumKey, threadKey: item.threadKey }; }) }); if (!response || !response.ok || !Array.isArray(response.threads)) { return []; } return response.threads.map(function(item) { return normalizeCloudThreadMagnetsPayload({ forumKey: payload.forumKey, threadKey: item.threadKey, url: item.url, title: item.title, magnets: item.magnets, lastSeenAt: item.lastSeenAt }); }).filter(Boolean); } async function cloudUpsertThreadMagnets(payload) { var allThreads = normalizeThreadsForCloudIndex(payload.threads, payload.forumKey, Number(payload.syncedAt) || Date.now()).map(function(item) { return { forumKey: payload.forumKey, threadKey: item.threadKey, url: item.url, title: item.title, magnets: item.magnets, lastSeenAt: item.lastSeenAt }; }); if (allThreads.length === 0) { return { ok: true, savedCount: 0 }; } return callCloudCache('/api/shared-cache/threads/upsert', { threads: allThreads }, CLOUD_CACHE_TIMEOUT, { 'x-shared-cache-write-token': CLOUD_SHARED_CACHE_WRITE_TOKEN }); } async function cloudLookupCoverageByStrategy(forumKey, startPage, endPage, strategy) { var response = await callCloudCache('/api/shared-cache/coverages/lookup', { forumKey: forumKey, startPage: startPage, endPage: endPage, strategy: strategy }); if (!response || !response.ok || !response.coverage) { return null; } return normalizeCloudCoveragePayload(response.coverage); } async function cloudLookupCoverage(payload) { var strategies = ['full_live', 'exact_cache', 'smart_incremental', 'assembled_pages']; for (var index = 0; index < strategies.length; index++) { var coverage = await cloudLookupCoverageByStrategy(payload.forumKey, payload.startPage, payload.endPage, strategies[index]); if (coverage) { return coverage; } } return null; } async function cloudLookupCoveragePlan(payload) { var response = await callCloudCache('/api/shared-cache/coverages/plan', { forumKey: payload.forumKey, startPage: payload.startPage, endPage: payload.endPage, frontRefreshPages: payload.frontRefreshPages }); if (!response || response.ok === false) { return null; } return response; } async function cloudUpsertCoverage(payload) { return callCloudCache('/api/shared-cache/coverages/upsert', { forumKey: payload.forumKey, startPage: payload.startPage, endPage: payload.endPage, strategy: payload.strategy, crawledAt: payload.crawledAt, threads: normalizeThreadsForCoverage(payload.threads, payload.forumKey, payload.crawledAt) }, CLOUD_CACHE_TIMEOUT, { 'x-shared-cache-write-token': CLOUD_SHARED_CACHE_WRITE_TOKEN }); } async function cloudLookupPageCoverage(payload) { var response = await callCloudCache('/api/shared-cache/pages/lookup', { forumKey: payload.forumKey, page: payload.page }); if (!response || !response.ok || !response.coverage) { return null; } return normalizeCloudPageCoveragePayload(response.coverage); } async function cloudUpsertPageCoverage(payload) { return callCloudCache('/api/shared-cache/pages/upsert', { forumKey: payload.forumKey, page: payload.page, crawledAt: payload.crawledAt, threads: normalizeThreadsForCoverage(payload.threads, payload.forumKey, payload.crawledAt) }, CLOUD_CACHE_TIMEOUT, { 'x-shared-cache-write-token': CLOUD_SHARED_CACHE_WRITE_TOKEN }); } async function getCloudAssembledPageCoverage(forumKey, startPage, endPage) { var requests = []; for (var page = startPage; page <= endPage; page++) { requests.push(cloudLookupPageCoverage({ forumKey: forumKey, page: page })); } var results = await Promise.all(requests); if (results.some(function(item) { return !item; })) { return null; } var merged = Object.create(null); var threads = []; var latestCrawledAt = 0; results.forEach(function(item) { latestCrawledAt = Math.max(latestCrawledAt, Number(item.crawledAt || 0)); (Array.isArray(item.threads) ? item.threads : []).forEach(function(thread) { var key = typeof thread.threadKey === 'string' && thread.threadKey ? thread.threadKey : getThreadKeyFromUrl(thread.url); if (!key || merged[key]) { return; } merged[key] = true; threads.push({ threadKey: key, url: normalizeUrl(thread.url), title: normalizeTitle(thread.title) }); }); }); return { forumKey: forumKey, startPage: startPage, endPage: endPage, crawledAt: latestCrawledAt || Date.now(), strategy: 'assembled_pages', frontRefreshPages: 0, threads: threads }; } async function upsertThreadEntries(threadStore, forumKey, rawThreads, crawledAt) { var seenThreadKeys = Object.create(null); var threadKeys = []; for (var threadIndex = 0; threadIndex < rawThreads.length; threadIndex++) { var thread = rawThreads[threadIndex]; var normalized = normalizeThreadEntry(thread, forumKey, crawledAt); if (!normalized || seenThreadKeys[normalized.cacheKey]) { continue; } seenThreadKeys[normalized.cacheKey] = true; threadKeys.push(normalized.cacheKey); var existingRecord = await requestToPromise(threadStore.get(normalized.cacheKey)); var incomingMagnets = normalizeMagnets(normalized.magnets); var mergedMagnets = incomingMagnets.length > 0 ? incomingMagnets : (existingRecord && Array.isArray(existingRecord.magnets) ? existingRecord.magnets : []); var mergedLastMagnetSyncAt = incomingMagnets.length > 0 ? (Number(normalized.lastMagnetSyncAt) || crawledAt) : (existingRecord && Number(existingRecord.lastMagnetSyncAt) ? Number(existingRecord.lastMagnetSyncAt) : (normalized.lastMagnetSyncAt || 0)); threadStore.put({ cacheKey: normalized.cacheKey, forumKey: normalized.forumKey, threadKey: normalized.threadKey, url: normalized.url, title: normalized.title, titleNorm: normalized.titleNorm, firstSeenAt: existingRecord && Number(existingRecord.firstSeenAt) ? Number(existingRecord.firstSeenAt) : normalized.firstSeenAt, lastSeenAt: normalized.lastSeenAt, magnets: normalizeMagnets(mergedMagnets), magnetCount: normalizeMagnets(mergedMagnets).length, lastMagnetSyncAt: mergedLastMagnetSyncAt }); } return threadKeys; } async function getAssembledPageCoverage(db, forumKey, startPage, endPage) { var pageTx = db.transaction('pageCoverages', 'readonly'); var pageStore = pageTx.objectStore('pageCoverages'); var pageRequests = []; for (var page = startPage; page <= endPage; page++) { pageRequests.push(requestToPromise(pageStore.get(getPageCoverageKey(forumKey, page)))); } var pageRecords = await Promise.all(pageRequests); await transactionDone(pageTx); if (pageRecords.some(function(record) { return !record; })) { return null; } var merged = Object.create(null); var mergedThreadKeys = []; var latestCrawledAt = 0; pageRecords.forEach(function(record) { latestCrawledAt = Math.max(latestCrawledAt, Number(record.crawledAt || 0)); (Array.isArray(record.threadKeys) ? record.threadKeys : []).forEach(function(threadKey) { if (merged[threadKey]) { return; } merged[threadKey] = true; mergedThreadKeys.push(threadKey); }); }); return getCoverageWithThreads(db, { forumKey: forumKey, startPage: startPage, endPage: endPage, crawledAt: latestCrawledAt, strategy: 'assembled_pages', frontRefreshPages: 0, threadKeys: mergedThreadKeys }); } async function getCachedPageCoverageBlocks(db, forumKey, startPage, endPage) { var tx = db.transaction('pageCoverages', 'readonly'); var store = tx.objectStore('pageCoverages'); var index = store.index('forumKey'); var request = index.openCursor(IDBKeyRange.only(forumKey)); var pageRecords = []; var records = await new Promise(function(resolve, reject) { request.onsuccess = function(event) { var cursor = event.target.result; var record = null; if (!cursor) { resolve(pageRecords); return; } record = cursor.value; if (record && Number(record.page) >= startPage && Number(record.page) <= endPage) { pageRecords.push(record); } cursor.continue(); }; request.onerror = function(event) { reject(event.target.error || new Error('读取页缓存块失败')); }; }); await transactionDone(tx); if (!records.length) { return []; } records.sort(function(a, b) { return Number(a.page || 0) - Number(b.page || 0); }); var blocks = []; var currentBlock = null; var latestCrawledAt = 0; var mergedThreadKeys = null; records.forEach(function(record) { var page = Number(record.page || 0); if (!currentBlock) { currentBlock = { forumKey: forumKey, startPage: page, endPage: page, pages: [record], crawledAt: Number(record.crawledAt || 0) }; return; } if (page === currentBlock.endPage + 1) { currentBlock.endPage = page; currentBlock.pages.push(record); currentBlock.crawledAt = Math.max(currentBlock.crawledAt, Number(record.crawledAt || 0)); return; } blocks.push(currentBlock); currentBlock = { forumKey: forumKey, startPage: page, endPage: page, pages: [record], crawledAt: Number(record.crawledAt || 0) }; }); if (currentBlock) { blocks.push(currentBlock); } return blocks.map(function(block) { var merged = Object.create(null); mergedThreadKeys = []; latestCrawledAt = 0; block.pages.forEach(function(record) { latestCrawledAt = Math.max(latestCrawledAt, Number(record.crawledAt || 0)); (Array.isArray(record.threadKeys) ? record.threadKeys : []).forEach(function(threadKey) { if (merged[threadKey]) { return; } merged[threadKey] = true; mergedThreadKeys.push(threadKey); }); }); return { forumKey: forumKey, startPage: block.startPage, endPage: block.endPage, crawledAt: latestCrawledAt, strategy: 'assembled_pages', frontRefreshPages: 0, threadKeys: mergedThreadKeys }; }); } async function getCoverageWithThreads(db, coverageRecord) { if (!coverageRecord) { return null; } var threadKeys = Array.isArray(coverageRecord.threadKeys) ? coverageRecord.threadKeys : []; var threads = []; if (threadKeys.length > 0) { var threadTx = db.transaction('threads', 'readonly'); var threadStore = threadTx.objectStore('threads'); var threadRequests = threadKeys.map(function(threadKey) { return requestToPromise(threadStore.get(threadKey)); }); var threadResults = await Promise.all(threadRequests); await transactionDone(threadTx); threads = threadResults.filter(function(record) { return !!record; }).map(function(record) { return { url: record.url, title: record.title, threadKey: record.threadKey }; }); } return { forumKey: coverageRecord.forumKey, startPage: coverageRecord.startPage, endPage: coverageRecord.endPage, crawledAt: coverageRecord.crawledAt, strategy: coverageRecord.strategy || 'full_live', frontRefreshPages: coverageRecord.frontRefreshPages || 0, threads: threads }; } async function listForumCoverages(db, forumKey) { return new Promise(function(resolve, reject) { var tx = db.transaction('coverages', 'readonly'); var store = tx.objectStore('coverages'); var index = store.index('forumKey'); var request = index.openCursor(IDBKeyRange.only(forumKey)); var results = []; request.onsuccess = function(event) { var cursor = event.target.result; if (!cursor) { resolve(results); return; } results.push(cursor.value); cursor.continue(); }; request.onerror = function(event) { reject(event.target.error || new Error('读取覆盖快照失败')); }; }); } async function getIntersectingCoverageBlocks(db, forumKey, startPage, endPage) { var coverages = await listForumCoverages(db, forumKey); var filtered = coverages.filter(function(record) { var recordStart = Number(record.startPage || 0); var recordEnd = Number(record.endPage || 0); return recordStart <= endPage && recordEnd >= startPage; }).sort(function(a, b) { var aStart = Number(a.startPage || 0); var bStart = Number(b.startPage || 0); if (aStart !== bStart) { return aStart - bStart; } return Number(b.endPage || 0) - Number(a.endPage || 0); }); var result = []; var seenRanges = Object.create(null); var index = 0; var record = null; var hydrated = null; var clippedStart = 0; var clippedEnd = 0; var rangeKey = ''; for (index = 0; index < filtered.length; index++) { record = filtered[index]; hydrated = await getCoverageWithThreads(db, record); if (!hydrated || !Array.isArray(hydrated.threads) || hydrated.threads.length === 0) { continue; } clippedStart = Math.max(startPage, Number(hydrated.startPage || startPage)); clippedEnd = Math.min(endPage, Number(hydrated.endPage || endPage)); if (clippedStart > clippedEnd) { continue; } rangeKey = String(clippedStart) + '-' + String(clippedEnd); if (seenRanges[rangeKey]) { continue; } seenRanges[rangeKey] = true; result.push({ forumKey: hydrated.forumKey, startPage: clippedStart, endPage: clippedEnd, crawledAt: hydrated.crawledAt, strategy: hydrated.strategy || 'full_live', frontRefreshPages: hydrated.frontRefreshPages || 0, threads: hydrated.threads }); } return result; } async function pruneOldCoverages(db, forumKey) { var coverages = await listForumCoverages(db, forumKey); if (coverages.length <= COVERAGE_LIMIT_PER_FORUM) { return; } coverages.sort(function(a, b) { return Number(b.crawledAt || 0) - Number(a.crawledAt || 0); }); var stale = coverages.slice(COVERAGE_LIMIT_PER_FORUM); if (stale.length === 0) { return; } var tx = db.transaction('coverages', 'readwrite'); var store = tx.objectStore('coverages'); stale.forEach(function(record) { store.delete(record.coverageKey); }); await transactionDone(tx); } async function saveCoverageSnapshot(payload) { var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : ''; var startPage = Math.max(1, Number(payload.startPage) || 1); var endPage = Math.max(startPage, Number(payload.endPage) || startPage); var crawledAt = Number(payload.crawledAt) || Date.now(); var frontRefreshPages = Math.max(0, Number(payload.frontRefreshPages) || 0); var strategy = typeof payload.strategy === 'string' ? payload.strategy : 'full_live'; var skipCloudSync = !!payload.skipCloudSync; var rawThreads = Array.isArray(payload.threads) ? payload.threads : []; var normalizedThreadsForCoverage = normalizeThreadsForCoverage(rawThreads, forumKey, crawledAt); var intersectingBlocks = null; var historicalThreads = []; var mergedCoverageThreads = null; var result = null; var db = await openCacheDb(); try { intersectingBlocks = await getIntersectingCoverageBlocks(db, forumKey, startPage, endPage); historicalThreads = []; intersectingBlocks.forEach(function(block) { (Array.isArray(block.threads) ? block.threads : []).forEach(function(thread) { historicalThreads.push(thread); }); }); mergedCoverageThreads = mergeCoverageThreadLists(forumKey, crawledAt, rawThreads, historicalThreads); normalizedThreadsForCoverage = normalizeThreadsForCoverage(mergedCoverageThreads, forumKey, crawledAt); var tx = db.transaction(['threads', 'coverages'], 'readwrite'); var threadStore = tx.objectStore('threads'); var coverageStore = tx.objectStore('coverages'); var threadKeys = await upsertThreadEntries(threadStore, forumKey, mergedCoverageThreads, crawledAt); coverageStore.put({ coverageKey: getCoverageKey(forumKey, startPage, endPage), forumKey: forumKey, startPage: startPage, endPage: endPage, crawledAt: crawledAt, strategy: strategy, frontRefreshPages: frontRefreshPages, threadKeys: threadKeys, threadCount: threadKeys.length }); await transactionDone(tx); await pruneOldCoverages(db, forumKey); result = { ok: true, threadCount: threadKeys.length, coverageKey: getCoverageKey(forumKey, startPage, endPage) }; } finally { db.close(); } if (!skipCloudSync && forumKey && normalizedThreadsForCoverage.length > 0) { try { var coverageThreadUploadPlan = await filterThreadUploadsForCloud( forumKey, normalizeThreadsForCloudIndex(mergedCoverageThreads, forumKey, crawledAt), { ttlMs: CLOUD_THREAD_UPLOAD_TTL_MS } ); if (coverageThreadUploadPlan.uploads.length > 0) { var threadUpsertResult = await cloudUpsertThreadMagnets({ forumKey: forumKey, syncedAt: crawledAt, threads: coverageThreadUploadPlan.uploads }); if (threadUpsertResult && threadUpsertResult.ok) { await markUploadMetaEntries('thread', coverageThreadUploadPlan.pendingMeta); } } var coverageUploadCheck = await shouldUploadCoverageToCloud({ forumKey: forumKey, startPage: startPage, endPage: endPage, strategy: strategy, crawledAt: crawledAt, threads: normalizedThreadsForCoverage }); if (coverageUploadCheck.shouldUpload) { var coverageUpsertResult = await cloudUpsertCoverage({ forumKey: forumKey, startPage: startPage, endPage: endPage, strategy: strategy, crawledAt: crawledAt, threads: normalizedThreadsForCoverage }); if (coverageUpsertResult && coverageUpsertResult.ok) { await markUploadMetaEntries('coverage', [coverageUploadCheck]); } } } catch (error) { console.warn('[MagnetPlugin][Cloud] save coverage failed:', error && error.message ? error.message : error); } backfillLocalCacheToCloud({ force: false }).catch(function(backfillErr) { console.warn('[MagnetPlugin][Cloud] post-coverage backfill failed:', backfillErr && backfillErr.message ? backfillErr.message : backfillErr); }); } return result; } async function savePageCoverageSnapshot(payload) { var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : ''; var page = Math.max(1, Number(payload.page) || 1); var crawledAt = Number(payload.crawledAt) || Date.now(); var skipCloudSync = !!payload.skipCloudSync; var rawThreads = Array.isArray(payload.threads) ? payload.threads : []; var normalizedThreadsForCoverage = normalizeThreadsForCoverage(rawThreads, forumKey, crawledAt); var result = null; var db = await openCacheDb(); try { var tx = db.transaction(['threads', 'pageCoverages'], 'readwrite'); var threadStore = tx.objectStore('threads'); var pageCoverageStore = tx.objectStore('pageCoverages'); var threadKeys = await upsertThreadEntries(threadStore, forumKey, rawThreads, crawledAt); pageCoverageStore.put({ coverageKey: getPageCoverageKey(forumKey, page), forumKey: forumKey, page: page, crawledAt: crawledAt, threadKeys: threadKeys, threadCount: threadKeys.length }); await transactionDone(tx); result = { ok: true, page: page, threadCount: threadKeys.length, coverageKey: getPageCoverageKey(forumKey, page) }; } finally { db.close(); } if (!skipCloudSync && forumKey && normalizedThreadsForCoverage.length > 0) { try { var pageThreadUploadPlan = await filterThreadUploadsForCloud( forumKey, normalizeThreadsForCloudIndex(rawThreads, forumKey, crawledAt), { ttlMs: CLOUD_THREAD_UPLOAD_TTL_MS } ); if (pageThreadUploadPlan.uploads.length > 0) { var pageThreadUpsertResult = await cloudUpsertThreadMagnets({ forumKey: forumKey, syncedAt: crawledAt, threads: pageThreadUploadPlan.uploads }); if (pageThreadUpsertResult && pageThreadUpsertResult.ok) { await markUploadMetaEntries('thread', pageThreadUploadPlan.pendingMeta); } } var pageUploadCheck = await shouldUploadPageToCloud({ forumKey: forumKey, page: page, crawledAt: crawledAt, threads: normalizedThreadsForCoverage }); if (pageUploadCheck.shouldUpload) { var pageUpsertResult = await cloudUpsertPageCoverage({ forumKey: forumKey, page: page, crawledAt: crawledAt, threads: normalizedThreadsForCoverage }); if (pageUpsertResult && pageUpsertResult.ok) { await markUploadMetaEntries('page', [pageUploadCheck]); } } } catch (error) { console.warn('[MagnetPlugin][Cloud] save page coverage failed:', error && error.message ? error.message : error); } } return result; } async function getCachedThreadMagnets(payload) { var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : ''; var rawThreads = Array.isArray(payload.threads) ? payload.threads : []; var db = await openCacheDb(); var misses = []; var resultMap = Object.create(null); try { var tx = db.transaction('threads', 'readonly'); var threadStore = tx.objectStore('threads'); var results = []; for (var threadIndex = 0; threadIndex < rawThreads.length; threadIndex++) { var thread = rawThreads[threadIndex]; var normalized = normalizeThreadEntry(thread, forumKey, Date.now()); if (!normalized) { continue; } var record = await requestToPromise(threadStore.get(normalized.cacheKey)); if (!record) { results.push({ threadKey: normalized.threadKey, url: normalized.url, title: normalized.title, magnets: [] }); misses.push({ threadKey: normalized.threadKey, url: normalized.url, title: normalized.title }); continue; } var recordMagnets = normalizeMagnets(record.magnets); if (recordMagnets.length === 0 && !Number(record.lastMagnetSyncAt)) { misses.push({ threadKey: record.threadKey, url: record.url, title: record.title }); } results.push({ threadKey: record.threadKey, url: record.url, title: record.title, magnets: recordMagnets, lastMagnetSyncAt: Number(record.lastMagnetSyncAt) || 0 }); } await transactionDone(tx); results.forEach(function(item) { if (item && item.threadKey) { resultMap[item.threadKey] = item; } }); if (misses.length > 0) { try { var cloudThreads = await cloudLookupThreadMagnets({ forumKey: forumKey, threads: misses }); if (cloudThreads.length > 0) { await saveThreadMagnets({ forumKey: forumKey, syncedAt: Date.now(), threads: cloudThreads, skipCloudSync: true }); cloudThreads.forEach(function(item) { if (!item || !item.threadKey) { return; } resultMap[item.threadKey] = { threadKey: item.threadKey, url: item.url, title: item.title, magnets: normalizeMagnets(item.magnets), lastMagnetSyncAt: Number(item.lastSeenAt) || Date.now() }; }); } } catch (error) { console.warn('[MagnetPlugin][Cloud] lookup thread magnets failed:', error && error.message ? error.message : error); } } return { ok: true, threads: results.map(function(item) { return item && item.threadKey && resultMap[item.threadKey] ? resultMap[item.threadKey] : item; }) }; } finally { db.close(); } } async function saveThreadMagnets(payload) { var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : ''; var rawThreads = Array.isArray(payload.threads) ? payload.threads : []; var syncedAt = Number(payload.syncedAt) || Date.now(); var skipCloudSync = !!payload.skipCloudSync; var cloudThreads = []; var result = null; var db = await openCacheDb(); try { var tx = db.transaction('threads', 'readwrite'); var threadStore = tx.objectStore('threads'); var savedCount = 0; for (var threadIndex = 0; threadIndex < rawThreads.length; threadIndex++) { var thread = rawThreads[threadIndex]; var normalized = normalizeThreadEntry(thread, forumKey, syncedAt); if (!normalized) { continue; } var existingRecord = await requestToPromise(threadStore.get(normalized.cacheKey)); var normalizedMagnets = normalizeMagnets(thread.magnets); var finalMagnets = normalizedMagnets.length > 0 ? normalizedMagnets : (existingRecord && Array.isArray(existingRecord.magnets) ? normalizeMagnets(existingRecord.magnets) : []); threadStore.put({ cacheKey: normalized.cacheKey, forumKey: normalized.forumKey, threadKey: normalized.threadKey, url: normalized.url, title: normalized.title || (existingRecord ? existingRecord.title : ''), titleNorm: normalizeTitle(normalized.title || (existingRecord ? existingRecord.title : '')).toLowerCase(), firstSeenAt: existingRecord && Number(existingRecord.firstSeenAt) ? Number(existingRecord.firstSeenAt) : normalized.firstSeenAt, lastSeenAt: Math.max(Number(existingRecord && existingRecord.lastSeenAt || 0), Number(normalized.lastSeenAt || 0)) || Date.now(), magnets: finalMagnets, magnetCount: finalMagnets.length, lastMagnetSyncAt: finalMagnets.length > 0 ? syncedAt : (existingRecord && Number(existingRecord.lastMagnetSyncAt) ? Number(existingRecord.lastMagnetSyncAt) : 0) }); if (finalMagnets.length > 0) { cloudThreads.push({ threadKey: normalized.threadKey, url: normalized.url, title: normalized.title || (existingRecord ? existingRecord.title : ''), magnets: finalMagnets, lastSeenAt: syncedAt }); } savedCount += 1; } await transactionDone(tx); result = { ok: true, savedCount: savedCount }; } finally { db.close(); } if (!skipCloudSync && forumKey && cloudThreads.length > 0) { try { var threadUploadPlan = await filterThreadUploadsForCloud(forumKey, cloudThreads, { ttlMs: CLOUD_THREAD_UPLOAD_TTL_MS }); if (threadUploadPlan.uploads.length > 0) { var directThreadUpsertResult = await cloudUpsertThreadMagnets({ forumKey: forumKey, syncedAt: syncedAt, threads: threadUploadPlan.uploads }); if (directThreadUpsertResult && directThreadUpsertResult.ok) { await markUploadMetaEntries('thread', threadUploadPlan.pendingMeta); } } } catch (error) { console.warn('[MagnetPlugin][Cloud] save thread magnets failed:', error && error.message ? error.message : error); } backfillLocalCacheToCloud({ force: false }).catch(function(backfillErr) { console.warn('[MagnetPlugin][Cloud] post-thread-save backfill failed:', backfillErr && backfillErr.message ? backfillErr.message : backfillErr); }); } return result; } function estimateTextSize(value) { try { return JSON.stringify(value).length; } catch (e) { return 0; } } async function getStorageEstimate() { if (!navigator.storage || !navigator.storage.estimate) { return null; } try { return await navigator.storage.estimate(); } catch (e) { return null; } } async function readRecentThreadRecords(db, forumKey, limit) { var tx = db.transaction('threads', 'readonly'); var store = tx.objectStore('threads'); var index = forumKey ? store.index('forumKeyLastSeenAt') : store.index('lastSeenAt'); var range = forumKey ? IDBKeyRange.bound([forumKey, 0], [forumKey, Number.MAX_SAFE_INTEGER]) : null; var request = index.openCursor(range, 'prev'); return new Promise(function(resolve, reject) { var items = []; request.onsuccess = function(event) { var cursor = event.target.result; if (!cursor || items.length >= limit) { resolve(items); return; } items.push(cursor.value); cursor.continue(); }; request.onerror = function(event) { reject(event.target.error || new Error('读取最近帖子缓存失败')); }; }).finally(function() { return transactionDone(tx).catch(function() { return null; }); }); } async function readRecentCoverageRecords(db, forumKey, limit) { var tx = db.transaction('coverages', 'readonly'); var store = tx.objectStore('coverages'); var index = forumKey ? store.index('forumKeyCrawledAt') : store.index('crawledAt'); var range = forumKey ? IDBKeyRange.bound([forumKey, 0], [forumKey, Number.MAX_SAFE_INTEGER]) : null; var request = index.openCursor(range, 'prev'); return new Promise(function(resolve, reject) { var items = []; request.onsuccess = function(event) { var cursor = event.target.result; if (!cursor || items.length >= limit) { resolve(items); return; } items.push(cursor.value); cursor.continue(); }; request.onerror = function(event) { reject(event.target.error || new Error('读取最近范围快照失败')); }; }).finally(function() { return transactionDone(tx).catch(function() { return null; }); }); } async function readAllStoreRecords(db, storeName) { return new Promise(function(resolve, reject) { var tx = db.transaction(storeName, 'readonly'); var store = tx.objectStore(storeName); var request = store.openCursor(); var items = []; request.onsuccess = function(event) { var cursor = event.target.result; if (!cursor) { resolve(items); return; } items.push(cursor.value); cursor.continue(); }; request.onerror = function(event) { reject(event.target.error || new Error('读取 ' + storeName + ' 失败')); }; }); } async function getThreadsByKeys(db, threadKeys) { var keys = Array.isArray(threadKeys) ? threadKeys.filter(Boolean) : []; var threadTx = null; var threadStore = null; var threadRequests = null; var threadResults = null; if (keys.length === 0) { return []; } threadTx = db.transaction('threads', 'readonly'); threadStore = threadTx.objectStore('threads'); threadRequests = keys.map(function(threadKey) { return requestToPromise(threadStore.get(threadKey)); }); threadResults = await Promise.all(threadRequests); await transactionDone(threadTx); return threadResults.filter(Boolean).map(function(record) { return { threadKey: record.threadKey, url: record.url, title: record.title, magnets: normalizeMagnets(record.magnets), lastSeenAt: Number(record.lastSeenAt) || Date.now() }; }); } function chunkArray(items, size) { var chunks = []; var index = 0; var safeSize = Math.max(1, Number(size) || 1); for (index = 0; index < items.length; index += safeSize) { chunks.push(items.slice(index, index + safeSize)); } return chunks; } async function getCloudBackfillState() { var result = await storageLocalGet(CLOUD_CACHE_BACKFILL_KEY); return result[CLOUD_CACHE_BACKFILL_KEY] || null; } async function saveCloudBackfillState(state) { var data = {}; data[CLOUD_CACHE_BACKFILL_KEY] = state; await storageLocalSet(data); } function normalizeCloudUploadMetaState(state) { state = state && typeof state === 'object' ? state : {}; return { threads: state.threads && typeof state.threads === 'object' ? state.threads : {}, coverages: state.coverages && typeof state.coverages === 'object' ? state.coverages : {}, pages: state.pages && typeof state.pages === 'object' ? state.pages : {} }; } async function loadCloudUploadMetaState() { var result = null; if (cloudUploadMetaCache) { return cloudUploadMetaCache; } result = await storageLocalGet(CLOUD_UPLOAD_META_KEY); cloudUploadMetaCache = normalizeCloudUploadMetaState(result[CLOUD_UPLOAD_META_KEY]); return cloudUploadMetaCache; } function pruneUploadMetaStore(store, limit) { var keys = Object.keys(store || {}); var safeLimit = Math.max(100, Number(limit) || 1000); var staleKeys = null; if (keys.length <= safeLimit) { return store; } staleKeys = keys.sort(function(a, b) { return Number((store[a] && store[a].lastUploadedAt) || 0) - Number((store[b] && store[b].lastUploadedAt) || 0); }).slice(0, keys.length - safeLimit); staleKeys.forEach(function(key) { delete store[key]; }); return store; } async function saveCloudUploadMetaState(state) { var data = {}; var normalized = normalizeCloudUploadMetaState(state); pruneUploadMetaStore(normalized.threads, CLOUD_UPLOAD_META_THREAD_LIMIT); pruneUploadMetaStore(normalized.coverages, CLOUD_UPLOAD_META_OTHER_LIMIT); pruneUploadMetaStore(normalized.pages, CLOUD_UPLOAD_META_OTHER_LIMIT); cloudUploadMetaCache = normalized; data[CLOUD_UPLOAD_META_KEY] = normalized; await storageLocalSet(data); return normalized; } function getUploadMetaBucket(state, type) { if (type === 'thread') { return state.threads; } if (type === 'coverage') { return state.coverages; } return state.pages; } function buildThreadUploadMetaKey(forumKey, threadKey) { return String(forumKey || '') + '::' + String(threadKey || ''); } function buildCoverageUploadMetaKey(forumKey, startPage, endPage, strategy) { return String(forumKey || '') + '::' + String(startPage || 1) + '-' + String(endPage || 1) + '::' + String(strategy || 'full_live'); } function buildPageUploadMetaKey(forumKey, page) { return String(forumKey || '') + '::page::' + String(page || 1); } function shouldSkipUploadMetaEntry(entry, payloadHash, ttlMs, now) { return !!( entry && entry.payloadHash === payloadHash && Number(entry.lastUploadedAt || 0) > 0 && now - Number(entry.lastUploadedAt || 0) < ttlMs ); } function sortThreadsForHash(threads) { return (Array.isArray(threads) ? threads : []).map(function(thread) { return { threadKey: String(thread.threadKey || ''), url: normalizeUrl(thread.url), title: normalizeTitle(thread.title), magnets: normalizeMagnets(thread.magnets).slice().sort() }; }).sort(function(a, b) { return String(a.threadKey).localeCompare(String(b.threadKey)); }); } async function buildThreadUploadHash(forumKey, thread) { return sha256Hex(JSON.stringify({ forumKey: String(forumKey || ''), threadKey: String(thread.threadKey || ''), url: normalizeUrl(thread.url), title: normalizeTitle(thread.title), magnets: normalizeMagnets(thread.magnets).slice().sort() })); } async function buildCoverageUploadHash(payload) { return sha256Hex(JSON.stringify({ forumKey: String(payload.forumKey || ''), startPage: Number(payload.startPage || 1), endPage: Number(payload.endPage || payload.startPage || 1), strategy: String(payload.strategy || 'full_live'), threads: sortThreadsForHash(payload.threads) })); } async function buildPageUploadHash(payload) { return sha256Hex(JSON.stringify({ forumKey: String(payload.forumKey || ''), page: Number(payload.page || 1), threads: sortThreadsForHash(payload.threads) })); } async function filterThreadUploadsForCloud(forumKey, threads, options) { var state = await loadCloudUploadMetaState(); var bucket = getUploadMetaBucket(state, 'thread'); var now = Date.now(); var ttlMs = options && Number(options.ttlMs) >= 0 ? Number(options.ttlMs) : CLOUD_THREAD_UPLOAD_TTL_MS; var uploads = []; var pendingMeta = []; var index = 0; var item = null; var key = ''; var payloadHash = ''; for (index = 0; index < threads.length; index++) { item = threads[index]; if (!item || !item.threadKey) { continue; } key = buildThreadUploadMetaKey(forumKey, item.threadKey); payloadHash = await buildThreadUploadHash(forumKey, item); if (shouldSkipUploadMetaEntry(bucket[key], payloadHash, ttlMs, now)) { continue; } uploads.push(item); pendingMeta.push({ key: key, payloadHash: payloadHash, lastUploadedAt: now }); } return { uploads: uploads, pendingMeta: pendingMeta }; } async function shouldUploadCoverageToCloud(payload, options) { var state = await loadCloudUploadMetaState(); var bucket = getUploadMetaBucket(state, 'coverage'); var now = Date.now(); var ttlMs = options && Number(options.ttlMs) >= 0 ? Number(options.ttlMs) : CLOUD_COVERAGE_UPLOAD_TTL_MS; var key = buildCoverageUploadMetaKey(payload.forumKey, payload.startPage, payload.endPage, payload.strategy); var payloadHash = await buildCoverageUploadHash(payload); if (shouldSkipUploadMetaEntry(bucket[key], payloadHash, ttlMs, now)) { return { shouldUpload: false, key: key, payloadHash: payloadHash, lastUploadedAt: now }; } return { shouldUpload: true, key: key, payloadHash: payloadHash, lastUploadedAt: now }; } async function shouldUploadPageToCloud(payload, options) { var state = await loadCloudUploadMetaState(); var bucket = getUploadMetaBucket(state, 'page'); var now = Date.now(); var ttlMs = options && Number(options.ttlMs) >= 0 ? Number(options.ttlMs) : CLOUD_PAGE_UPLOAD_TTL_MS; var key = buildPageUploadMetaKey(payload.forumKey, payload.page); var payloadHash = await buildPageUploadHash(payload); if (shouldSkipUploadMetaEntry(bucket[key], payloadHash, ttlMs, now)) { return { shouldUpload: false, key: key, payloadHash: payloadHash, lastUploadedAt: now }; } return { shouldUpload: true, key: key, payloadHash: payloadHash, lastUploadedAt: now }; } async function markUploadMetaEntries(type, entries) { var state = await loadCloudUploadMetaState(); var bucket = getUploadMetaBucket(state, type); (Array.isArray(entries) ? entries : []).forEach(function(entry) { if (!entry || !entry.key) { return; } bucket[entry.key] = { payloadHash: entry.payloadHash, lastUploadedAt: Number(entry.lastUploadedAt) || Date.now() }; }); await saveCloudUploadMetaState(state); } async function backfillLocalCacheToCloud(options) { var settings = options && typeof options === 'object' ? options : {}; var force = !!settings.force; var state = null; var authState = null; var db = null; var threadRecords = []; var coverageRecords = []; var pageRecords = []; var groupedThreads = Object.create(null); var forumKeys = []; var forumKeySeen = Object.create(null); var coverageIndex = 0; var pageIndex = 0; var chunkIndex = 0; var chunks = null; var currentChunk = null; var counts = { threadCount: 0, coverageCount: 0, pageCount: 0 }; if (cloudBackfillPromise) { return cloudBackfillPromise; } cloudBackfillPromise = (async function() { authState = await loadCloudAuthState(); if (!authState.authenticated && !authState.token) { return { ok: true, skipped: true, reason: 'not_authenticated' }; } state = await getCloudBackfillState(); if (!force && state && Number(state.lastRunAt || 0) > Date.now() - 10 * 60 * 1000) { return { ok: true, skipped: true, reason: 'recently_ran', state: state }; } db = await openCacheDb(); try { threadRecords = await readAllStoreRecords(db, 'threads'); coverageRecords = await readAllStoreRecords(db, 'coverages'); pageRecords = await readAllStoreRecords(db, 'pageCoverages'); threadRecords.forEach(function(record) { if (!record || !record.forumKey || !record.threadKey) { return; } var recordMagnets = normalizeMagnets(record.magnets); if (!groupedThreads[record.forumKey]) { groupedThreads[record.forumKey] = []; } groupedThreads[record.forumKey].push({ threadKey: record.threadKey, url: record.url, title: record.title, magnets: recordMagnets, lastSeenAt: Number(record.lastSeenAt) || Date.now() }); if (!forumKeySeen[record.forumKey]) { forumKeySeen[record.forumKey] = true; forumKeys.push(record.forumKey); } }); forumKeys.forEach(function(fk) { if (groupedThreads[fk]) { groupedThreads[fk].sort(function(a, b) { return (b.magnets.length > 0 ? 1 : 0) - (a.magnets.length > 0 ? 1 : 0); }); } }); for (chunkIndex = 0; chunkIndex < forumKeys.length; chunkIndex++) { chunks = chunkArray(groupedThreads[forumKeys[chunkIndex]] || [], 200); for (pageIndex = 0; pageIndex < chunks.length; pageIndex++) { currentChunk = chunks[pageIndex]; var threadUploadPlan = null; if (currentChunk.length === 0) { continue; } threadUploadPlan = await filterThreadUploadsForCloud(forumKeys[chunkIndex], currentChunk, { ttlMs: CLOUD_THREAD_UPLOAD_TTL_MS }); if (threadUploadPlan.uploads.length === 0) { continue; } var backfillThreadResult = await cloudUpsertThreadMagnets({ forumKey: forumKeys[chunkIndex], syncedAt: Date.now(), threads: threadUploadPlan.uploads }); if (backfillThreadResult && backfillThreadResult.ok) { await markUploadMetaEntries('thread', threadUploadPlan.pendingMeta); counts.threadCount += threadUploadPlan.uploads.length; } } } for (coverageIndex = 0; coverageIndex < coverageRecords.length; coverageIndex++) { var coverageRecord = coverageRecords[coverageIndex]; var coverageThreads = await getThreadsByKeys(db, coverageRecord.threadKeys); var coverageUploadCheck = null; if (!coverageRecord || !coverageRecord.forumKey || coverageThreads.length === 0) { continue; } coverageUploadCheck = await shouldUploadCoverageToCloud({ forumKey: coverageRecord.forumKey, startPage: Number(coverageRecord.startPage) || 1, endPage: Number(coverageRecord.endPage) || Number(coverageRecord.startPage) || 1, strategy: coverageRecord.strategy || 'full_live', crawledAt: Number(coverageRecord.crawledAt) || Date.now(), threads: coverageThreads }, { ttlMs: CLOUD_COVERAGE_UPLOAD_TTL_MS }); if (!coverageUploadCheck.shouldUpload) { continue; } var backfillCoverageResult = await cloudUpsertCoverage({ forumKey: coverageRecord.forumKey, startPage: Number(coverageRecord.startPage) || 1, endPage: Number(coverageRecord.endPage) || Number(coverageRecord.startPage) || 1, strategy: coverageRecord.strategy || 'full_live', crawledAt: Number(coverageRecord.crawledAt) || Date.now(), threads: coverageThreads }); if (backfillCoverageResult && backfillCoverageResult.ok) { await markUploadMetaEntries('coverage', [coverageUploadCheck]); counts.coverageCount += 1; } } for (pageIndex = 0; pageIndex < pageRecords.length; pageIndex++) { var pageRecord = pageRecords[pageIndex]; var pageThreads = await getThreadsByKeys(db, pageRecord.threadKeys); var pageUploadCheck = null; if (!pageRecord || !pageRecord.forumKey || pageThreads.length === 0) { continue; } pageUploadCheck = await shouldUploadPageToCloud({ forumKey: pageRecord.forumKey, page: Number(pageRecord.page) || 1, crawledAt: Number(pageRecord.crawledAt) || Date.now(), threads: pageThreads }, { ttlMs: CLOUD_PAGE_UPLOAD_TTL_MS }); if (!pageUploadCheck.shouldUpload) { continue; } var backfillPageResult = await cloudUpsertPageCoverage({ forumKey: pageRecord.forumKey, page: Number(pageRecord.page) || 1, crawledAt: Number(pageRecord.crawledAt) || Date.now(), threads: pageThreads }); if (backfillPageResult && backfillPageResult.ok) { await markUploadMetaEntries('page', [pageUploadCheck]); counts.pageCount += 1; } } await saveCloudBackfillState({ lastRunAt: Date.now(), counts: counts }); await setCloudHealth(true); return { ok: true, counts: counts }; } finally { if (db) { db.close(); } } })().catch(async function(error) { console.warn('[MagnetPlugin][Cloud] backfill failed:', error && error.message ? error.message : error); throw error; }).finally(function() { cloudBackfillPromise = null; }); return cloudBackfillPromise; } async function getCacheOverview(payload) { var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : ''; var limit = Math.max(1, Math.min(Number(payload.limit) || 12, 50)); var db = await openCacheDb(); try { var threadTx = db.transaction('threads', 'readonly'); var coverageTx = db.transaction(['coverages', 'pageCoverages'], 'readonly'); var threadStore = threadTx.objectStore('threads'); var coverageStore = coverageTx.objectStore('coverages'); var pageCoverageStore = coverageTx.objectStore('pageCoverages'); var threadForumIndex = threadStore.index('forumKey'); var coverageForumIndex = coverageStore.index('forumKey'); var pageCoverageForumIndex = pageCoverageStore.index('forumKey'); var totals = await Promise.all([ requestToPromise(threadStore.count()), requestToPromise(coverageStore.count()), requestToPromise(pageCoverageStore.count()), forumKey ? requestToPromise(threadForumIndex.count(IDBKeyRange.only(forumKey))) : Promise.resolve(0), forumKey ? requestToPromise(coverageForumIndex.count(IDBKeyRange.only(forumKey))) : Promise.resolve(0), forumKey ? requestToPromise(pageCoverageForumIndex.count(IDBKeyRange.only(forumKey))) : Promise.resolve(0) ]); await Promise.all([transactionDone(threadTx), transactionDone(coverageTx)]); var recentThreads = await readRecentThreadRecords(db, forumKey, limit); var recentCoverages = await readRecentCoverageRecords(db, forumKey, Math.min(limit, 8)); var usedBytes = recentThreads.reduce(function(sum, item) { return sum + estimateTextSize(item); }, 0); var storageEstimate = await getStorageEstimate(); var magnetCachedThreads = recentThreads.filter(function(item) { return Number(item.magnetCount || 0) > 0; }).length; var magnetCount = recentThreads.reduce(function(sum, item) { return sum + Number(item.magnetCount || 0); }, 0); return { ok: true, summary: { forumKey: forumKey, totalThreadCount: Number(totals[0] || 0), totalCoverageCount: Number(totals[1] || 0), totalPageCoverageCount: Number(totals[2] || 0), forumThreadCount: Number(totals[3] || 0), forumCoverageCount: Number(totals[4] || 0), forumPageCoverageCount: Number(totals[5] || 0), sampleBytes: usedBytes, storageUsage: storageEstimate && Number(storageEstimate.usage) ? Number(storageEstimate.usage) : 0, storageQuota: storageEstimate && Number(storageEstimate.quota) ? Number(storageEstimate.quota) : 0, recentMagnetCachedThreads: magnetCachedThreads, recentMagnetCount: magnetCount }, recentThreads: recentThreads.map(function(item) { return { title: item.title, url: item.url, threadKey: item.threadKey, magnetCount: Number(item.magnetCount || 0), lastSeenAt: Number(item.lastSeenAt || 0), lastMagnetSyncAt: Number(item.lastMagnetSyncAt || 0) }; }), recentCoverages: recentCoverages.map(function(item) { return { forumKey: item.forumKey, startPage: Number(item.startPage || 0), endPage: Number(item.endPage || 0), threadCount: Number(item.threadCount || 0), crawledAt: Number(item.crawledAt || 0), strategy: item.strategy || 'full_live' }; }) }; } finally { db.close(); } } async function clearCacheData() { var db = await openCacheDb(); try { var tx = db.transaction(['threads', 'coverages', 'pageCoverages'], 'readwrite'); tx.objectStore('threads').clear(); tx.objectStore('coverages').clear(); tx.objectStore('pageCoverages').clear(); await transactionDone(tx); return { ok: true }; } finally { db.close(); } } async function getCachedCoveragePlan(payload) { var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : ''; var startPage = Math.max(1, Number(payload.startPage) || 1); var endPage = Math.max(startPage, Number(payload.endPage) || startPage); var frontRefreshPages = Math.max(0, Number(payload.frontRefreshPages) || 0); var cloudCoverage = null; var cloudAssembledCoverage = null; var cachedBlocks = []; var db = await openCacheDb(); try { var exactKey = getCoverageKey(forumKey, startPage, endPage); var serverPlan = null; var exactTx = db.transaction('coverages', 'readonly'); var exactStore = exactTx.objectStore('coverages'); var exactRecord = await requestToPromise(exactStore.get(exactKey)); await transactionDone(exactTx); var exactCoverage = exactRecord ? await getCoverageWithThreads(db, exactRecord) : null; if (!exactCoverage) { exactCoverage = await getAssembledPageCoverage(db, forumKey, startPage, endPage); } if (!exactCoverage) { try { cloudCoverage = await cloudLookupCoverage({ forumKey: forumKey, startPage: startPage, endPage: endPage }); if (cloudCoverage && Array.isArray(cloudCoverage.threads) && cloudCoverage.threads.length > 0) { await saveCoverageSnapshot({ forumKey: cloudCoverage.forumKey, startPage: cloudCoverage.startPage, endPage: cloudCoverage.endPage, crawledAt: cloudCoverage.crawledAt, strategy: cloudCoverage.strategy, frontRefreshPages: cloudCoverage.frontRefreshPages, threads: cloudCoverage.threads, skipCloudSync: true }); exactCoverage = cloudCoverage; } } catch (error) { console.warn('[MagnetPlugin][Cloud] lookup exact coverage failed:', error && error.message ? error.message : error); } } if (!exactCoverage) { try { cloudAssembledCoverage = await getCloudAssembledPageCoverage(forumKey, startPage, endPage); if (cloudAssembledCoverage && Array.isArray(cloudAssembledCoverage.threads) && cloudAssembledCoverage.threads.length > 0) { await saveCoverageSnapshot({ forumKey: cloudAssembledCoverage.forumKey, startPage: cloudAssembledCoverage.startPage, endPage: cloudAssembledCoverage.endPage, crawledAt: cloudAssembledCoverage.crawledAt, strategy: cloudAssembledCoverage.strategy, frontRefreshPages: cloudAssembledCoverage.frontRefreshPages, threads: cloudAssembledCoverage.threads, skipCloudSync: true }); exactCoverage = cloudAssembledCoverage; } } catch (error) { console.warn('[MagnetPlugin][Cloud] assemble page coverage failed:', error && error.message ? error.message : error); } } if (!exactCoverage) { try { serverPlan = await cloudLookupCoveragePlan({ forumKey: forumKey, startPage: startPage, endPage: endPage, frontRefreshPages: frontRefreshPages }); if (serverPlan && serverPlan.exactCoverage && Array.isArray(serverPlan.exactCoverage.threads) && serverPlan.exactCoverage.threads.length > 0) { await saveCoverageSnapshot({ forumKey: serverPlan.exactCoverage.forumKey, startPage: serverPlan.exactCoverage.startPage, endPage: serverPlan.exactCoverage.endPage, crawledAt: serverPlan.exactCoverage.crawledAt, strategy: serverPlan.exactCoverage.strategy, frontRefreshPages: serverPlan.exactCoverage.frontRefreshPages, threads: serverPlan.exactCoverage.threads, skipCloudSync: true }); exactCoverage = serverPlan.exactCoverage; } } catch (error) { console.warn('[MagnetPlugin][Cloud] coverage plan failed:', error && error.message ? error.message : error); } } if (exactCoverage) { return { ok: true, exactCoverage: exactCoverage, shiftedCoverage: null, cachedBlocks: [] }; } try { var localBlocks = await getCachedPageCoverageBlocks(db, forumKey, startPage, endPage); var intersectingCoverages = await getIntersectingCoverageBlocks(db, forumKey, startPage, endPage); for (var blockIndex = 0; blockIndex < localBlocks.length; blockIndex++) { var hydratedBlock = await getCoverageWithThreads(db, localBlocks[blockIndex]); if (hydratedBlock && Array.isArray(hydratedBlock.threads) && hydratedBlock.threads.length > 0) { cachedBlocks.push(hydratedBlock); } } intersectingCoverages.forEach(function(block) { cachedBlocks.push(block); }); if (serverPlan && Array.isArray(serverPlan.cachedBlocks)) { serverPlan.cachedBlocks.forEach(function(block) { if (block && Array.isArray(block.threads) && block.threads.length > 0) { cachedBlocks.push(block); } }); } cachedBlocks.sort(function(a, b) { var startDiff = Number(a.startPage || 0) - Number(b.startPage || 0); if (startDiff !== 0) { return startDiff; } return Number(b.endPage || 0) - Number(a.endPage || 0); }); } catch (error) { console.warn('[MagnetPlugin][Cache] build local cached blocks failed:', error && error.message ? error.message : error); } var shiftedCoverage = serverPlan && serverPlan.shiftedCoverage ? serverPlan.shiftedCoverage : null; if (!shiftedCoverage && startPage === 1 && frontRefreshPages > 0) { var coverages = await listForumCoverages(db, forumKey); coverages = coverages.filter(function(record) { return Number(record.startPage) === 1 && Number(record.endPage) >= 1; }).sort(function(a, b) { var crawlDiff = Number(b.crawledAt || 0) - Number(a.crawledAt || 0); if (crawlDiff !== 0) { return crawlDiff; } return Number(b.endPage || 0) - Number(a.endPage || 0); }); if (coverages.length > 0) { var bestCoverage = coverages[0]; var hydratedCoverage = await getCoverageWithThreads(db, bestCoverage); if (hydratedCoverage && Array.isArray(hydratedCoverage.threads) && hydratedCoverage.threads.length > 0) { shiftedCoverage = { forumKey: hydratedCoverage.forumKey, sourceStartPage: hydratedCoverage.startPage, sourceEndPage: hydratedCoverage.endPage, crawledAt: hydratedCoverage.crawledAt, strategy: hydratedCoverage.strategy, threads: hydratedCoverage.threads, reusedStartPage: Math.min(endPage, startPage + frontRefreshPages), reusedEndPage: Math.min(endPage, bestCoverage.endPage + frontRefreshPages) }; if (shiftedCoverage.reusedStartPage > shiftedCoverage.reusedEndPage) { shiftedCoverage = null; } } } } return { ok: true, exactCoverage: null, shiftedCoverage: shiftedCoverage, cachedBlocks: cachedBlocks }; } finally { db.close(); } } function isProgressStateKey(key) { return typeof key === 'string' && key.indexOf(STORAGE_KEY_PREFIX) === 0; } function flushSessionReadyCallbacks() { var callbacks = sessionReadyCallbacks.slice(); sessionReadyCallbacks = []; callbacks.forEach(function(callback) { try { callback(); } catch (e) { // ignore callback errors } }); } function finishSessionPrepare() { isSessionReady = true; isPreparingSession = false; flushSessionReadyCallbacks(); } function clearAllProgressStates(done) { chrome.storage.local.get(null, function(result) { if (chrome.runtime.lastError) { done(); return; } var keys = Object.keys(result || {}).filter(isProgressStateKey); if (keys.length === 0) { done(); return; } chrome.storage.local.remove(keys, function() { done(); }); }); } function ensureSessionReady(callback) { if (isSessionReady) { callback(); return; } sessionReadyCallbacks.push(callback); if (isPreparingSession) { return; } isPreparingSession = true; if (!chrome.storage || !chrome.storage.session) { finishSessionPrepare(); return; } chrome.storage.session.get(SESSION_MARKER_KEY, function(result) { if (chrome.runtime.lastError) { finishSessionPrepare(); return; } if (result && result[SESSION_MARKER_KEY]) { finishSessionPrepare(); return; } clearAllProgressStates(function() { var marker = {}; marker[SESSION_MARKER_KEY] = Date.now(); chrome.storage.session.set(marker, function() { finishSessionPrepare(); }); }); }); } function clampTimeout(timeoutMs) { var parsed = Number(timeoutMs); if (!Number.isFinite(parsed)) return 15000; return Math.max(3000, Math.min(parsed, 30000)); } function getStorageKeyBySender(sender) { if (!sender || !sender.tab || typeof sender.tab.id !== 'number') { return null; } return STORAGE_KEY_PREFIX + sender.tab.id; } function normalizeProgressState(state) { if (!state || typeof state !== 'object') { return null; } var speed = typeof state.speedMode === 'string' ? state.speedMode : 'fast'; if (speed !== 'slow' && speed !== 'medium' && speed !== 'fast' && speed !== 'ultrafast') { speed = 'fast'; } var startPage = Number(state.startPage); var endPage = Number(state.endPage); if (!Number.isFinite(startPage) || startPage < 1) startPage = 1; if (!Number.isFinite(endPage) || endPage < 1) endPage = startPage; var resumeFromPage = Number(state.resumeFromPage); if (!Number.isFinite(resumeFromPage) || resumeFromPage < 1) { resumeFromPage = startPage; } var isRunning = !!state.isRunning; var stoppedByUser = !!state.stoppedByUser; var linkSet = new Set(); var links = []; if (Array.isArray(state.links)) { state.links.forEach(function(link) { if (typeof link !== 'string' || !link) return; if (linkSet.has(link)) return; linkSet.add(link); links.push(link); }); } var items = []; if (Array.isArray(state.items)) { state.items.forEach(function(item) { if (!item || typeof item !== 'object') return; var link = typeof item.link === 'string' ? item.link : ''; if (!link) return; var title = typeof item.title === 'string' ? item.title : '恢复记录'; if (!linkSet.has(link)) { linkSet.add(link); links.push(link); } items.push({ title: title.slice(0, 200), link: link }); }); } if (links.length > MAX_PROGRESS_ITEMS) { links = links.slice(links.length - MAX_PROGRESS_ITEMS); } if (items.length > MAX_PROGRESS_ITEMS) { items = items.slice(items.length - MAX_PROGRESS_ITEMS); } return { links: links, items: items, keyword: typeof state.keyword === 'string' ? state.keyword.slice(0, 200) : '', speedMode: speed, startPage: startPage, endPage: endPage, resumeFromPage: resumeFromPage, isRunning: isRunning, stoppedByUser: stoppedByUser, statusText: typeof state.statusText === 'string' ? state.statusText.slice(0, 300) : '', statusType: typeof state.statusType === 'string' ? state.statusType.slice(0, 30) : '', pageUrl: typeof state.pageUrl === 'string' ? state.pageUrl.slice(0, 1000) : '', updatedAt: Date.now() }; } function fetchPageHtml(url, timeoutMs) { var controller = new AbortController(); var realTimeout = clampTimeout(timeoutMs); var timer = setTimeout(function() { controller.abort(); }, realTimeout); return fetch(url, { signal: controller.signal }) .then(function(response) { if (!response.ok) { throw new Error('HTTP ' + response.status + ' ' + response.statusText); } return response.text(); }) .catch(function(err) { if (err && err.name === 'AbortError') { throw new Error('请求超时(' + realTimeout + 'ms)'); } throw err; }) .finally(function() { clearTimeout(timer); }); } function extractMagnetsFromHtml(html) { const magnetPattern = /magnet:\?xt=urn:btih:[a-fA-F0-9]{32,40}/gi; const magnets = html.match(magnetPattern) || []; return [...new Set(magnets)]; } chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { if (request.action === 'openCloudSyncPage') { chrome.tabs.create({ url: chrome.runtime.getURL('popup.html?view=cloud') }, function(tab) { if (chrome.runtime.lastError) { sendResponse({ ok: false, error: chrome.runtime.lastError.message }); return; } sendResponse({ ok: true, tabId: tab && typeof tab.id === 'number' ? tab.id : null }); }); return true; } if (request.action === 'cloudRegister') { cloudRegisterAccount(request) .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cloudLogin') { cloudLoginAccount(request) .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cloudLogout') { cloudLogoutAccount() .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cloudGetSyncStatus') { cloudGetSyncStatus() .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, authenticated: false, healthy: false, color: 'red', text: '云同步异常', error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cloudPullVaultItems') { cloudPullVaultItems(request.itemTypes) .then(function(items) { sendResponse({ ok: true, items: items, status: buildCloudStatus(cloudAuthStateCache) }); }) .catch(function(err) { sendResponse({ ok: false, items: [], error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cloudPushVaultItems') { cloudPushVaultItems(request) .then(function(status) { sendResponse({ ok: true, status: status }); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cloudBackfillLocalCache') { backfillLocalCacheToCloud({ force: !!request.force }) .then(function(result) { sendResponse({ ok: true, result: result, status: buildCloudStatus(cloudAuthStateCache) }); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cloudGetCacheStats') { cloudGetCacheStats() .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'localGetCacheStats') { getLocalCacheStats() .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cacheSaveCoverage') { saveCoverageSnapshot(request) .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cacheGetPageThreadKeys') { (async function() { var forumKey = typeof request.forumKey === 'string' ? request.forumKey : ''; var page = Math.max(1, Number(request.page) || 1); if (!forumKey) { return { ok: true, threadKeys: [], crawledAt: 0 }; } var db = await openCacheDb(); try { var tx = db.transaction('pageCoverages', 'readonly'); var store = tx.objectStore('pageCoverages'); var record = await requestToPromise(store.get(getPageCoverageKey(forumKey, page))); await transactionDone(tx); if (!record || !Array.isArray(record.threadKeys)) { return { ok: true, threadKeys: [], crawledAt: 0 }; } return { ok: true, threadKeys: record.threadKeys, crawledAt: Number(record.crawledAt || 0) }; } finally { db.close(); } })() .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, threadKeys: [], error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cacheGetCoveragePlan') { getCachedCoveragePlan(request) .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cacheGetThreadMagnets') { getCachedThreadMagnets(request) .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cacheSaveThreadMagnets') { saveThreadMagnets(request) .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cacheSavePageCoverage') { savePageCoverageSnapshot(request) .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cacheGetOverview') { getCacheOverview(request) .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'cacheClearAll') { clearCacheData() .then(function(result) { sendResponse(result); }) .catch(function(err) { sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); }); return true; } if (request.action === 'saveProgressState') { ensureSessionReady(function() { const saveKey = getStorageKeyBySender(sender); if (!saveKey) { sendResponse({ ok: false, error: '无法识别标签页' }); return; } const normalizedState = normalizeProgressState(request.state); if (!normalizedState) { sendResponse({ ok: false, error: '状态数据无效' }); return; } chrome.storage.local.set({ [saveKey]: normalizedState }, function() { if (chrome.runtime.lastError) { sendResponse({ ok: false, error: chrome.runtime.lastError.message }); return; } cloudPushVaultItems({ items: [{ itemType: 'progress_state', itemKey: 'latest', data: normalizedState }] }).catch(function() { return null; }); sendResponse({ ok: true }); }); }); return true; } if (request.action === 'loadProgressState') { ensureSessionReady(function() { const loadKey = getStorageKeyBySender(sender); if (!loadKey) { sendResponse({ ok: false, error: '无法识别标签页', state: null }); return; } chrome.storage.local.get(loadKey, function(result) { if (chrome.runtime.lastError) { sendResponse({ ok: false, error: chrome.runtime.lastError.message, state: null }); return; } sendResponse({ ok: true, state: result && result[loadKey] ? result[loadKey] : null }); }); }); return true; } if (request.action === 'clearProgressState') { ensureSessionReady(function() { const clearKey = getStorageKeyBySender(sender); if (!clearKey) { sendResponse({ ok: false, error: '无法识别标签页' }); return; } chrome.storage.local.remove(clearKey, function() { if (chrome.runtime.lastError) { sendResponse({ ok: false, error: chrome.runtime.lastError.message }); return; } sendResponse({ ok: true }); }); }); return true; } if (request.action === 'fetchHtml') { fetchPageHtml(request.url, request.timeoutMs) .then(html => { sendResponse({ html: html }); }) .catch(err => { sendResponse({ html: '', error: err.message }); }); return true; } if (request.action === 'openAndFetch') { fetchPageHtml(request.url, request.timeoutMs) .then(html => { sendResponse({ magnets: extractMagnetsFromHtml(html) }); }) .catch(err => { sendResponse({ magnets: [], error: err.message }); }); return true; } return false; }); if (chrome.tabs && chrome.tabs.onRemoved) { chrome.tabs.onRemoved.addListener(function(tabId) { var key = STORAGE_KEY_PREFIX + tabId; chrome.storage.local.remove(key, function() { // ignore cleanup errors }); }); }