From 1bf9c072c87843d783e3827d0df4307a1b9e64ff Mon Sep 17 00:00:00 2001 From: Developer Date: Wed, 18 Mar 2026 00:27:27 +0800 Subject: [PATCH] =?UTF-8?q?feat(ext):=20=E6=8E=A5=E5=85=A5=E4=BA=91?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E3=80=81=E5=90=8C=E6=AD=A5=E4=B8=AD=E5=BF=83?= =?UTF-8?q?=E4=B8=8E=E7=8A=B6=E6=80=81=E7=BB=9F=E8=AE=A1=E9=93=BE=E8=B7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- background.js | 2065 ++++++++++++++++++++++++++++++++++++++++++++++++- manifest.json | 6 +- 2 files changed, 2049 insertions(+), 22 deletions(-) diff --git a/background.js b/background.js index 00c2857..44c9b66 100644 --- a/background.js +++ b/background.js @@ -3,7 +3,24 @@ var MAX_PROGRESS_ITEMS = 2000; var SESSION_MARKER_KEY = 'magnet-progress-session-marker-v1'; var CACHE_DB_NAME = 'magnet-thread-cache'; var CACHE_DB_VERSION = 3; -var COVERAGE_LIMIT_PER_FORUM = 12; +var COVERAGE_LIMIT_PER_FORUM = 999; +var CLOUD_CACHE_ENABLED = true; +var CLOUD_CACHE_BASE_URL = 'https://s.52oai.com'; +var CLOUD_CACHE_TIMEOUT = 6000; +var CLOUD_SHARED_CACHE_WRITE_TOKEN = 's52oai-shared-cache-write-v1'; +var CLOUD_AUTH_STORAGE_KEY = 'magnet-cloud-auth-v1'; +var CLOUD_DEVICE_ID_KEY = 'magnet-cloud-device-id-v1'; +var CLOUD_CACHE_BACKFILL_KEY = 'magnet-cloud-cache-backfill-v1'; +var CLOUD_UPLOAD_META_KEY = 'magnet-cloud-upload-meta-v1'; +var CLOUD_VAULT_KEY_VERSION = 1; +var CLOUD_THREAD_UPLOAD_TTL_MS = 10 * 60 * 1000; +var CLOUD_PAGE_UPLOAD_TTL_MS = 30 * 60 * 1000; +var CLOUD_COVERAGE_UPLOAD_TTL_MS = 60 * 60 * 1000; +var CLOUD_UPLOAD_META_THREAD_LIMIT = 50000; +var CLOUD_UPLOAD_META_OTHER_LIMIT = 4000; +var cloudAuthStateCache = null; +var cloudUploadMetaCache = null; +var cloudBackfillPromise = null; var isSessionReady = false; var isPreparingSession = false; var sessionReadyCallbacks = []; @@ -33,6 +50,635 @@ function transactionDone(transaction) { }); } +function normalizeCloudBaseUrl(url) { + return String(url || '').replace(/\/+$/, ''); +} + +function callCloudCache(path, payload, timeoutMs, extraHeaders) { + if (!CLOUD_CACHE_ENABLED) { + return Promise.resolve(null); + } + + var controller = new AbortController(); + var realTimeout = Math.max(2000, Math.min(Number(timeoutMs) || CLOUD_CACHE_TIMEOUT, 15000)); + var timer = setTimeout(function() { + controller.abort(); + }, realTimeout); + + return fetch(normalizeCloudBaseUrl(CLOUD_CACHE_BASE_URL) + path, { + method: 'POST', + headers: Object.assign({ + 'Content-Type': 'application/json' + }, extraHeaders || {}), + body: JSON.stringify(payload || {}), + signal: controller.signal + }).then(function(response) { + if (!response.ok) { + throw new Error('云缓存 HTTP ' + response.status); + } + return response.json(); + }).catch(function(error) { + if (error && error.name === 'AbortError') { + throw new Error('云缓存请求超时(' + realTimeout + 'ms)'); + } + throw error; + }).finally(function() { + clearTimeout(timer); + }); +} + +function storageLocalGet(key) { + return new Promise(function(resolve, reject) { + chrome.storage.local.get(key, function(result) { + if (chrome.runtime.lastError) { + reject(new Error(chrome.runtime.lastError.message)); + return; + } + resolve(result || {}); + }); + }); +} + +function storageLocalSet(data) { + return new Promise(function(resolve, reject) { + chrome.storage.local.set(data, function() { + if (chrome.runtime.lastError) { + reject(new Error(chrome.runtime.lastError.message)); + return; + } + resolve(); + }); + }); +} + +function storageLocalRemove(key) { + return new Promise(function(resolve, reject) { + chrome.storage.local.remove(key, function() { + if (chrome.runtime.lastError) { + reject(new Error(chrome.runtime.lastError.message)); + return; + } + resolve(); + }); + }); +} + +function textEncoder() { + return new TextEncoder(); +} + +function textDecoder() { + return new TextDecoder(); +} + +function bytesToBase64(bytes) { + var binary = ''; + var chunkSize = 0x8000; + var index = 0; + for (index = 0; index < bytes.length; index += chunkSize) { + binary += String.fromCharCode.apply(null, bytes.subarray(index, index + chunkSize)); + } + return btoa(binary); +} + +function base64ToBytes(base64) { + var binary = atob(String(base64 || '')); + var bytes = new Uint8Array(binary.length); + var index = 0; + for (index = 0; index < binary.length; index++) { + bytes[index] = binary.charCodeAt(index); + } + return bytes; +} + +function arrayBufferToBase64(buffer) { + return bytesToBase64(new Uint8Array(buffer)); +} + +function randomBase64(size) { + var bytes = new Uint8Array(size); + crypto.getRandomValues(bytes); + return bytesToBase64(bytes); +} + +async function sha256Hex(value) { + var digest = await crypto.subtle.digest('SHA-256', textEncoder().encode(String(value || ''))); + return Array.from(new Uint8Array(digest)).map(function(byte) { + return byte.toString(16).padStart(2, '0'); + }).join(''); +} + +async function derivePasswordKey(password, saltBase64, keyUsages) { + var baseKey = await crypto.subtle.importKey( + 'raw', + textEncoder().encode(String(password || '')), + 'PBKDF2', + false, + ['deriveKey'] + ); + return crypto.subtle.deriveKey({ + name: 'PBKDF2', + salt: base64ToBytes(saltBase64), + iterations: 250000, + hash: 'SHA-256' + }, baseKey, { + name: 'AES-GCM', + length: 256 + }, false, keyUsages || ['encrypt', 'decrypt']); +} + +async function importVaultKey(vaultKeyBase64, keyUsages) { + return crypto.subtle.importKey( + 'raw', + base64ToBytes(vaultKeyBase64), + { name: 'AES-GCM', length: 256 }, + false, + keyUsages || ['encrypt', 'decrypt'] + ); +} + +function splitCipherAndTag(buffer) { + var bytes = new Uint8Array(buffer); + var tagLength = 16; + return { + ciphertext: bytesToBase64(bytes.slice(0, bytes.length - tagLength)), + tag: bytesToBase64(bytes.slice(bytes.length - tagLength)) + }; +} + +function joinCipherAndTag(ciphertextBase64, tagBase64) { + var ciphertext = base64ToBytes(ciphertextBase64); + var tag = base64ToBytes(tagBase64); + var merged = new Uint8Array(ciphertext.length + tag.length); + merged.set(ciphertext, 0); + merged.set(tag, ciphertext.length); + return merged; +} + +async function createVaultEnvelope(password) { + var saltBase64 = randomBase64(16); + var vaultKeyBase64 = randomBase64(32); + var ivBase64 = randomBase64(12); + var passwordKey = await derivePasswordKey(password, saltBase64, ['encrypt', 'decrypt']); + var encrypted = await crypto.subtle.encrypt( + { name: 'AES-GCM', iv: base64ToBytes(ivBase64) }, + passwordKey, + base64ToBytes(vaultKeyBase64) + ); + return { + vaultKeyBase64: vaultKeyBase64, + wrappedDek: JSON.stringify({ + ciphertext: arrayBufferToBase64(encrypted), + iv: ivBase64 + }), + kdfSalt: saltBase64, + kdfParams: { + algorithm: 'PBKDF2', + hash: 'SHA-256', + iterations: 250000 + }, + keyVersion: CLOUD_VAULT_KEY_VERSION + }; +} + +async function unlockVaultKey(password, keyring) { + var envelope = keyring && keyring.wrappedDek ? JSON.parse(keyring.wrappedDek) : null; + if (!envelope || !envelope.ciphertext || !envelope.iv || !keyring.kdfSalt) { + throw new Error('保险柜密钥无效'); + } + var passwordKey = await derivePasswordKey(password, keyring.kdfSalt, ['decrypt']); + var decrypted = await crypto.subtle.decrypt( + { name: 'AES-GCM', iv: base64ToBytes(envelope.iv) }, + passwordKey, + base64ToBytes(envelope.ciphertext) + ); + return bytesToBase64(new Uint8Array(decrypted)); +} + +async function encryptVaultPayload(vaultKeyBase64, payload) { + var json = JSON.stringify(payload); + var ivBase64 = randomBase64(12); + var vaultKey = await importVaultKey(vaultKeyBase64, ['encrypt']); + var encrypted = await crypto.subtle.encrypt( + { name: 'AES-GCM', iv: base64ToBytes(ivBase64) }, + vaultKey, + textEncoder().encode(json) + ); + var split = splitCipherAndTag(encrypted); + return { + payloadCiphertext: split.ciphertext, + payloadIv: ivBase64, + payloadTag: split.tag, + payloadHash: await sha256Hex(json), + keyVersion: CLOUD_VAULT_KEY_VERSION + }; +} + +async function decryptVaultPayload(vaultKeyBase64, item) { + var vaultKey = await importVaultKey(vaultKeyBase64, ['decrypt']); + var decrypted = await crypto.subtle.decrypt( + { name: 'AES-GCM', iv: base64ToBytes(item.payloadIv) }, + vaultKey, + joinCipherAndTag(item.payloadCiphertext, item.payloadTag) + ); + return JSON.parse(textDecoder().decode(decrypted)); +} + +function normalizeCloudAuthState(state) { + state = state && typeof state === 'object' ? state : {}; + return { + token: typeof state.token === 'string' ? state.token : '', + vaultKeyBase64: typeof state.vaultKeyBase64 === 'string' ? state.vaultKeyBase64 : '', + userId: Number(state.userId) || 0, + email: typeof state.email === 'string' ? state.email : '', + syncEnabled: !!state.syncEnabled, + healthy: !!state.healthy, + lastError: typeof state.lastError === 'string' ? state.lastError : '', + lastSyncAt: Number(state.lastSyncAt) || 0 + }; +} + +async function loadCloudAuthState() { + if (cloudAuthStateCache) { + return cloudAuthStateCache; + } + var result = await storageLocalGet(CLOUD_AUTH_STORAGE_KEY); + cloudAuthStateCache = normalizeCloudAuthState(result[CLOUD_AUTH_STORAGE_KEY]); + return cloudAuthStateCache; +} + +async function saveCloudAuthState(state) { + cloudAuthStateCache = normalizeCloudAuthState(state); + await storageLocalSet((function() { + var data = {}; + data[CLOUD_AUTH_STORAGE_KEY] = cloudAuthStateCache; + return data; + })()); + return cloudAuthStateCache; +} + +async function clearCloudAuthState() { + cloudAuthStateCache = normalizeCloudAuthState(null); + await storageLocalRemove(CLOUD_AUTH_STORAGE_KEY); + return cloudAuthStateCache; +} + +async function ensureCloudDeviceFingerprint() { + var result = await storageLocalGet(CLOUD_DEVICE_ID_KEY); + var existing = typeof result[CLOUD_DEVICE_ID_KEY] === 'string' ? result[CLOUD_DEVICE_ID_KEY] : ''; + if (existing) { + return existing; + } + existing = 'device-' + randomBase64(18).replace(/[^a-zA-Z0-9_-]/g, ''); + await storageLocalSet((function() { + var data = {}; + data[CLOUD_DEVICE_ID_KEY] = existing; + return data; + })()); + return existing; +} + +function buildCloudStatus(state) { + var normalized = normalizeCloudAuthState(state); + var healthy = !!(normalized.token && normalized.syncEnabled && normalized.healthy); + return { + ok: true, + syncEnabled: !!normalized.syncEnabled, + authenticated: !!normalized.token, + healthy: healthy, + color: healthy ? 'green' : 'red', + email: normalized.email, + lastError: normalized.lastError, + text: healthy ? '云同步正常' : (normalized.token ? '云同步需确认' : '云同步未登录'), + accountText: healthy ? '账号状态正常' : (normalized.token ? '账号状态需确认' : '未登录') + }; +} + +function callCloudApi(path, options) { + var method = options && options.method ? options.method : 'GET'; + var payload = options && Object.prototype.hasOwnProperty.call(options, 'payload') ? options.payload : null; + var token = options && options.token ? options.token : ''; + var timeoutMs = options && options.timeoutMs ? options.timeoutMs : CLOUD_CACHE_TIMEOUT; + var controller = new AbortController(); + var realTimeout = Math.max(2000, Math.min(Number(timeoutMs) || CLOUD_CACHE_TIMEOUT, 15000)); + var timer = setTimeout(function() { + controller.abort(); + }, realTimeout); + var headers = { 'Content-Type': 'application/json' }; + if (token) { + headers.Authorization = 'Bearer ' + token; + } + return fetch(normalizeCloudBaseUrl(CLOUD_CACHE_BASE_URL) + path, { + method: method, + headers: headers, + body: payload !== null ? JSON.stringify(payload) : undefined, + signal: controller.signal + }).then(function(response) { + if (!response.ok) { + return response.text().then(function(text) { + var payload = null; + try { + payload = text ? JSON.parse(text) : null; + } catch (e) { + payload = null; + } + throw new Error(payload && payload.error ? payload.error : ('云接口 HTTP ' + response.status)); + }); + } + return response.json(); + }).finally(function() { + clearTimeout(timer); + }); +} + +async function setCloudHealth(ok, errorMessage) { + var state = await loadCloudAuthState(); + state.healthy = !!ok; + state.lastError = ok ? '' : String(errorMessage || '同步失败'); + state.lastSyncAt = Date.now(); + await saveCloudAuthState(state); + return buildCloudStatus(state); +} + +async function cloudRegisterAccount(payload) { + var email = String(payload.email || '').trim().toLowerCase(); + var password = String(payload.password || ''); + var deviceFingerprint = await ensureCloudDeviceFingerprint(); + var envelope = await createVaultEnvelope(password); + var response = await callCloudApi('/api/auth/register', { + method: 'POST', + payload: { + email: email, + password: password, + wrappedDek: envelope.wrappedDek, + kdfSalt: envelope.kdfSalt, + kdfParams: envelope.kdfParams, + keyVersion: envelope.keyVersion, + deviceName: 'Magnet Chrome Extension', + deviceFingerprint: deviceFingerprint + } + }); + if (!response || !response.ok || !response.token) { + throw new Error(response && response.error ? response.error : '注册失败'); + } + await saveCloudAuthState({ + token: response.token, + vaultKeyBase64: envelope.vaultKeyBase64, + userId: response.user && response.user.id, + email: response.user && response.user.email ? response.user.email : email, + syncEnabled: true, + healthy: true, + lastError: '', + lastSyncAt: Date.now() + }); + backfillLocalCacheToCloud({ force: true }).catch(function(error) { + console.warn('[MagnetPlugin][Cloud] register backfill failed:', error && error.message ? error.message : error); + }); + return { + ok: true, + status: buildCloudStatus(await loadCloudAuthState()), + vaultItems: [] + }; +} + +async function cloudPullVaultItems(itemTypes) { + var state = await loadCloudAuthState(); + var response = null; + var items = []; + if (!state.token || !state.vaultKeyBase64) { + return []; + } + response = await callCloudApi('/api/vault/pull', { + method: 'POST', + token: state.token, + payload: { + itemTypes: Array.isArray(itemTypes) ? itemTypes : [] + } + }); + if (!response || !response.ok || !Array.isArray(response.items)) { + throw new Error(response && response.error ? response.error : '读取保险柜失败'); + } + items = []; + for (var index = 0; index < response.items.length; index++) { + var item = response.items[index]; + items.push({ + itemType: item.itemType, + itemKey: item.itemKey, + data: await decryptVaultPayload(state.vaultKeyBase64, item), + updatedAt: item.updatedAt, + keyVersion: item.keyVersion + }); + } + await setCloudHealth(true); + return items; +} + +async function cloudLoginAccount(payload) { + var email = String(payload.email || '').trim().toLowerCase(); + var password = String(payload.password || ''); + var deviceFingerprint = await ensureCloudDeviceFingerprint(); + var response = await callCloudApi('/api/auth/login', { + method: 'POST', + payload: { + email: email, + password: password, + deviceName: 'Magnet Chrome Extension', + deviceFingerprint: deviceFingerprint + } + }); + var vaultKeyBase64 = null; + var vaultItems = []; + if (!response || !response.ok || !response.token || !response.keyring) { + throw new Error(response && response.error ? response.error : '登录失败'); + } + vaultKeyBase64 = await unlockVaultKey(password, response.keyring); + await saveCloudAuthState({ + token: response.token, + vaultKeyBase64: vaultKeyBase64, + userId: response.user && response.user.id, + email: response.user && response.user.email ? response.user.email : email, + syncEnabled: true, + healthy: true, + lastError: '', + lastSyncAt: Date.now() + }); + backfillLocalCacheToCloud({ force: true }).catch(function(error) { + console.warn('[MagnetPlugin][Cloud] login backfill failed:', error && error.message ? error.message : error); + }); + cloudPullVaultItems(['favorites', 'search_history', 'ui_settings', 'progress_state']).catch(function(error) { + console.warn('[MagnetPlugin][Cloud] login vault pull failed:', error && error.message ? error.message : error); + return []; + }); + return { + ok: true, + status: buildCloudStatus(await loadCloudAuthState()), + vaultItems: vaultItems + }; +} + +async function cloudLogoutAccount() { + var state = await loadCloudAuthState(); + if (state.token) { + try { + await callCloudApi('/api/auth/logout', { + method: 'POST', + token: state.token, + payload: {} + }); + } catch (error) { + console.warn('[MagnetPlugin][Cloud] logout failed:', error && error.message ? error.message : error); + } + } + await clearCloudAuthState(); + return { + ok: true, + status: buildCloudStatus(await loadCloudAuthState()) + }; +} + +async function cloudGetSyncStatus() { + var state = await loadCloudAuthState(); + if (!state.token) { + return buildCloudStatus(state); + } + try { + var response = await callCloudApi('/api/auth/me', { + method: 'GET', + token: state.token + }); + if (response && response.ok) { + state.email = response.user && response.user.email ? response.user.email : state.email; + state.userId = response.user && response.user.id ? response.user.id : state.userId; + state.syncEnabled = true; + state.healthy = true; + state.lastError = ''; + state.lastSyncAt = Date.now(); + await saveCloudAuthState(state); + backfillLocalCacheToCloud({ force: false }).catch(function(error) { + console.warn('[MagnetPlugin][Cloud] background backfill failed:', error && error.message ? error.message : error); + }); + return buildCloudStatus(state); + } + throw new Error(response && response.error ? response.error : '状态检查失败'); + } catch (error) { + return { + ok: true, + syncEnabled: !!state.syncEnabled, + authenticated: !!state.token, + healthy: false, + color: 'red', + email: state.email, + lastError: error && error.message ? error.message : String(error), + text: state.token ? '云同步需确认' : '云同步未登录', + accountText: state.token ? '账号状态需确认' : '未登录' + }; + } +} + +async function cloudGetCacheStats() { + return callCloudApi('/api/shared-cache/stats', { + method: 'GET', + timeoutMs: 12000 + }); +} + +async function getLocalCacheStats() { + var db = await openCacheDb(); + var privateState = null; + var uploadMeta = null; + var threadCount = 0; + var magnetThreadCount = 0; + var coverageCount = 0; + var pageCount = 0; + var storageEstimate = null; + var allThreads = []; + try { + var tx = db.transaction(['threads', 'coverages', 'pageCoverages'], 'readonly'); + threadCount = Number(await requestToPromise(tx.objectStore('threads').count()) || 0); + coverageCount = Number(await requestToPromise(tx.objectStore('coverages').count()) || 0); + pageCount = Number(await requestToPromise(tx.objectStore('pageCoverages').count()) || 0); + await transactionDone(tx); + allThreads = await readAllStoreRecords(db, 'threads'); + magnetThreadCount = allThreads.filter(function(record) { + return Array.isArray(record.magnets) && normalizeMagnets(record.magnets).length > 0; + }).length; + } finally { + db.close(); + } + + privateState = await storageLocalGet('magnet-private-state-v1'); + uploadMeta = await loadCloudUploadMetaState(); + storageEstimate = await getStorageEstimate(); + + var privateData = privateState['magnet-private-state-v1'] && typeof privateState['magnet-private-state-v1'] === 'object' + ? privateState['magnet-private-state-v1'] + : {}; + + return { + ok: true, + counts: { + threads: threadCount, + magnetThreads: magnetThreadCount, + coverages: coverageCount, + pages: pageCount, + favorites: Array.isArray(privateData.favorites) ? privateData.favorites.length : 0, + history: Array.isArray(privateData.searchHistory) ? privateData.searchHistory.length : 0, + uploadMetaThreads: Object.keys(uploadMeta.threads || {}).length, + uploadMetaCoverages: Object.keys(uploadMeta.coverages || {}).length, + uploadMetaPages: Object.keys(uploadMeta.pages || {}).length + }, + storage: { + usage: storageEstimate && Number(storageEstimate.usage) ? Number(storageEstimate.usage) : 0, + quota: storageEstimate && Number(storageEstimate.quota) ? Number(storageEstimate.quota) : 0 + } + }; +} + +async function cloudPushVaultItems(payload) { + var state = await loadCloudAuthState(); + var items = Array.isArray(payload.items) ? payload.items : []; + var preparedItems = []; + var index = 0; + var item = null; + if (!state.token || !state.vaultKeyBase64) { + return buildCloudStatus(state); + } + for (index = 0; index < items.length; index++) { + item = items[index]; + if (!item || typeof item !== 'object') { + continue; + } + var itemType = String(item.itemType || '').trim(); + var itemKey = String(item.itemKey || '').trim() || itemType; + if (!itemType) { + continue; + } + var encrypted = await encryptVaultPayload(state.vaultKeyBase64, item.data); + preparedItems.push({ + itemType: itemType, + itemKey: itemKey, + payloadCiphertext: encrypted.payloadCiphertext, + payloadIv: encrypted.payloadIv, + payloadTag: encrypted.payloadTag, + payloadHash: encrypted.payloadHash, + keyVersion: encrypted.keyVersion + }); + } + if (preparedItems.length === 0) { + return buildCloudStatus(state); + } + try { + await callCloudApi('/api/vault/push', { + method: 'POST', + token: state.token, + payload: { items: preparedItems } + }); + return await setCloudHealth(true); + } catch (error) { + return await setCloudHealth(false, error && error.message ? error.message : error); + } +} + function openCacheDb() { return new Promise(function(resolve, reject) { var request = indexedDB.open(CACHE_DB_NAME, CACHE_DB_VERSION); @@ -177,6 +823,330 @@ function getPageCoverageKey(forumKey, page) { return forumKey + '::page::' + page; } + +function normalizeThreadsForCoverage(rawThreads, forumKey, crawledAt) { + var seen = Object.create(null); + var normalizedThreads = []; + + (Array.isArray(rawThreads) ? rawThreads : []).forEach(function(thread) { + var normalized = normalizeThreadEntry(thread, forumKey, crawledAt); + if (!normalized || seen[normalized.cacheKey]) { + return; + } + + seen[normalized.cacheKey] = true; + normalizedThreads.push({ + threadKey: normalized.threadKey, + url: normalized.url, + title: normalized.title + }); + }); + + return normalizedThreads; +} + +function mergeCoverageThreadLists(forumKey, crawledAt, primaryThreads, secondaryThreads) { + var seen = Object.create(null); + var merged = []; + + [primaryThreads, secondaryThreads].forEach(function(source) { + normalizeThreadsForCoverage(source, forumKey, crawledAt).forEach(function(thread) { + var key = String(thread.threadKey || ''); + if (!key || seen[key]) { + return; + } + seen[key] = true; + merged.push({ + threadKey: thread.threadKey, + url: thread.url, + title: thread.title + }); + }); + }); + + return merged; +} + +function normalizeThreadsForCloudMagnets(rawThreads, forumKey, crawledAt) { + var seen = Object.create(null); + var normalizedThreads = []; + + (Array.isArray(rawThreads) ? rawThreads : []).forEach(function(thread) { + var normalized = normalizeThreadEntry(thread, forumKey, crawledAt); + if (!normalized || seen[normalized.cacheKey]) { + return; + } + + var magnets = normalizeMagnets(thread.magnets); + if (magnets.length === 0) { + return; + } + + seen[normalized.cacheKey] = true; + normalizedThreads.push({ + threadKey: normalized.threadKey, + url: normalized.url, + title: normalized.title, + magnets: magnets, + lastSeenAt: crawledAt + }); + }); + + return normalizedThreads; +} + +function normalizeThreadsForCloudIndex(rawThreads, forumKey, crawledAt) { + var seen = Object.create(null); + var normalizedThreads = []; + + (Array.isArray(rawThreads) ? rawThreads : []).forEach(function(thread) { + var normalized = normalizeThreadEntry(thread, forumKey, crawledAt); + if (!normalized || seen[normalized.cacheKey]) { + return; + } + + seen[normalized.cacheKey] = true; + normalizedThreads.push({ + threadKey: normalized.threadKey, + url: normalized.url, + title: normalized.title, + magnets: normalizeMagnets(thread.magnets), + lastSeenAt: Number(thread.lastSeenAt) || crawledAt + }); + }); + + return normalizedThreads; +} + +function normalizeCloudCoveragePayload(coverage) { + if (!coverage || typeof coverage !== 'object') { + return null; + } + + var forumKey = typeof coverage.forumKey === 'string' ? coverage.forumKey : ''; + var startPage = Math.max(1, Number(coverage.startPage) || 1); + var endPage = Math.max(startPage, Number(coverage.endPage) || startPage); + if (!forumKey) { + return null; + } + + return { + forumKey: forumKey, + startPage: startPage, + endPage: endPage, + crawledAt: Number(coverage.crawledAt) || Date.now(), + strategy: typeof coverage.strategy === 'string' && coverage.strategy ? coverage.strategy : 'full_live', + frontRefreshPages: Math.max(0, Number(coverage.frontRefreshPages) || 0), + threads: normalizeThreadsForCoverage(coverage.threads, forumKey, Number(coverage.crawledAt) || Date.now()) + }; +} + +function normalizeCloudPageCoveragePayload(coverage) { + if (!coverage || typeof coverage !== 'object') { + return null; + } + + var forumKey = typeof coverage.forumKey === 'string' ? coverage.forumKey : ''; + var page = Math.max(1, Number(coverage.page) || 1); + if (!forumKey) { + return null; + } + + return { + forumKey: forumKey, + page: page, + crawledAt: Number(coverage.crawledAt) || Date.now(), + threads: normalizeThreadsForCoverage(coverage.threads, forumKey, Number(coverage.crawledAt) || Date.now()) + }; +} + +function normalizeCloudThreadMagnetsPayload(result) { + if (!result || typeof result !== 'object') { + return null; + } + + var forumKey = typeof result.forumKey === 'string' ? result.forumKey : ''; + var threadKey = typeof result.threadKey === 'string' ? result.threadKey : getThreadKeyFromUrl(result.url); + var url = normalizeUrl(result.url); + var magnets = normalizeMagnets(result.magnets); + if (!forumKey || !threadKey || !url || magnets.length === 0) { + return null; + } + + return { + forumKey: forumKey, + threadKey: threadKey, + url: url, + title: normalizeTitle(result.title), + magnets: magnets, + lastSeenAt: Number(result.lastSeenAt) || Date.now() + }; +} + +async function cloudLookupThreadMagnets(payload) { + var response = await callCloudCache('/api/shared-cache/threads/lookup', { + threads: normalizeThreadsForCoverage(payload.threads, payload.forumKey, Date.now()).map(function(item) { + return { + forumKey: payload.forumKey, + threadKey: item.threadKey + }; + }) + }); + + if (!response || !response.ok || !Array.isArray(response.threads)) { + return []; + } + + return response.threads.map(function(item) { + return normalizeCloudThreadMagnetsPayload({ + forumKey: payload.forumKey, + threadKey: item.threadKey, + url: item.url, + title: item.title, + magnets: item.magnets, + lastSeenAt: item.lastSeenAt + }); + }).filter(Boolean); +} + +async function cloudUpsertThreadMagnets(payload) { + var allThreads = normalizeThreadsForCloudIndex(payload.threads, payload.forumKey, Number(payload.syncedAt) || Date.now()).map(function(item) { + return { + forumKey: payload.forumKey, + threadKey: item.threadKey, + url: item.url, + title: item.title, + magnets: item.magnets, + lastSeenAt: item.lastSeenAt + }; + }); + if (allThreads.length === 0) { + return { ok: true, savedCount: 0 }; + } + return callCloudCache('/api/shared-cache/threads/upsert', { + threads: allThreads + }, CLOUD_CACHE_TIMEOUT, { + 'x-shared-cache-write-token': CLOUD_SHARED_CACHE_WRITE_TOKEN + }); +} + +async function cloudLookupCoverageByStrategy(forumKey, startPage, endPage, strategy) { + var response = await callCloudCache('/api/shared-cache/coverages/lookup', { + forumKey: forumKey, + startPage: startPage, + endPage: endPage, + strategy: strategy + }); + + if (!response || !response.ok || !response.coverage) { + return null; + } + + return normalizeCloudCoveragePayload(response.coverage); +} + +async function cloudLookupCoverage(payload) { + var strategies = ['full_live', 'exact_cache', 'smart_incremental', 'assembled_pages']; + for (var index = 0; index < strategies.length; index++) { + var coverage = await cloudLookupCoverageByStrategy(payload.forumKey, payload.startPage, payload.endPage, strategies[index]); + if (coverage) { + return coverage; + } + } + return null; +} + +async function cloudLookupCoveragePlan(payload) { + var response = await callCloudCache('/api/shared-cache/coverages/plan', { + forumKey: payload.forumKey, + startPage: payload.startPage, + endPage: payload.endPage, + frontRefreshPages: payload.frontRefreshPages + }); + if (!response || response.ok === false) { + return null; + } + return response; +} + +async function cloudUpsertCoverage(payload) { + return callCloudCache('/api/shared-cache/coverages/upsert', { + forumKey: payload.forumKey, + startPage: payload.startPage, + endPage: payload.endPage, + strategy: payload.strategy, + crawledAt: payload.crawledAt, + threads: normalizeThreadsForCoverage(payload.threads, payload.forumKey, payload.crawledAt) + }, CLOUD_CACHE_TIMEOUT, { + 'x-shared-cache-write-token': CLOUD_SHARED_CACHE_WRITE_TOKEN + }); +} + +async function cloudLookupPageCoverage(payload) { + var response = await callCloudCache('/api/shared-cache/pages/lookup', { + forumKey: payload.forumKey, + page: payload.page + }); + + if (!response || !response.ok || !response.coverage) { + return null; + } + + return normalizeCloudPageCoveragePayload(response.coverage); +} + +async function cloudUpsertPageCoverage(payload) { + return callCloudCache('/api/shared-cache/pages/upsert', { + forumKey: payload.forumKey, + page: payload.page, + crawledAt: payload.crawledAt, + threads: normalizeThreadsForCoverage(payload.threads, payload.forumKey, payload.crawledAt) + }, CLOUD_CACHE_TIMEOUT, { + 'x-shared-cache-write-token': CLOUD_SHARED_CACHE_WRITE_TOKEN + }); +} + +async function getCloudAssembledPageCoverage(forumKey, startPage, endPage) { + var requests = []; + for (var page = startPage; page <= endPage; page++) { + requests.push(cloudLookupPageCoverage({ forumKey: forumKey, page: page })); + } + + var results = await Promise.all(requests); + if (results.some(function(item) { return !item; })) { + return null; + } + + var merged = Object.create(null); + var threads = []; + var latestCrawledAt = 0; + results.forEach(function(item) { + latestCrawledAt = Math.max(latestCrawledAt, Number(item.crawledAt || 0)); + (Array.isArray(item.threads) ? item.threads : []).forEach(function(thread) { + var key = typeof thread.threadKey === 'string' && thread.threadKey ? thread.threadKey : getThreadKeyFromUrl(thread.url); + if (!key || merged[key]) { + return; + } + merged[key] = true; + threads.push({ + threadKey: key, + url: normalizeUrl(thread.url), + title: normalizeTitle(thread.title) + }); + }); + }); + + return { + forumKey: forumKey, + startPage: startPage, + endPage: endPage, + crawledAt: latestCrawledAt || Date.now(), + strategy: 'assembled_pages', + frontRefreshPages: 0, + threads: threads + }; +} + async function upsertThreadEntries(threadStore, forumKey, rawThreads, crawledAt) { var seenThreadKeys = Object.create(null); var threadKeys = []; @@ -192,12 +1162,13 @@ async function upsertThreadEntries(threadStore, forumKey, rawThreads, crawledAt) threadKeys.push(normalized.cacheKey); var existingRecord = await requestToPromise(threadStore.get(normalized.cacheKey)); - var mergedMagnets = existingRecord && Array.isArray(existingRecord.magnets) - ? existingRecord.magnets - : normalized.magnets; - var mergedLastMagnetSyncAt = existingRecord && Number(existingRecord.lastMagnetSyncAt) - ? Number(existingRecord.lastMagnetSyncAt) - : normalized.lastMagnetSyncAt; + var incomingMagnets = normalizeMagnets(normalized.magnets); + var mergedMagnets = incomingMagnets.length > 0 + ? incomingMagnets + : (existingRecord && Array.isArray(existingRecord.magnets) ? existingRecord.magnets : []); + var mergedLastMagnetSyncAt = incomingMagnets.length > 0 + ? (Number(normalized.lastMagnetSyncAt) || crawledAt) + : (existingRecord && Number(existingRecord.lastMagnetSyncAt) ? Number(existingRecord.lastMagnetSyncAt) : (normalized.lastMagnetSyncAt || 0)); threadStore.put({ cacheKey: normalized.cacheKey, @@ -261,6 +1232,106 @@ async function getAssembledPageCoverage(db, forumKey, startPage, endPage) { }); } +async function getCachedPageCoverageBlocks(db, forumKey, startPage, endPage) { + var tx = db.transaction('pageCoverages', 'readonly'); + var store = tx.objectStore('pageCoverages'); + var index = store.index('forumKey'); + var request = index.openCursor(IDBKeyRange.only(forumKey)); + var pageRecords = []; + + var records = await new Promise(function(resolve, reject) { + request.onsuccess = function(event) { + var cursor = event.target.result; + var record = null; + if (!cursor) { + resolve(pageRecords); + return; + } + record = cursor.value; + if (record && Number(record.page) >= startPage && Number(record.page) <= endPage) { + pageRecords.push(record); + } + cursor.continue(); + }; + request.onerror = function(event) { + reject(event.target.error || new Error('读取页缓存块失败')); + }; + }); + await transactionDone(tx); + + if (!records.length) { + return []; + } + + records.sort(function(a, b) { + return Number(a.page || 0) - Number(b.page || 0); + }); + + var blocks = []; + var currentBlock = null; + var latestCrawledAt = 0; + var mergedThreadKeys = null; + + records.forEach(function(record) { + var page = Number(record.page || 0); + if (!currentBlock) { + currentBlock = { + forumKey: forumKey, + startPage: page, + endPage: page, + pages: [record], + crawledAt: Number(record.crawledAt || 0) + }; + return; + } + + if (page === currentBlock.endPage + 1) { + currentBlock.endPage = page; + currentBlock.pages.push(record); + currentBlock.crawledAt = Math.max(currentBlock.crawledAt, Number(record.crawledAt || 0)); + return; + } + + blocks.push(currentBlock); + currentBlock = { + forumKey: forumKey, + startPage: page, + endPage: page, + pages: [record], + crawledAt: Number(record.crawledAt || 0) + }; + }); + + if (currentBlock) { + blocks.push(currentBlock); + } + + return blocks.map(function(block) { + var merged = Object.create(null); + mergedThreadKeys = []; + latestCrawledAt = 0; + block.pages.forEach(function(record) { + latestCrawledAt = Math.max(latestCrawledAt, Number(record.crawledAt || 0)); + (Array.isArray(record.threadKeys) ? record.threadKeys : []).forEach(function(threadKey) { + if (merged[threadKey]) { + return; + } + merged[threadKey] = true; + mergedThreadKeys.push(threadKey); + }); + }); + return { + forumKey: forumKey, + startPage: block.startPage, + endPage: block.endPage, + crawledAt: latestCrawledAt, + strategy: 'assembled_pages', + frontRefreshPages: 0, + threadKeys: mergedThreadKeys + }; + }); +} + async function getCoverageWithThreads(db, coverageRecord) { if (!coverageRecord) { return null; @@ -325,6 +1396,60 @@ async function listForumCoverages(db, forumKey) { }); } +async function getIntersectingCoverageBlocks(db, forumKey, startPage, endPage) { + var coverages = await listForumCoverages(db, forumKey); + var filtered = coverages.filter(function(record) { + var recordStart = Number(record.startPage || 0); + var recordEnd = Number(record.endPage || 0); + return recordStart <= endPage && recordEnd >= startPage; + }).sort(function(a, b) { + var aStart = Number(a.startPage || 0); + var bStart = Number(b.startPage || 0); + if (aStart !== bStart) { + return aStart - bStart; + } + return Number(b.endPage || 0) - Number(a.endPage || 0); + }); + + var result = []; + var seenRanges = Object.create(null); + var index = 0; + var record = null; + var hydrated = null; + var clippedStart = 0; + var clippedEnd = 0; + var rangeKey = ''; + + for (index = 0; index < filtered.length; index++) { + record = filtered[index]; + hydrated = await getCoverageWithThreads(db, record); + if (!hydrated || !Array.isArray(hydrated.threads) || hydrated.threads.length === 0) { + continue; + } + clippedStart = Math.max(startPage, Number(hydrated.startPage || startPage)); + clippedEnd = Math.min(endPage, Number(hydrated.endPage || endPage)); + if (clippedStart > clippedEnd) { + continue; + } + rangeKey = String(clippedStart) + '-' + String(clippedEnd); + if (seenRanges[rangeKey]) { + continue; + } + seenRanges[rangeKey] = true; + result.push({ + forumKey: hydrated.forumKey, + startPage: clippedStart, + endPage: clippedEnd, + crawledAt: hydrated.crawledAt, + strategy: hydrated.strategy || 'full_live', + frontRefreshPages: hydrated.frontRefreshPages || 0, + threads: hydrated.threads + }); + } + + return result; +} + async function pruneOldCoverages(db, forumKey) { var coverages = await listForumCoverages(db, forumKey); if (coverages.length <= COVERAGE_LIMIT_PER_FORUM) { @@ -355,14 +1480,30 @@ async function saveCoverageSnapshot(payload) { var crawledAt = Number(payload.crawledAt) || Date.now(); var frontRefreshPages = Math.max(0, Number(payload.frontRefreshPages) || 0); var strategy = typeof payload.strategy === 'string' ? payload.strategy : 'full_live'; + var skipCloudSync = !!payload.skipCloudSync; var rawThreads = Array.isArray(payload.threads) ? payload.threads : []; + var normalizedThreadsForCoverage = normalizeThreadsForCoverage(rawThreads, forumKey, crawledAt); + var intersectingBlocks = null; + var historicalThreads = []; + var mergedCoverageThreads = null; + var result = null; var db = await openCacheDb(); try { + intersectingBlocks = await getIntersectingCoverageBlocks(db, forumKey, startPage, endPage); + historicalThreads = []; + intersectingBlocks.forEach(function(block) { + (Array.isArray(block.threads) ? block.threads : []).forEach(function(thread) { + historicalThreads.push(thread); + }); + }); + mergedCoverageThreads = mergeCoverageThreadLists(forumKey, crawledAt, rawThreads, historicalThreads); + normalizedThreadsForCoverage = normalizeThreadsForCoverage(mergedCoverageThreads, forumKey, crawledAt); + var tx = db.transaction(['threads', 'coverages'], 'readwrite'); var threadStore = tx.objectStore('threads'); var coverageStore = tx.objectStore('coverages'); - var threadKeys = await upsertThreadEntries(threadStore, forumKey, rawThreads, crawledAt); + var threadKeys = await upsertThreadEntries(threadStore, forumKey, mergedCoverageThreads, crawledAt); coverageStore.put({ coverageKey: getCoverageKey(forumKey, startPage, endPage), @@ -379,7 +1520,7 @@ async function saveCoverageSnapshot(payload) { await transactionDone(tx); await pruneOldCoverages(db, forumKey); - return { + result = { ok: true, threadCount: threadKeys.length, coverageKey: getCoverageKey(forumKey, startPage, endPage) @@ -387,13 +1528,65 @@ async function saveCoverageSnapshot(payload) { } finally { db.close(); } + + if (!skipCloudSync && forumKey && normalizedThreadsForCoverage.length > 0) { + try { + var coverageThreadUploadPlan = await filterThreadUploadsForCloud( + forumKey, + normalizeThreadsForCloudIndex(mergedCoverageThreads, forumKey, crawledAt), + { ttlMs: CLOUD_THREAD_UPLOAD_TTL_MS } + ); + if (coverageThreadUploadPlan.uploads.length > 0) { + var threadUpsertResult = await cloudUpsertThreadMagnets({ + forumKey: forumKey, + syncedAt: crawledAt, + threads: coverageThreadUploadPlan.uploads + }); + if (threadUpsertResult && threadUpsertResult.ok) { + await markUploadMetaEntries('thread', coverageThreadUploadPlan.pendingMeta); + } + } + + var coverageUploadCheck = await shouldUploadCoverageToCloud({ + forumKey: forumKey, + startPage: startPage, + endPage: endPage, + strategy: strategy, + crawledAt: crawledAt, + threads: normalizedThreadsForCoverage + }); + if (coverageUploadCheck.shouldUpload) { + var coverageUpsertResult = await cloudUpsertCoverage({ + forumKey: forumKey, + startPage: startPage, + endPage: endPage, + strategy: strategy, + crawledAt: crawledAt, + threads: normalizedThreadsForCoverage + }); + if (coverageUpsertResult && coverageUpsertResult.ok) { + await markUploadMetaEntries('coverage', [coverageUploadCheck]); + } + } + } catch (error) { + console.warn('[MagnetPlugin][Cloud] save coverage failed:', error && error.message ? error.message : error); + } + backfillLocalCacheToCloud({ force: false }).catch(function(backfillErr) { + console.warn('[MagnetPlugin][Cloud] post-coverage backfill failed:', backfillErr && backfillErr.message ? backfillErr.message : backfillErr); + }); + } + + return result; } async function savePageCoverageSnapshot(payload) { var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : ''; var page = Math.max(1, Number(payload.page) || 1); var crawledAt = Number(payload.crawledAt) || Date.now(); + var skipCloudSync = !!payload.skipCloudSync; var rawThreads = Array.isArray(payload.threads) ? payload.threads : []; + var normalizedThreadsForCoverage = normalizeThreadsForCoverage(rawThreads, forumKey, crawledAt); + var result = null; var db = await openCacheDb(); try { @@ -413,7 +1606,7 @@ async function savePageCoverageSnapshot(payload) { await transactionDone(tx); - return { + result = { ok: true, page: page, threadCount: threadKeys.length, @@ -422,12 +1615,56 @@ async function savePageCoverageSnapshot(payload) { } finally { db.close(); } + + if (!skipCloudSync && forumKey && normalizedThreadsForCoverage.length > 0) { + try { + var pageThreadUploadPlan = await filterThreadUploadsForCloud( + forumKey, + normalizeThreadsForCloudIndex(rawThreads, forumKey, crawledAt), + { ttlMs: CLOUD_THREAD_UPLOAD_TTL_MS } + ); + if (pageThreadUploadPlan.uploads.length > 0) { + var pageThreadUpsertResult = await cloudUpsertThreadMagnets({ + forumKey: forumKey, + syncedAt: crawledAt, + threads: pageThreadUploadPlan.uploads + }); + if (pageThreadUpsertResult && pageThreadUpsertResult.ok) { + await markUploadMetaEntries('thread', pageThreadUploadPlan.pendingMeta); + } + } + + var pageUploadCheck = await shouldUploadPageToCloud({ + forumKey: forumKey, + page: page, + crawledAt: crawledAt, + threads: normalizedThreadsForCoverage + }); + if (pageUploadCheck.shouldUpload) { + var pageUpsertResult = await cloudUpsertPageCoverage({ + forumKey: forumKey, + page: page, + crawledAt: crawledAt, + threads: normalizedThreadsForCoverage + }); + if (pageUpsertResult && pageUpsertResult.ok) { + await markUploadMetaEntries('page', [pageUploadCheck]); + } + } + } catch (error) { + console.warn('[MagnetPlugin][Cloud] save page coverage failed:', error && error.message ? error.message : error); + } + } + + return result; } async function getCachedThreadMagnets(payload) { var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : ''; var rawThreads = Array.isArray(payload.threads) ? payload.threads : []; var db = await openCacheDb(); + var misses = []; + var resultMap = Object.create(null); try { var tx = db.transaction('threads', 'readonly'); @@ -449,23 +1686,76 @@ async function getCachedThreadMagnets(payload) { title: normalized.title, magnets: [] }); + misses.push({ + threadKey: normalized.threadKey, + url: normalized.url, + title: normalized.title + }); continue; } + var recordMagnets = normalizeMagnets(record.magnets); + if (recordMagnets.length === 0 && !Number(record.lastMagnetSyncAt)) { + misses.push({ + threadKey: record.threadKey, + url: record.url, + title: record.title + }); + } + results.push({ threadKey: record.threadKey, url: record.url, title: record.title, - magnets: normalizeMagnets(record.magnets), + magnets: recordMagnets, lastMagnetSyncAt: Number(record.lastMagnetSyncAt) || 0 }); } await transactionDone(tx); + results.forEach(function(item) { + if (item && item.threadKey) { + resultMap[item.threadKey] = item; + } + }); + + if (misses.length > 0) { + try { + var cloudThreads = await cloudLookupThreadMagnets({ + forumKey: forumKey, + threads: misses + }); + if (cloudThreads.length > 0) { + await saveThreadMagnets({ + forumKey: forumKey, + syncedAt: Date.now(), + threads: cloudThreads, + skipCloudSync: true + }); + cloudThreads.forEach(function(item) { + if (!item || !item.threadKey) { + return; + } + resultMap[item.threadKey] = { + threadKey: item.threadKey, + url: item.url, + title: item.title, + magnets: normalizeMagnets(item.magnets), + lastMagnetSyncAt: Number(item.lastSeenAt) || Date.now() + }; + }); + } + } catch (error) { + console.warn('[MagnetPlugin][Cloud] lookup thread magnets failed:', error && error.message ? error.message : error); + } + } + return { ok: true, - threads: results + threads: results.map(function(item) { + return item && item.threadKey && resultMap[item.threadKey] ? resultMap[item.threadKey] : item; + }) }; } finally { db.close(); @@ -476,6 +1766,9 @@ async function saveThreadMagnets(payload) { var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : ''; var rawThreads = Array.isArray(payload.threads) ? payload.threads : []; var syncedAt = Number(payload.syncedAt) || Date.now(); + var skipCloudSync = !!payload.skipCloudSync; + var cloudThreads = []; + var result = null; var db = await openCacheDb(); try { @@ -506,26 +1799,59 @@ async function saveThreadMagnets(payload) { firstSeenAt: existingRecord && Number(existingRecord.firstSeenAt) ? Number(existingRecord.firstSeenAt) : normalized.firstSeenAt, - lastSeenAt: existingRecord && Number(existingRecord.lastSeenAt) - ? Number(existingRecord.lastSeenAt) - : normalized.lastSeenAt, + lastSeenAt: Math.max(Number(existingRecord && existingRecord.lastSeenAt || 0), Number(normalized.lastSeenAt || 0)) || Date.now(), magnets: finalMagnets, magnetCount: finalMagnets.length, lastMagnetSyncAt: finalMagnets.length > 0 ? syncedAt : (existingRecord && Number(existingRecord.lastMagnetSyncAt) ? Number(existingRecord.lastMagnetSyncAt) : 0) }); + if (finalMagnets.length > 0) { + cloudThreads.push({ + threadKey: normalized.threadKey, + url: normalized.url, + title: normalized.title || (existingRecord ? existingRecord.title : ''), + magnets: finalMagnets, + lastSeenAt: syncedAt + }); + } + savedCount += 1; } await transactionDone(tx); - return { + result = { ok: true, savedCount: savedCount }; } finally { db.close(); } + + if (!skipCloudSync && forumKey && cloudThreads.length > 0) { + try { + var threadUploadPlan = await filterThreadUploadsForCloud(forumKey, cloudThreads, { + ttlMs: CLOUD_THREAD_UPLOAD_TTL_MS + }); + if (threadUploadPlan.uploads.length > 0) { + var directThreadUpsertResult = await cloudUpsertThreadMagnets({ + forumKey: forumKey, + syncedAt: syncedAt, + threads: threadUploadPlan.uploads + }); + if (directThreadUpsertResult && directThreadUpsertResult.ok) { + await markUploadMetaEntries('thread', threadUploadPlan.pendingMeta); + } + } + } catch (error) { + console.warn('[MagnetPlugin][Cloud] save thread magnets failed:', error && error.message ? error.message : error); + } + backfillLocalCacheToCloud({ force: false }).catch(function(backfillErr) { + console.warn('[MagnetPlugin][Cloud] post-thread-save backfill failed:', backfillErr && backfillErr.message ? backfillErr.message : backfillErr); + }); + } + + return result; } function estimateTextSize(value) { @@ -614,6 +1940,450 @@ async function readRecentCoverageRecords(db, forumKey, limit) { }); } +async function readAllStoreRecords(db, storeName) { + return new Promise(function(resolve, reject) { + var tx = db.transaction(storeName, 'readonly'); + var store = tx.objectStore(storeName); + var request = store.openCursor(); + var items = []; + + request.onsuccess = function(event) { + var cursor = event.target.result; + if (!cursor) { + resolve(items); + return; + } + items.push(cursor.value); + cursor.continue(); + }; + + request.onerror = function(event) { + reject(event.target.error || new Error('读取 ' + storeName + ' 失败')); + }; + }); +} + +async function getThreadsByKeys(db, threadKeys) { + var keys = Array.isArray(threadKeys) ? threadKeys.filter(Boolean) : []; + var threadTx = null; + var threadStore = null; + var threadRequests = null; + var threadResults = null; + if (keys.length === 0) { + return []; + } + threadTx = db.transaction('threads', 'readonly'); + threadStore = threadTx.objectStore('threads'); + threadRequests = keys.map(function(threadKey) { + return requestToPromise(threadStore.get(threadKey)); + }); + threadResults = await Promise.all(threadRequests); + await transactionDone(threadTx); + return threadResults.filter(Boolean).map(function(record) { + return { + threadKey: record.threadKey, + url: record.url, + title: record.title, + magnets: normalizeMagnets(record.magnets), + lastSeenAt: Number(record.lastSeenAt) || Date.now() + }; + }); +} + +function chunkArray(items, size) { + var chunks = []; + var index = 0; + var safeSize = Math.max(1, Number(size) || 1); + for (index = 0; index < items.length; index += safeSize) { + chunks.push(items.slice(index, index + safeSize)); + } + return chunks; +} + +async function getCloudBackfillState() { + var result = await storageLocalGet(CLOUD_CACHE_BACKFILL_KEY); + return result[CLOUD_CACHE_BACKFILL_KEY] || null; +} + +async function saveCloudBackfillState(state) { + var data = {}; + data[CLOUD_CACHE_BACKFILL_KEY] = state; + await storageLocalSet(data); +} + +function normalizeCloudUploadMetaState(state) { + state = state && typeof state === 'object' ? state : {}; + return { + threads: state.threads && typeof state.threads === 'object' ? state.threads : {}, + coverages: state.coverages && typeof state.coverages === 'object' ? state.coverages : {}, + pages: state.pages && typeof state.pages === 'object' ? state.pages : {} + }; +} + +async function loadCloudUploadMetaState() { + var result = null; + if (cloudUploadMetaCache) { + return cloudUploadMetaCache; + } + result = await storageLocalGet(CLOUD_UPLOAD_META_KEY); + cloudUploadMetaCache = normalizeCloudUploadMetaState(result[CLOUD_UPLOAD_META_KEY]); + return cloudUploadMetaCache; +} + +function pruneUploadMetaStore(store, limit) { + var keys = Object.keys(store || {}); + var safeLimit = Math.max(100, Number(limit) || 1000); + var staleKeys = null; + if (keys.length <= safeLimit) { + return store; + } + staleKeys = keys.sort(function(a, b) { + return Number((store[a] && store[a].lastUploadedAt) || 0) - Number((store[b] && store[b].lastUploadedAt) || 0); + }).slice(0, keys.length - safeLimit); + staleKeys.forEach(function(key) { + delete store[key]; + }); + return store; +} + +async function saveCloudUploadMetaState(state) { + var data = {}; + var normalized = normalizeCloudUploadMetaState(state); + pruneUploadMetaStore(normalized.threads, CLOUD_UPLOAD_META_THREAD_LIMIT); + pruneUploadMetaStore(normalized.coverages, CLOUD_UPLOAD_META_OTHER_LIMIT); + pruneUploadMetaStore(normalized.pages, CLOUD_UPLOAD_META_OTHER_LIMIT); + cloudUploadMetaCache = normalized; + data[CLOUD_UPLOAD_META_KEY] = normalized; + await storageLocalSet(data); + return normalized; +} + +function getUploadMetaBucket(state, type) { + if (type === 'thread') { + return state.threads; + } + if (type === 'coverage') { + return state.coverages; + } + return state.pages; +} + +function buildThreadUploadMetaKey(forumKey, threadKey) { + return String(forumKey || '') + '::' + String(threadKey || ''); +} + +function buildCoverageUploadMetaKey(forumKey, startPage, endPage, strategy) { + return String(forumKey || '') + '::' + String(startPage || 1) + '-' + String(endPage || 1) + '::' + String(strategy || 'full_live'); +} + +function buildPageUploadMetaKey(forumKey, page) { + return String(forumKey || '') + '::page::' + String(page || 1); +} + +function shouldSkipUploadMetaEntry(entry, payloadHash, ttlMs, now) { + return !!( + entry && + entry.payloadHash === payloadHash && + Number(entry.lastUploadedAt || 0) > 0 && + now - Number(entry.lastUploadedAt || 0) < ttlMs + ); +} + +function sortThreadsForHash(threads) { + return (Array.isArray(threads) ? threads : []).map(function(thread) { + return { + threadKey: String(thread.threadKey || ''), + url: normalizeUrl(thread.url), + title: normalizeTitle(thread.title), + magnets: normalizeMagnets(thread.magnets).slice().sort() + }; + }).sort(function(a, b) { + return String(a.threadKey).localeCompare(String(b.threadKey)); + }); +} + +async function buildThreadUploadHash(forumKey, thread) { + return sha256Hex(JSON.stringify({ + forumKey: String(forumKey || ''), + threadKey: String(thread.threadKey || ''), + url: normalizeUrl(thread.url), + title: normalizeTitle(thread.title), + magnets: normalizeMagnets(thread.magnets).slice().sort() + })); +} + +async function buildCoverageUploadHash(payload) { + return sha256Hex(JSON.stringify({ + forumKey: String(payload.forumKey || ''), + startPage: Number(payload.startPage || 1), + endPage: Number(payload.endPage || payload.startPage || 1), + strategy: String(payload.strategy || 'full_live'), + threads: sortThreadsForHash(payload.threads) + })); +} + +async function buildPageUploadHash(payload) { + return sha256Hex(JSON.stringify({ + forumKey: String(payload.forumKey || ''), + page: Number(payload.page || 1), + threads: sortThreadsForHash(payload.threads) + })); +} + +async function filterThreadUploadsForCloud(forumKey, threads, options) { + var state = await loadCloudUploadMetaState(); + var bucket = getUploadMetaBucket(state, 'thread'); + var now = Date.now(); + var ttlMs = options && Number(options.ttlMs) >= 0 ? Number(options.ttlMs) : CLOUD_THREAD_UPLOAD_TTL_MS; + var uploads = []; + var pendingMeta = []; + var index = 0; + var item = null; + var key = ''; + var payloadHash = ''; + + for (index = 0; index < threads.length; index++) { + item = threads[index]; + if (!item || !item.threadKey) { + continue; + } + key = buildThreadUploadMetaKey(forumKey, item.threadKey); + payloadHash = await buildThreadUploadHash(forumKey, item); + if (shouldSkipUploadMetaEntry(bucket[key], payloadHash, ttlMs, now)) { + continue; + } + uploads.push(item); + pendingMeta.push({ key: key, payloadHash: payloadHash, lastUploadedAt: now }); + } + + return { uploads: uploads, pendingMeta: pendingMeta }; +} + +async function shouldUploadCoverageToCloud(payload, options) { + var state = await loadCloudUploadMetaState(); + var bucket = getUploadMetaBucket(state, 'coverage'); + var now = Date.now(); + var ttlMs = options && Number(options.ttlMs) >= 0 ? Number(options.ttlMs) : CLOUD_COVERAGE_UPLOAD_TTL_MS; + var key = buildCoverageUploadMetaKey(payload.forumKey, payload.startPage, payload.endPage, payload.strategy); + var payloadHash = await buildCoverageUploadHash(payload); + if (shouldSkipUploadMetaEntry(bucket[key], payloadHash, ttlMs, now)) { + return { shouldUpload: false, key: key, payloadHash: payloadHash, lastUploadedAt: now }; + } + return { shouldUpload: true, key: key, payloadHash: payloadHash, lastUploadedAt: now }; +} + +async function shouldUploadPageToCloud(payload, options) { + var state = await loadCloudUploadMetaState(); + var bucket = getUploadMetaBucket(state, 'page'); + var now = Date.now(); + var ttlMs = options && Number(options.ttlMs) >= 0 ? Number(options.ttlMs) : CLOUD_PAGE_UPLOAD_TTL_MS; + var key = buildPageUploadMetaKey(payload.forumKey, payload.page); + var payloadHash = await buildPageUploadHash(payload); + if (shouldSkipUploadMetaEntry(bucket[key], payloadHash, ttlMs, now)) { + return { shouldUpload: false, key: key, payloadHash: payloadHash, lastUploadedAt: now }; + } + return { shouldUpload: true, key: key, payloadHash: payloadHash, lastUploadedAt: now }; +} + +async function markUploadMetaEntries(type, entries) { + var state = await loadCloudUploadMetaState(); + var bucket = getUploadMetaBucket(state, type); + (Array.isArray(entries) ? entries : []).forEach(function(entry) { + if (!entry || !entry.key) { + return; + } + bucket[entry.key] = { + payloadHash: entry.payloadHash, + lastUploadedAt: Number(entry.lastUploadedAt) || Date.now() + }; + }); + await saveCloudUploadMetaState(state); +} + +async function backfillLocalCacheToCloud(options) { + var settings = options && typeof options === 'object' ? options : {}; + var force = !!settings.force; + var state = null; + var authState = null; + var db = null; + var threadRecords = []; + var coverageRecords = []; + var pageRecords = []; + var groupedThreads = Object.create(null); + var forumKeys = []; + var forumKeySeen = Object.create(null); + var coverageIndex = 0; + var pageIndex = 0; + var chunkIndex = 0; + var chunks = null; + var currentChunk = null; + var counts = { + threadCount: 0, + coverageCount: 0, + pageCount: 0 + }; + + if (cloudBackfillPromise) { + return cloudBackfillPromise; + } + + cloudBackfillPromise = (async function() { + authState = await loadCloudAuthState(); + if (!authState.authenticated && !authState.token) { + return { ok: true, skipped: true, reason: 'not_authenticated' }; + } + + state = await getCloudBackfillState(); + if (!force && state && Number(state.lastRunAt || 0) > Date.now() - 10 * 60 * 1000) { + return { ok: true, skipped: true, reason: 'recently_ran', state: state }; + } + + db = await openCacheDb(); + try { + threadRecords = await readAllStoreRecords(db, 'threads'); + coverageRecords = await readAllStoreRecords(db, 'coverages'); + pageRecords = await readAllStoreRecords(db, 'pageCoverages'); + + threadRecords.forEach(function(record) { + if (!record || !record.forumKey || !record.threadKey) { + return; + } + var recordMagnets = normalizeMagnets(record.magnets); + if (!groupedThreads[record.forumKey]) { + groupedThreads[record.forumKey] = []; + } + groupedThreads[record.forumKey].push({ + threadKey: record.threadKey, + url: record.url, + title: record.title, + magnets: recordMagnets, + lastSeenAt: Number(record.lastSeenAt) || Date.now() + }); + if (!forumKeySeen[record.forumKey]) { + forumKeySeen[record.forumKey] = true; + forumKeys.push(record.forumKey); + } + }); + + forumKeys.forEach(function(fk) { + if (groupedThreads[fk]) { + groupedThreads[fk].sort(function(a, b) { + return (b.magnets.length > 0 ? 1 : 0) - (a.magnets.length > 0 ? 1 : 0); + }); + } + }); + + for (chunkIndex = 0; chunkIndex < forumKeys.length; chunkIndex++) { + chunks = chunkArray(groupedThreads[forumKeys[chunkIndex]] || [], 200); + for (pageIndex = 0; pageIndex < chunks.length; pageIndex++) { + currentChunk = chunks[pageIndex]; + var threadUploadPlan = null; + if (currentChunk.length === 0) { + continue; + } + threadUploadPlan = await filterThreadUploadsForCloud(forumKeys[chunkIndex], currentChunk, { + ttlMs: CLOUD_THREAD_UPLOAD_TTL_MS + }); + if (threadUploadPlan.uploads.length === 0) { + continue; + } + var backfillThreadResult = await cloudUpsertThreadMagnets({ + forumKey: forumKeys[chunkIndex], + syncedAt: Date.now(), + threads: threadUploadPlan.uploads + }); + if (backfillThreadResult && backfillThreadResult.ok) { + await markUploadMetaEntries('thread', threadUploadPlan.pendingMeta); + counts.threadCount += threadUploadPlan.uploads.length; + } + } + } + + for (coverageIndex = 0; coverageIndex < coverageRecords.length; coverageIndex++) { + var coverageRecord = coverageRecords[coverageIndex]; + var coverageThreads = await getThreadsByKeys(db, coverageRecord.threadKeys); + var coverageUploadCheck = null; + if (!coverageRecord || !coverageRecord.forumKey || coverageThreads.length === 0) { + continue; + } + coverageUploadCheck = await shouldUploadCoverageToCloud({ + forumKey: coverageRecord.forumKey, + startPage: Number(coverageRecord.startPage) || 1, + endPage: Number(coverageRecord.endPage) || Number(coverageRecord.startPage) || 1, + strategy: coverageRecord.strategy || 'full_live', + crawledAt: Number(coverageRecord.crawledAt) || Date.now(), + threads: coverageThreads + }, { + ttlMs: CLOUD_COVERAGE_UPLOAD_TTL_MS + }); + if (!coverageUploadCheck.shouldUpload) { + continue; + } + var backfillCoverageResult = await cloudUpsertCoverage({ + forumKey: coverageRecord.forumKey, + startPage: Number(coverageRecord.startPage) || 1, + endPage: Number(coverageRecord.endPage) || Number(coverageRecord.startPage) || 1, + strategy: coverageRecord.strategy || 'full_live', + crawledAt: Number(coverageRecord.crawledAt) || Date.now(), + threads: coverageThreads + }); + if (backfillCoverageResult && backfillCoverageResult.ok) { + await markUploadMetaEntries('coverage', [coverageUploadCheck]); + counts.coverageCount += 1; + } + } + + for (pageIndex = 0; pageIndex < pageRecords.length; pageIndex++) { + var pageRecord = pageRecords[pageIndex]; + var pageThreads = await getThreadsByKeys(db, pageRecord.threadKeys); + var pageUploadCheck = null; + if (!pageRecord || !pageRecord.forumKey || pageThreads.length === 0) { + continue; + } + pageUploadCheck = await shouldUploadPageToCloud({ + forumKey: pageRecord.forumKey, + page: Number(pageRecord.page) || 1, + crawledAt: Number(pageRecord.crawledAt) || Date.now(), + threads: pageThreads + }, { + ttlMs: CLOUD_PAGE_UPLOAD_TTL_MS + }); + if (!pageUploadCheck.shouldUpload) { + continue; + } + var backfillPageResult = await cloudUpsertPageCoverage({ + forumKey: pageRecord.forumKey, + page: Number(pageRecord.page) || 1, + crawledAt: Number(pageRecord.crawledAt) || Date.now(), + threads: pageThreads + }); + if (backfillPageResult && backfillPageResult.ok) { + await markUploadMetaEntries('page', [pageUploadCheck]); + counts.pageCount += 1; + } + } + + await saveCloudBackfillState({ + lastRunAt: Date.now(), + counts: counts + }); + await setCloudHealth(true); + return { ok: true, counts: counts }; + } finally { + if (db) { + db.close(); + } + } + })().catch(async function(error) { + console.warn('[MagnetPlugin][Cloud] backfill failed:', error && error.message ? error.message : error); + throw error; + }).finally(function() { + cloudBackfillPromise = null; + }); + + return cloudBackfillPromise; +} + async function getCacheOverview(payload) { var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : ''; var limit = Math.max(1, Math.min(Number(payload.limit) || 12, 50)); @@ -716,10 +2486,14 @@ async function getCachedCoveragePlan(payload) { var startPage = Math.max(1, Number(payload.startPage) || 1); var endPage = Math.max(startPage, Number(payload.endPage) || startPage); var frontRefreshPages = Math.max(0, Number(payload.frontRefreshPages) || 0); + var cloudCoverage = null; + var cloudAssembledCoverage = null; + var cachedBlocks = []; var db = await openCacheDb(); try { var exactKey = getCoverageKey(forumKey, startPage, endPage); + var serverPlan = null; var exactTx = db.transaction('coverages', 'readonly'); var exactStore = exactTx.objectStore('coverages'); var exactRecord = await requestToPromise(exactStore.get(exactKey)); @@ -729,16 +2503,117 @@ async function getCachedCoveragePlan(payload) { if (!exactCoverage) { exactCoverage = await getAssembledPageCoverage(db, forumKey, startPage, endPage); } + if (!exactCoverage) { + try { + cloudCoverage = await cloudLookupCoverage({ + forumKey: forumKey, + startPage: startPage, + endPage: endPage + }); + if (cloudCoverage && Array.isArray(cloudCoverage.threads) && cloudCoverage.threads.length > 0) { + await saveCoverageSnapshot({ + forumKey: cloudCoverage.forumKey, + startPage: cloudCoverage.startPage, + endPage: cloudCoverage.endPage, + crawledAt: cloudCoverage.crawledAt, + strategy: cloudCoverage.strategy, + frontRefreshPages: cloudCoverage.frontRefreshPages, + threads: cloudCoverage.threads, + skipCloudSync: true + }); + exactCoverage = cloudCoverage; + } + } catch (error) { + console.warn('[MagnetPlugin][Cloud] lookup exact coverage failed:', error && error.message ? error.message : error); + } + } + if (!exactCoverage) { + try { + cloudAssembledCoverage = await getCloudAssembledPageCoverage(forumKey, startPage, endPage); + if (cloudAssembledCoverage && Array.isArray(cloudAssembledCoverage.threads) && cloudAssembledCoverage.threads.length > 0) { + await saveCoverageSnapshot({ + forumKey: cloudAssembledCoverage.forumKey, + startPage: cloudAssembledCoverage.startPage, + endPage: cloudAssembledCoverage.endPage, + crawledAt: cloudAssembledCoverage.crawledAt, + strategy: cloudAssembledCoverage.strategy, + frontRefreshPages: cloudAssembledCoverage.frontRefreshPages, + threads: cloudAssembledCoverage.threads, + skipCloudSync: true + }); + exactCoverage = cloudAssembledCoverage; + } + } catch (error) { + console.warn('[MagnetPlugin][Cloud] assemble page coverage failed:', error && error.message ? error.message : error); + } + } + if (!exactCoverage) { + try { + serverPlan = await cloudLookupCoveragePlan({ + forumKey: forumKey, + startPage: startPage, + endPage: endPage, + frontRefreshPages: frontRefreshPages + }); + if (serverPlan && serverPlan.exactCoverage && Array.isArray(serverPlan.exactCoverage.threads) && serverPlan.exactCoverage.threads.length > 0) { + await saveCoverageSnapshot({ + forumKey: serverPlan.exactCoverage.forumKey, + startPage: serverPlan.exactCoverage.startPage, + endPage: serverPlan.exactCoverage.endPage, + crawledAt: serverPlan.exactCoverage.crawledAt, + strategy: serverPlan.exactCoverage.strategy, + frontRefreshPages: serverPlan.exactCoverage.frontRefreshPages, + threads: serverPlan.exactCoverage.threads, + skipCloudSync: true + }); + exactCoverage = serverPlan.exactCoverage; + } + } catch (error) { + console.warn('[MagnetPlugin][Cloud] coverage plan failed:', error && error.message ? error.message : error); + } + } + if (exactCoverage) { return { ok: true, exactCoverage: exactCoverage, - shiftedCoverage: null + shiftedCoverage: null, + cachedBlocks: [] }; } - var shiftedCoverage = null; - if (startPage === 1 && frontRefreshPages > 0) { + try { + var localBlocks = await getCachedPageCoverageBlocks(db, forumKey, startPage, endPage); + var intersectingCoverages = await getIntersectingCoverageBlocks(db, forumKey, startPage, endPage); + for (var blockIndex = 0; blockIndex < localBlocks.length; blockIndex++) { + var hydratedBlock = await getCoverageWithThreads(db, localBlocks[blockIndex]); + if (hydratedBlock && Array.isArray(hydratedBlock.threads) && hydratedBlock.threads.length > 0) { + cachedBlocks.push(hydratedBlock); + } + } + intersectingCoverages.forEach(function(block) { + cachedBlocks.push(block); + }); + if (serverPlan && Array.isArray(serverPlan.cachedBlocks)) { + serverPlan.cachedBlocks.forEach(function(block) { + if (block && Array.isArray(block.threads) && block.threads.length > 0) { + cachedBlocks.push(block); + } + }); + } + cachedBlocks.sort(function(a, b) { + var startDiff = Number(a.startPage || 0) - Number(b.startPage || 0); + if (startDiff !== 0) { + return startDiff; + } + return Number(b.endPage || 0) - Number(a.endPage || 0); + }); + } catch (error) { + console.warn('[MagnetPlugin][Cache] build local cached blocks failed:', error && error.message ? error.message : error); + } + + var shiftedCoverage = serverPlan && serverPlan.shiftedCoverage ? serverPlan.shiftedCoverage : null; + if (!shiftedCoverage && startPage === 1 && frontRefreshPages > 0) { var coverages = await listForumCoverages(db, forumKey); coverages = coverages.filter(function(record) { return Number(record.startPage) === 1 && Number(record.endPage) >= 1; @@ -775,7 +2650,8 @@ async function getCachedCoveragePlan(payload) { return { ok: true, exactCoverage: null, - shiftedCoverage: shiftedCoverage + shiftedCoverage: shiftedCoverage, + cachedBlocks: cachedBlocks }; } finally { db.close(); @@ -982,6 +2858,116 @@ function extractMagnetsFromHtml(html) { } chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { + if (request.action === 'openCloudSyncPage') { + chrome.tabs.create({ url: chrome.runtime.getURL('popup.html?view=cloud') }, function(tab) { + if (chrome.runtime.lastError) { + sendResponse({ ok: false, error: chrome.runtime.lastError.message }); + return; + } + sendResponse({ ok: true, tabId: tab && typeof tab.id === 'number' ? tab.id : null }); + }); + return true; + } + + if (request.action === 'cloudRegister') { + cloudRegisterAccount(request) + .then(function(result) { + sendResponse(result); + }) + .catch(function(err) { + sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); + }); + return true; + } + + if (request.action === 'cloudLogin') { + cloudLoginAccount(request) + .then(function(result) { + sendResponse(result); + }) + .catch(function(err) { + sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); + }); + return true; + } + + if (request.action === 'cloudLogout') { + cloudLogoutAccount() + .then(function(result) { + sendResponse(result); + }) + .catch(function(err) { + sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); + }); + return true; + } + + if (request.action === 'cloudGetSyncStatus') { + cloudGetSyncStatus() + .then(function(result) { + sendResponse(result); + }) + .catch(function(err) { + sendResponse({ ok: false, authenticated: false, healthy: false, color: 'red', text: '云同步异常', error: err && err.message ? err.message : String(err) }); + }); + return true; + } + + if (request.action === 'cloudPullVaultItems') { + cloudPullVaultItems(request.itemTypes) + .then(function(items) { + sendResponse({ ok: true, items: items, status: buildCloudStatus(cloudAuthStateCache) }); + }) + .catch(function(err) { + sendResponse({ ok: false, items: [], error: err && err.message ? err.message : String(err) }); + }); + return true; + } + + if (request.action === 'cloudPushVaultItems') { + cloudPushVaultItems(request) + .then(function(status) { + sendResponse({ ok: true, status: status }); + }) + .catch(function(err) { + sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); + }); + return true; + } + + if (request.action === 'cloudBackfillLocalCache') { + backfillLocalCacheToCloud({ force: !!request.force }) + .then(function(result) { + sendResponse({ ok: true, result: result, status: buildCloudStatus(cloudAuthStateCache) }); + }) + .catch(function(err) { + sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); + }); + return true; + } + + if (request.action === 'cloudGetCacheStats') { + cloudGetCacheStats() + .then(function(result) { + sendResponse(result); + }) + .catch(function(err) { + sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); + }); + return true; + } + + if (request.action === 'localGetCacheStats') { + getLocalCacheStats() + .then(function(result) { + sendResponse(result); + }) + .catch(function(err) { + sendResponse({ ok: false, error: err && err.message ? err.message : String(err) }); + }); + return true; + } + if (request.action === 'cacheSaveCoverage') { saveCoverageSnapshot(request) .then(function(result) { @@ -994,6 +2980,36 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { return true; } + if (request.action === 'cacheGetPageThreadKeys') { + (async function() { + var forumKey = typeof request.forumKey === 'string' ? request.forumKey : ''; + var page = Math.max(1, Number(request.page) || 1); + if (!forumKey) { + return { ok: true, threadKeys: [], crawledAt: 0 }; + } + var db = await openCacheDb(); + try { + var tx = db.transaction('pageCoverages', 'readonly'); + var store = tx.objectStore('pageCoverages'); + var record = await requestToPromise(store.get(getPageCoverageKey(forumKey, page))); + await transactionDone(tx); + if (!record || !Array.isArray(record.threadKeys)) { + return { ok: true, threadKeys: [], crawledAt: 0 }; + } + return { + ok: true, + threadKeys: record.threadKeys, + crawledAt: Number(record.crawledAt || 0) + }; + } finally { + db.close(); + } + })() + .then(function(result) { sendResponse(result); }) + .catch(function(err) { sendResponse({ ok: false, threadKeys: [], error: err && err.message ? err.message : String(err) }); }); + return true; + } + if (request.action === 'cacheGetCoveragePlan') { getCachedCoveragePlan(request) .then(function(result) { @@ -1085,6 +3101,15 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { sendResponse({ ok: false, error: chrome.runtime.lastError.message }); return; } + cloudPushVaultItems({ + items: [{ + itemType: 'progress_state', + itemKey: 'latest', + data: normalizedState + }] + }).catch(function() { + return null; + }); sendResponse({ ok: true }); }); }); diff --git a/manifest.json b/manifest.json index cc52dfa..98e379e 100644 --- a/manifest.json +++ b/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 3, "name": "涩花塘磁力助手", "version": "1.3", - "description": "一键获取磁力链接 - 暗色科技风UI + 收藏夹 + 吜索历史 + 宔完成通知", + "description": "一键获取磁力链接 - 暗色科技风UI + 收藏夹 + 搜索历史 + 完成通知", "permissions": [ "activeTab", "clipboardWrite", @@ -18,7 +18,9 @@ "https://sehuatang.net/*", "https://www.sehuatang.net/*", "https://sehuatang.org/*", - "https://www.sehuatang.org/*" + "https://www.sehuatang.org/*", + "http://s.52oai.com/*", + "https://s.52oai.com/*" ], "content_scripts": [ {