1bf9c072c8
Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
3191 lines
101 KiB
JavaScript
3191 lines
101 KiB
JavaScript
var STORAGE_KEY_PREFIX = 'magnet-progress-state:';
|
|
var MAX_PROGRESS_ITEMS = 2000;
|
|
var SESSION_MARKER_KEY = 'magnet-progress-session-marker-v1';
|
|
var CACHE_DB_NAME = 'magnet-thread-cache';
|
|
var CACHE_DB_VERSION = 3;
|
|
var COVERAGE_LIMIT_PER_FORUM = 999;
|
|
var CLOUD_CACHE_ENABLED = true;
|
|
var CLOUD_CACHE_BASE_URL = 'https://s.52oai.com';
|
|
var CLOUD_CACHE_TIMEOUT = 6000;
|
|
var CLOUD_SHARED_CACHE_WRITE_TOKEN = 's52oai-shared-cache-write-v1';
|
|
var CLOUD_AUTH_STORAGE_KEY = 'magnet-cloud-auth-v1';
|
|
var CLOUD_DEVICE_ID_KEY = 'magnet-cloud-device-id-v1';
|
|
var CLOUD_CACHE_BACKFILL_KEY = 'magnet-cloud-cache-backfill-v1';
|
|
var CLOUD_UPLOAD_META_KEY = 'magnet-cloud-upload-meta-v1';
|
|
var CLOUD_VAULT_KEY_VERSION = 1;
|
|
var CLOUD_THREAD_UPLOAD_TTL_MS = 10 * 60 * 1000;
|
|
var CLOUD_PAGE_UPLOAD_TTL_MS = 30 * 60 * 1000;
|
|
var CLOUD_COVERAGE_UPLOAD_TTL_MS = 60 * 60 * 1000;
|
|
var CLOUD_UPLOAD_META_THREAD_LIMIT = 50000;
|
|
var CLOUD_UPLOAD_META_OTHER_LIMIT = 4000;
|
|
var cloudAuthStateCache = null;
|
|
var cloudUploadMetaCache = null;
|
|
var cloudBackfillPromise = null;
|
|
var isSessionReady = false;
|
|
var isPreparingSession = false;
|
|
var sessionReadyCallbacks = [];
|
|
|
|
function requestToPromise(request) {
|
|
return new Promise(function(resolve, reject) {
|
|
request.onsuccess = function(event) {
|
|
resolve(event.target.result);
|
|
};
|
|
request.onerror = function(event) {
|
|
reject(event.target.error || new Error('IndexedDB 请求失败'));
|
|
};
|
|
});
|
|
}
|
|
|
|
function transactionDone(transaction) {
|
|
return new Promise(function(resolve, reject) {
|
|
transaction.oncomplete = function() {
|
|
resolve();
|
|
};
|
|
transaction.onerror = function() {
|
|
reject(transaction.error || new Error('IndexedDB 事务失败'));
|
|
};
|
|
transaction.onabort = function() {
|
|
reject(transaction.error || new Error('IndexedDB 事务已中止'));
|
|
};
|
|
});
|
|
}
|
|
|
|
function normalizeCloudBaseUrl(url) {
|
|
return String(url || '').replace(/\/+$/, '');
|
|
}
|
|
|
|
function callCloudCache(path, payload, timeoutMs, extraHeaders) {
|
|
if (!CLOUD_CACHE_ENABLED) {
|
|
return Promise.resolve(null);
|
|
}
|
|
|
|
var controller = new AbortController();
|
|
var realTimeout = Math.max(2000, Math.min(Number(timeoutMs) || CLOUD_CACHE_TIMEOUT, 15000));
|
|
var timer = setTimeout(function() {
|
|
controller.abort();
|
|
}, realTimeout);
|
|
|
|
return fetch(normalizeCloudBaseUrl(CLOUD_CACHE_BASE_URL) + path, {
|
|
method: 'POST',
|
|
headers: Object.assign({
|
|
'Content-Type': 'application/json'
|
|
}, extraHeaders || {}),
|
|
body: JSON.stringify(payload || {}),
|
|
signal: controller.signal
|
|
}).then(function(response) {
|
|
if (!response.ok) {
|
|
throw new Error('云缓存 HTTP ' + response.status);
|
|
}
|
|
return response.json();
|
|
}).catch(function(error) {
|
|
if (error && error.name === 'AbortError') {
|
|
throw new Error('云缓存请求超时(' + realTimeout + 'ms)');
|
|
}
|
|
throw error;
|
|
}).finally(function() {
|
|
clearTimeout(timer);
|
|
});
|
|
}
|
|
|
|
function storageLocalGet(key) {
|
|
return new Promise(function(resolve, reject) {
|
|
chrome.storage.local.get(key, function(result) {
|
|
if (chrome.runtime.lastError) {
|
|
reject(new Error(chrome.runtime.lastError.message));
|
|
return;
|
|
}
|
|
resolve(result || {});
|
|
});
|
|
});
|
|
}
|
|
|
|
function storageLocalSet(data) {
|
|
return new Promise(function(resolve, reject) {
|
|
chrome.storage.local.set(data, function() {
|
|
if (chrome.runtime.lastError) {
|
|
reject(new Error(chrome.runtime.lastError.message));
|
|
return;
|
|
}
|
|
resolve();
|
|
});
|
|
});
|
|
}
|
|
|
|
function storageLocalRemove(key) {
|
|
return new Promise(function(resolve, reject) {
|
|
chrome.storage.local.remove(key, function() {
|
|
if (chrome.runtime.lastError) {
|
|
reject(new Error(chrome.runtime.lastError.message));
|
|
return;
|
|
}
|
|
resolve();
|
|
});
|
|
});
|
|
}
|
|
|
|
function textEncoder() {
|
|
return new TextEncoder();
|
|
}
|
|
|
|
function textDecoder() {
|
|
return new TextDecoder();
|
|
}
|
|
|
|
function bytesToBase64(bytes) {
|
|
var binary = '';
|
|
var chunkSize = 0x8000;
|
|
var index = 0;
|
|
for (index = 0; index < bytes.length; index += chunkSize) {
|
|
binary += String.fromCharCode.apply(null, bytes.subarray(index, index + chunkSize));
|
|
}
|
|
return btoa(binary);
|
|
}
|
|
|
|
function base64ToBytes(base64) {
|
|
var binary = atob(String(base64 || ''));
|
|
var bytes = new Uint8Array(binary.length);
|
|
var index = 0;
|
|
for (index = 0; index < binary.length; index++) {
|
|
bytes[index] = binary.charCodeAt(index);
|
|
}
|
|
return bytes;
|
|
}
|
|
|
|
function arrayBufferToBase64(buffer) {
|
|
return bytesToBase64(new Uint8Array(buffer));
|
|
}
|
|
|
|
function randomBase64(size) {
|
|
var bytes = new Uint8Array(size);
|
|
crypto.getRandomValues(bytes);
|
|
return bytesToBase64(bytes);
|
|
}
|
|
|
|
async function sha256Hex(value) {
|
|
var digest = await crypto.subtle.digest('SHA-256', textEncoder().encode(String(value || '')));
|
|
return Array.from(new Uint8Array(digest)).map(function(byte) {
|
|
return byte.toString(16).padStart(2, '0');
|
|
}).join('');
|
|
}
|
|
|
|
async function derivePasswordKey(password, saltBase64, keyUsages) {
|
|
var baseKey = await crypto.subtle.importKey(
|
|
'raw',
|
|
textEncoder().encode(String(password || '')),
|
|
'PBKDF2',
|
|
false,
|
|
['deriveKey']
|
|
);
|
|
return crypto.subtle.deriveKey({
|
|
name: 'PBKDF2',
|
|
salt: base64ToBytes(saltBase64),
|
|
iterations: 250000,
|
|
hash: 'SHA-256'
|
|
}, baseKey, {
|
|
name: 'AES-GCM',
|
|
length: 256
|
|
}, false, keyUsages || ['encrypt', 'decrypt']);
|
|
}
|
|
|
|
async function importVaultKey(vaultKeyBase64, keyUsages) {
|
|
return crypto.subtle.importKey(
|
|
'raw',
|
|
base64ToBytes(vaultKeyBase64),
|
|
{ name: 'AES-GCM', length: 256 },
|
|
false,
|
|
keyUsages || ['encrypt', 'decrypt']
|
|
);
|
|
}
|
|
|
|
function splitCipherAndTag(buffer) {
|
|
var bytes = new Uint8Array(buffer);
|
|
var tagLength = 16;
|
|
return {
|
|
ciphertext: bytesToBase64(bytes.slice(0, bytes.length - tagLength)),
|
|
tag: bytesToBase64(bytes.slice(bytes.length - tagLength))
|
|
};
|
|
}
|
|
|
|
function joinCipherAndTag(ciphertextBase64, tagBase64) {
|
|
var ciphertext = base64ToBytes(ciphertextBase64);
|
|
var tag = base64ToBytes(tagBase64);
|
|
var merged = new Uint8Array(ciphertext.length + tag.length);
|
|
merged.set(ciphertext, 0);
|
|
merged.set(tag, ciphertext.length);
|
|
return merged;
|
|
}
|
|
|
|
async function createVaultEnvelope(password) {
|
|
var saltBase64 = randomBase64(16);
|
|
var vaultKeyBase64 = randomBase64(32);
|
|
var ivBase64 = randomBase64(12);
|
|
var passwordKey = await derivePasswordKey(password, saltBase64, ['encrypt', 'decrypt']);
|
|
var encrypted = await crypto.subtle.encrypt(
|
|
{ name: 'AES-GCM', iv: base64ToBytes(ivBase64) },
|
|
passwordKey,
|
|
base64ToBytes(vaultKeyBase64)
|
|
);
|
|
return {
|
|
vaultKeyBase64: vaultKeyBase64,
|
|
wrappedDek: JSON.stringify({
|
|
ciphertext: arrayBufferToBase64(encrypted),
|
|
iv: ivBase64
|
|
}),
|
|
kdfSalt: saltBase64,
|
|
kdfParams: {
|
|
algorithm: 'PBKDF2',
|
|
hash: 'SHA-256',
|
|
iterations: 250000
|
|
},
|
|
keyVersion: CLOUD_VAULT_KEY_VERSION
|
|
};
|
|
}
|
|
|
|
async function unlockVaultKey(password, keyring) {
|
|
var envelope = keyring && keyring.wrappedDek ? JSON.parse(keyring.wrappedDek) : null;
|
|
if (!envelope || !envelope.ciphertext || !envelope.iv || !keyring.kdfSalt) {
|
|
throw new Error('保险柜密钥无效');
|
|
}
|
|
var passwordKey = await derivePasswordKey(password, keyring.kdfSalt, ['decrypt']);
|
|
var decrypted = await crypto.subtle.decrypt(
|
|
{ name: 'AES-GCM', iv: base64ToBytes(envelope.iv) },
|
|
passwordKey,
|
|
base64ToBytes(envelope.ciphertext)
|
|
);
|
|
return bytesToBase64(new Uint8Array(decrypted));
|
|
}
|
|
|
|
async function encryptVaultPayload(vaultKeyBase64, payload) {
|
|
var json = JSON.stringify(payload);
|
|
var ivBase64 = randomBase64(12);
|
|
var vaultKey = await importVaultKey(vaultKeyBase64, ['encrypt']);
|
|
var encrypted = await crypto.subtle.encrypt(
|
|
{ name: 'AES-GCM', iv: base64ToBytes(ivBase64) },
|
|
vaultKey,
|
|
textEncoder().encode(json)
|
|
);
|
|
var split = splitCipherAndTag(encrypted);
|
|
return {
|
|
payloadCiphertext: split.ciphertext,
|
|
payloadIv: ivBase64,
|
|
payloadTag: split.tag,
|
|
payloadHash: await sha256Hex(json),
|
|
keyVersion: CLOUD_VAULT_KEY_VERSION
|
|
};
|
|
}
|
|
|
|
async function decryptVaultPayload(vaultKeyBase64, item) {
|
|
var vaultKey = await importVaultKey(vaultKeyBase64, ['decrypt']);
|
|
var decrypted = await crypto.subtle.decrypt(
|
|
{ name: 'AES-GCM', iv: base64ToBytes(item.payloadIv) },
|
|
vaultKey,
|
|
joinCipherAndTag(item.payloadCiphertext, item.payloadTag)
|
|
);
|
|
return JSON.parse(textDecoder().decode(decrypted));
|
|
}
|
|
|
|
function normalizeCloudAuthState(state) {
|
|
state = state && typeof state === 'object' ? state : {};
|
|
return {
|
|
token: typeof state.token === 'string' ? state.token : '',
|
|
vaultKeyBase64: typeof state.vaultKeyBase64 === 'string' ? state.vaultKeyBase64 : '',
|
|
userId: Number(state.userId) || 0,
|
|
email: typeof state.email === 'string' ? state.email : '',
|
|
syncEnabled: !!state.syncEnabled,
|
|
healthy: !!state.healthy,
|
|
lastError: typeof state.lastError === 'string' ? state.lastError : '',
|
|
lastSyncAt: Number(state.lastSyncAt) || 0
|
|
};
|
|
}
|
|
|
|
async function loadCloudAuthState() {
|
|
if (cloudAuthStateCache) {
|
|
return cloudAuthStateCache;
|
|
}
|
|
var result = await storageLocalGet(CLOUD_AUTH_STORAGE_KEY);
|
|
cloudAuthStateCache = normalizeCloudAuthState(result[CLOUD_AUTH_STORAGE_KEY]);
|
|
return cloudAuthStateCache;
|
|
}
|
|
|
|
async function saveCloudAuthState(state) {
|
|
cloudAuthStateCache = normalizeCloudAuthState(state);
|
|
await storageLocalSet((function() {
|
|
var data = {};
|
|
data[CLOUD_AUTH_STORAGE_KEY] = cloudAuthStateCache;
|
|
return data;
|
|
})());
|
|
return cloudAuthStateCache;
|
|
}
|
|
|
|
async function clearCloudAuthState() {
|
|
cloudAuthStateCache = normalizeCloudAuthState(null);
|
|
await storageLocalRemove(CLOUD_AUTH_STORAGE_KEY);
|
|
return cloudAuthStateCache;
|
|
}
|
|
|
|
async function ensureCloudDeviceFingerprint() {
|
|
var result = await storageLocalGet(CLOUD_DEVICE_ID_KEY);
|
|
var existing = typeof result[CLOUD_DEVICE_ID_KEY] === 'string' ? result[CLOUD_DEVICE_ID_KEY] : '';
|
|
if (existing) {
|
|
return existing;
|
|
}
|
|
existing = 'device-' + randomBase64(18).replace(/[^a-zA-Z0-9_-]/g, '');
|
|
await storageLocalSet((function() {
|
|
var data = {};
|
|
data[CLOUD_DEVICE_ID_KEY] = existing;
|
|
return data;
|
|
})());
|
|
return existing;
|
|
}
|
|
|
|
function buildCloudStatus(state) {
|
|
var normalized = normalizeCloudAuthState(state);
|
|
var healthy = !!(normalized.token && normalized.syncEnabled && normalized.healthy);
|
|
return {
|
|
ok: true,
|
|
syncEnabled: !!normalized.syncEnabled,
|
|
authenticated: !!normalized.token,
|
|
healthy: healthy,
|
|
color: healthy ? 'green' : 'red',
|
|
email: normalized.email,
|
|
lastError: normalized.lastError,
|
|
text: healthy ? '云同步正常' : (normalized.token ? '云同步需确认' : '云同步未登录'),
|
|
accountText: healthy ? '账号状态正常' : (normalized.token ? '账号状态需确认' : '未登录')
|
|
};
|
|
}
|
|
|
|
function callCloudApi(path, options) {
|
|
var method = options && options.method ? options.method : 'GET';
|
|
var payload = options && Object.prototype.hasOwnProperty.call(options, 'payload') ? options.payload : null;
|
|
var token = options && options.token ? options.token : '';
|
|
var timeoutMs = options && options.timeoutMs ? options.timeoutMs : CLOUD_CACHE_TIMEOUT;
|
|
var controller = new AbortController();
|
|
var realTimeout = Math.max(2000, Math.min(Number(timeoutMs) || CLOUD_CACHE_TIMEOUT, 15000));
|
|
var timer = setTimeout(function() {
|
|
controller.abort();
|
|
}, realTimeout);
|
|
var headers = { 'Content-Type': 'application/json' };
|
|
if (token) {
|
|
headers.Authorization = 'Bearer ' + token;
|
|
}
|
|
return fetch(normalizeCloudBaseUrl(CLOUD_CACHE_BASE_URL) + path, {
|
|
method: method,
|
|
headers: headers,
|
|
body: payload !== null ? JSON.stringify(payload) : undefined,
|
|
signal: controller.signal
|
|
}).then(function(response) {
|
|
if (!response.ok) {
|
|
return response.text().then(function(text) {
|
|
var payload = null;
|
|
try {
|
|
payload = text ? JSON.parse(text) : null;
|
|
} catch (e) {
|
|
payload = null;
|
|
}
|
|
throw new Error(payload && payload.error ? payload.error : ('云接口 HTTP ' + response.status));
|
|
});
|
|
}
|
|
return response.json();
|
|
}).finally(function() {
|
|
clearTimeout(timer);
|
|
});
|
|
}
|
|
|
|
async function setCloudHealth(ok, errorMessage) {
|
|
var state = await loadCloudAuthState();
|
|
state.healthy = !!ok;
|
|
state.lastError = ok ? '' : String(errorMessage || '同步失败');
|
|
state.lastSyncAt = Date.now();
|
|
await saveCloudAuthState(state);
|
|
return buildCloudStatus(state);
|
|
}
|
|
|
|
async function cloudRegisterAccount(payload) {
|
|
var email = String(payload.email || '').trim().toLowerCase();
|
|
var password = String(payload.password || '');
|
|
var deviceFingerprint = await ensureCloudDeviceFingerprint();
|
|
var envelope = await createVaultEnvelope(password);
|
|
var response = await callCloudApi('/api/auth/register', {
|
|
method: 'POST',
|
|
payload: {
|
|
email: email,
|
|
password: password,
|
|
wrappedDek: envelope.wrappedDek,
|
|
kdfSalt: envelope.kdfSalt,
|
|
kdfParams: envelope.kdfParams,
|
|
keyVersion: envelope.keyVersion,
|
|
deviceName: 'Magnet Chrome Extension',
|
|
deviceFingerprint: deviceFingerprint
|
|
}
|
|
});
|
|
if (!response || !response.ok || !response.token) {
|
|
throw new Error(response && response.error ? response.error : '注册失败');
|
|
}
|
|
await saveCloudAuthState({
|
|
token: response.token,
|
|
vaultKeyBase64: envelope.vaultKeyBase64,
|
|
userId: response.user && response.user.id,
|
|
email: response.user && response.user.email ? response.user.email : email,
|
|
syncEnabled: true,
|
|
healthy: true,
|
|
lastError: '',
|
|
lastSyncAt: Date.now()
|
|
});
|
|
backfillLocalCacheToCloud({ force: true }).catch(function(error) {
|
|
console.warn('[MagnetPlugin][Cloud] register backfill failed:', error && error.message ? error.message : error);
|
|
});
|
|
return {
|
|
ok: true,
|
|
status: buildCloudStatus(await loadCloudAuthState()),
|
|
vaultItems: []
|
|
};
|
|
}
|
|
|
|
async function cloudPullVaultItems(itemTypes) {
|
|
var state = await loadCloudAuthState();
|
|
var response = null;
|
|
var items = [];
|
|
if (!state.token || !state.vaultKeyBase64) {
|
|
return [];
|
|
}
|
|
response = await callCloudApi('/api/vault/pull', {
|
|
method: 'POST',
|
|
token: state.token,
|
|
payload: {
|
|
itemTypes: Array.isArray(itemTypes) ? itemTypes : []
|
|
}
|
|
});
|
|
if (!response || !response.ok || !Array.isArray(response.items)) {
|
|
throw new Error(response && response.error ? response.error : '读取保险柜失败');
|
|
}
|
|
items = [];
|
|
for (var index = 0; index < response.items.length; index++) {
|
|
var item = response.items[index];
|
|
items.push({
|
|
itemType: item.itemType,
|
|
itemKey: item.itemKey,
|
|
data: await decryptVaultPayload(state.vaultKeyBase64, item),
|
|
updatedAt: item.updatedAt,
|
|
keyVersion: item.keyVersion
|
|
});
|
|
}
|
|
await setCloudHealth(true);
|
|
return items;
|
|
}
|
|
|
|
async function cloudLoginAccount(payload) {
|
|
var email = String(payload.email || '').trim().toLowerCase();
|
|
var password = String(payload.password || '');
|
|
var deviceFingerprint = await ensureCloudDeviceFingerprint();
|
|
var response = await callCloudApi('/api/auth/login', {
|
|
method: 'POST',
|
|
payload: {
|
|
email: email,
|
|
password: password,
|
|
deviceName: 'Magnet Chrome Extension',
|
|
deviceFingerprint: deviceFingerprint
|
|
}
|
|
});
|
|
var vaultKeyBase64 = null;
|
|
var vaultItems = [];
|
|
if (!response || !response.ok || !response.token || !response.keyring) {
|
|
throw new Error(response && response.error ? response.error : '登录失败');
|
|
}
|
|
vaultKeyBase64 = await unlockVaultKey(password, response.keyring);
|
|
await saveCloudAuthState({
|
|
token: response.token,
|
|
vaultKeyBase64: vaultKeyBase64,
|
|
userId: response.user && response.user.id,
|
|
email: response.user && response.user.email ? response.user.email : email,
|
|
syncEnabled: true,
|
|
healthy: true,
|
|
lastError: '',
|
|
lastSyncAt: Date.now()
|
|
});
|
|
backfillLocalCacheToCloud({ force: true }).catch(function(error) {
|
|
console.warn('[MagnetPlugin][Cloud] login backfill failed:', error && error.message ? error.message : error);
|
|
});
|
|
cloudPullVaultItems(['favorites', 'search_history', 'ui_settings', 'progress_state']).catch(function(error) {
|
|
console.warn('[MagnetPlugin][Cloud] login vault pull failed:', error && error.message ? error.message : error);
|
|
return [];
|
|
});
|
|
return {
|
|
ok: true,
|
|
status: buildCloudStatus(await loadCloudAuthState()),
|
|
vaultItems: vaultItems
|
|
};
|
|
}
|
|
|
|
async function cloudLogoutAccount() {
|
|
var state = await loadCloudAuthState();
|
|
if (state.token) {
|
|
try {
|
|
await callCloudApi('/api/auth/logout', {
|
|
method: 'POST',
|
|
token: state.token,
|
|
payload: {}
|
|
});
|
|
} catch (error) {
|
|
console.warn('[MagnetPlugin][Cloud] logout failed:', error && error.message ? error.message : error);
|
|
}
|
|
}
|
|
await clearCloudAuthState();
|
|
return {
|
|
ok: true,
|
|
status: buildCloudStatus(await loadCloudAuthState())
|
|
};
|
|
}
|
|
|
|
async function cloudGetSyncStatus() {
|
|
var state = await loadCloudAuthState();
|
|
if (!state.token) {
|
|
return buildCloudStatus(state);
|
|
}
|
|
try {
|
|
var response = await callCloudApi('/api/auth/me', {
|
|
method: 'GET',
|
|
token: state.token
|
|
});
|
|
if (response && response.ok) {
|
|
state.email = response.user && response.user.email ? response.user.email : state.email;
|
|
state.userId = response.user && response.user.id ? response.user.id : state.userId;
|
|
state.syncEnabled = true;
|
|
state.healthy = true;
|
|
state.lastError = '';
|
|
state.lastSyncAt = Date.now();
|
|
await saveCloudAuthState(state);
|
|
backfillLocalCacheToCloud({ force: false }).catch(function(error) {
|
|
console.warn('[MagnetPlugin][Cloud] background backfill failed:', error && error.message ? error.message : error);
|
|
});
|
|
return buildCloudStatus(state);
|
|
}
|
|
throw new Error(response && response.error ? response.error : '状态检查失败');
|
|
} catch (error) {
|
|
return {
|
|
ok: true,
|
|
syncEnabled: !!state.syncEnabled,
|
|
authenticated: !!state.token,
|
|
healthy: false,
|
|
color: 'red',
|
|
email: state.email,
|
|
lastError: error && error.message ? error.message : String(error),
|
|
text: state.token ? '云同步需确认' : '云同步未登录',
|
|
accountText: state.token ? '账号状态需确认' : '未登录'
|
|
};
|
|
}
|
|
}
|
|
|
|
async function cloudGetCacheStats() {
|
|
return callCloudApi('/api/shared-cache/stats', {
|
|
method: 'GET',
|
|
timeoutMs: 12000
|
|
});
|
|
}
|
|
|
|
async function getLocalCacheStats() {
|
|
var db = await openCacheDb();
|
|
var privateState = null;
|
|
var uploadMeta = null;
|
|
var threadCount = 0;
|
|
var magnetThreadCount = 0;
|
|
var coverageCount = 0;
|
|
var pageCount = 0;
|
|
var storageEstimate = null;
|
|
var allThreads = [];
|
|
try {
|
|
var tx = db.transaction(['threads', 'coverages', 'pageCoverages'], 'readonly');
|
|
threadCount = Number(await requestToPromise(tx.objectStore('threads').count()) || 0);
|
|
coverageCount = Number(await requestToPromise(tx.objectStore('coverages').count()) || 0);
|
|
pageCount = Number(await requestToPromise(tx.objectStore('pageCoverages').count()) || 0);
|
|
await transactionDone(tx);
|
|
allThreads = await readAllStoreRecords(db, 'threads');
|
|
magnetThreadCount = allThreads.filter(function(record) {
|
|
return Array.isArray(record.magnets) && normalizeMagnets(record.magnets).length > 0;
|
|
}).length;
|
|
} finally {
|
|
db.close();
|
|
}
|
|
|
|
privateState = await storageLocalGet('magnet-private-state-v1');
|
|
uploadMeta = await loadCloudUploadMetaState();
|
|
storageEstimate = await getStorageEstimate();
|
|
|
|
var privateData = privateState['magnet-private-state-v1'] && typeof privateState['magnet-private-state-v1'] === 'object'
|
|
? privateState['magnet-private-state-v1']
|
|
: {};
|
|
|
|
return {
|
|
ok: true,
|
|
counts: {
|
|
threads: threadCount,
|
|
magnetThreads: magnetThreadCount,
|
|
coverages: coverageCount,
|
|
pages: pageCount,
|
|
favorites: Array.isArray(privateData.favorites) ? privateData.favorites.length : 0,
|
|
history: Array.isArray(privateData.searchHistory) ? privateData.searchHistory.length : 0,
|
|
uploadMetaThreads: Object.keys(uploadMeta.threads || {}).length,
|
|
uploadMetaCoverages: Object.keys(uploadMeta.coverages || {}).length,
|
|
uploadMetaPages: Object.keys(uploadMeta.pages || {}).length
|
|
},
|
|
storage: {
|
|
usage: storageEstimate && Number(storageEstimate.usage) ? Number(storageEstimate.usage) : 0,
|
|
quota: storageEstimate && Number(storageEstimate.quota) ? Number(storageEstimate.quota) : 0
|
|
}
|
|
};
|
|
}
|
|
|
|
async function cloudPushVaultItems(payload) {
|
|
var state = await loadCloudAuthState();
|
|
var items = Array.isArray(payload.items) ? payload.items : [];
|
|
var preparedItems = [];
|
|
var index = 0;
|
|
var item = null;
|
|
if (!state.token || !state.vaultKeyBase64) {
|
|
return buildCloudStatus(state);
|
|
}
|
|
for (index = 0; index < items.length; index++) {
|
|
item = items[index];
|
|
if (!item || typeof item !== 'object') {
|
|
continue;
|
|
}
|
|
var itemType = String(item.itemType || '').trim();
|
|
var itemKey = String(item.itemKey || '').trim() || itemType;
|
|
if (!itemType) {
|
|
continue;
|
|
}
|
|
var encrypted = await encryptVaultPayload(state.vaultKeyBase64, item.data);
|
|
preparedItems.push({
|
|
itemType: itemType,
|
|
itemKey: itemKey,
|
|
payloadCiphertext: encrypted.payloadCiphertext,
|
|
payloadIv: encrypted.payloadIv,
|
|
payloadTag: encrypted.payloadTag,
|
|
payloadHash: encrypted.payloadHash,
|
|
keyVersion: encrypted.keyVersion
|
|
});
|
|
}
|
|
if (preparedItems.length === 0) {
|
|
return buildCloudStatus(state);
|
|
}
|
|
try {
|
|
await callCloudApi('/api/vault/push', {
|
|
method: 'POST',
|
|
token: state.token,
|
|
payload: { items: preparedItems }
|
|
});
|
|
return await setCloudHealth(true);
|
|
} catch (error) {
|
|
return await setCloudHealth(false, error && error.message ? error.message : error);
|
|
}
|
|
}
|
|
|
|
function openCacheDb() {
|
|
return new Promise(function(resolve, reject) {
|
|
var request = indexedDB.open(CACHE_DB_NAME, CACHE_DB_VERSION);
|
|
|
|
request.onupgradeneeded = function(event) {
|
|
var db = event.target.result;
|
|
|
|
if (!db.objectStoreNames.contains('threads')) {
|
|
var threadStore = db.createObjectStore('threads', { keyPath: 'cacheKey' });
|
|
threadStore.createIndex('forumKey', 'forumKey', { unique: false });
|
|
threadStore.createIndex('lastSeenAt', 'lastSeenAt', { unique: false });
|
|
threadStore.createIndex('forumKeyLastSeenAt', ['forumKey', 'lastSeenAt'], { unique: false });
|
|
} else {
|
|
var existingThreadStore = request.transaction.objectStore('threads');
|
|
if (!existingThreadStore.indexNames.contains('forumKeyLastSeenAt')) {
|
|
existingThreadStore.createIndex('forumKeyLastSeenAt', ['forumKey', 'lastSeenAt'], { unique: false });
|
|
}
|
|
}
|
|
|
|
if (!db.objectStoreNames.contains('coverages')) {
|
|
var coverageStore = db.createObjectStore('coverages', { keyPath: 'coverageKey' });
|
|
coverageStore.createIndex('forumKey', 'forumKey', { unique: false });
|
|
coverageStore.createIndex('crawledAt', 'crawledAt', { unique: false });
|
|
coverageStore.createIndex('forumKeyCrawledAt', ['forumKey', 'crawledAt'], { unique: false });
|
|
} else {
|
|
var existingCoverageStore = request.transaction.objectStore('coverages');
|
|
if (!existingCoverageStore.indexNames.contains('forumKeyCrawledAt')) {
|
|
existingCoverageStore.createIndex('forumKeyCrawledAt', ['forumKey', 'crawledAt'], { unique: false });
|
|
}
|
|
}
|
|
|
|
if (!db.objectStoreNames.contains('pageCoverages')) {
|
|
var pageCoverageStore = db.createObjectStore('pageCoverages', { keyPath: 'coverageKey' });
|
|
pageCoverageStore.createIndex('forumKey', 'forumKey', { unique: false });
|
|
pageCoverageStore.createIndex('crawledAt', 'crawledAt', { unique: false });
|
|
pageCoverageStore.createIndex('forumKeyCrawledAt', ['forumKey', 'crawledAt'], { unique: false });
|
|
} else {
|
|
var existingPageCoverageStore = request.transaction.objectStore('pageCoverages');
|
|
if (!existingPageCoverageStore.indexNames.contains('forumKeyCrawledAt')) {
|
|
existingPageCoverageStore.createIndex('forumKeyCrawledAt', ['forumKey', 'crawledAt'], { unique: false });
|
|
}
|
|
}
|
|
};
|
|
|
|
request.onsuccess = function(event) {
|
|
resolve(event.target.result);
|
|
};
|
|
|
|
request.onerror = function(event) {
|
|
reject(event.target.error || new Error('无法打开缓存数据库'));
|
|
};
|
|
});
|
|
}
|
|
|
|
function normalizeUrl(url) {
|
|
if (typeof url !== 'string' || !url) {
|
|
return '';
|
|
}
|
|
|
|
try {
|
|
var parsed = new URL(url);
|
|
parsed.hash = '';
|
|
return parsed.href;
|
|
} catch (e) {
|
|
return url;
|
|
}
|
|
}
|
|
|
|
function getThreadKeyFromUrl(url) {
|
|
var normalized = normalizeUrl(url);
|
|
if (!normalized) {
|
|
return '';
|
|
}
|
|
|
|
var match = normalized.match(/thread-(\d+)-/i) || normalized.match(/[?&]tid=(\d+)/i);
|
|
return match ? match[1] : normalized;
|
|
}
|
|
|
|
function normalizeTitle(title) {
|
|
return String(title || '').replace(/\s+/g, ' ').trim();
|
|
}
|
|
|
|
function normalizeMagnets(magnets) {
|
|
var seen = Object.create(null);
|
|
var result = [];
|
|
|
|
if (!Array.isArray(magnets)) {
|
|
return result;
|
|
}
|
|
|
|
magnets.forEach(function(magnet) {
|
|
if (typeof magnet !== 'string' || !magnet) {
|
|
return;
|
|
}
|
|
|
|
if (seen[magnet]) {
|
|
return;
|
|
}
|
|
|
|
seen[magnet] = true;
|
|
result.push(magnet);
|
|
});
|
|
|
|
return result;
|
|
}
|
|
|
|
function normalizeThreadEntry(thread, forumKey, crawledAt) {
|
|
if (!thread || typeof thread !== 'object') {
|
|
return null;
|
|
}
|
|
|
|
var normalizedUrl = normalizeUrl(thread.url);
|
|
var normalizedTitle = normalizeTitle(thread.title);
|
|
var threadKey = typeof thread.threadKey === 'string' && thread.threadKey
|
|
? thread.threadKey
|
|
: getThreadKeyFromUrl(normalizedUrl);
|
|
|
|
if (!forumKey || !normalizedUrl || !threadKey) {
|
|
return null;
|
|
}
|
|
|
|
return {
|
|
cacheKey: forumKey + '::' + threadKey,
|
|
forumKey: forumKey,
|
|
threadKey: threadKey,
|
|
url: normalizedUrl,
|
|
title: normalizedTitle,
|
|
titleNorm: normalizedTitle.toLowerCase(),
|
|
lastSeenAt: crawledAt,
|
|
firstSeenAt: crawledAt,
|
|
magnets: normalizeMagnets(thread.magnets),
|
|
magnetCount: Array.isArray(thread.magnets) ? normalizeMagnets(thread.magnets).length : 0,
|
|
lastMagnetSyncAt: Array.isArray(thread.magnets) && thread.magnets.length > 0 ? crawledAt : 0
|
|
};
|
|
}
|
|
|
|
function getCoverageKey(forumKey, startPage, endPage) {
|
|
return forumKey + '::' + startPage + '-' + endPage;
|
|
}
|
|
|
|
function getPageCoverageKey(forumKey, page) {
|
|
return forumKey + '::page::' + page;
|
|
}
|
|
|
|
|
|
function normalizeThreadsForCoverage(rawThreads, forumKey, crawledAt) {
|
|
var seen = Object.create(null);
|
|
var normalizedThreads = [];
|
|
|
|
(Array.isArray(rawThreads) ? rawThreads : []).forEach(function(thread) {
|
|
var normalized = normalizeThreadEntry(thread, forumKey, crawledAt);
|
|
if (!normalized || seen[normalized.cacheKey]) {
|
|
return;
|
|
}
|
|
|
|
seen[normalized.cacheKey] = true;
|
|
normalizedThreads.push({
|
|
threadKey: normalized.threadKey,
|
|
url: normalized.url,
|
|
title: normalized.title
|
|
});
|
|
});
|
|
|
|
return normalizedThreads;
|
|
}
|
|
|
|
function mergeCoverageThreadLists(forumKey, crawledAt, primaryThreads, secondaryThreads) {
|
|
var seen = Object.create(null);
|
|
var merged = [];
|
|
|
|
[primaryThreads, secondaryThreads].forEach(function(source) {
|
|
normalizeThreadsForCoverage(source, forumKey, crawledAt).forEach(function(thread) {
|
|
var key = String(thread.threadKey || '');
|
|
if (!key || seen[key]) {
|
|
return;
|
|
}
|
|
seen[key] = true;
|
|
merged.push({
|
|
threadKey: thread.threadKey,
|
|
url: thread.url,
|
|
title: thread.title
|
|
});
|
|
});
|
|
});
|
|
|
|
return merged;
|
|
}
|
|
|
|
function normalizeThreadsForCloudMagnets(rawThreads, forumKey, crawledAt) {
|
|
var seen = Object.create(null);
|
|
var normalizedThreads = [];
|
|
|
|
(Array.isArray(rawThreads) ? rawThreads : []).forEach(function(thread) {
|
|
var normalized = normalizeThreadEntry(thread, forumKey, crawledAt);
|
|
if (!normalized || seen[normalized.cacheKey]) {
|
|
return;
|
|
}
|
|
|
|
var magnets = normalizeMagnets(thread.magnets);
|
|
if (magnets.length === 0) {
|
|
return;
|
|
}
|
|
|
|
seen[normalized.cacheKey] = true;
|
|
normalizedThreads.push({
|
|
threadKey: normalized.threadKey,
|
|
url: normalized.url,
|
|
title: normalized.title,
|
|
magnets: magnets,
|
|
lastSeenAt: crawledAt
|
|
});
|
|
});
|
|
|
|
return normalizedThreads;
|
|
}
|
|
|
|
function normalizeThreadsForCloudIndex(rawThreads, forumKey, crawledAt) {
|
|
var seen = Object.create(null);
|
|
var normalizedThreads = [];
|
|
|
|
(Array.isArray(rawThreads) ? rawThreads : []).forEach(function(thread) {
|
|
var normalized = normalizeThreadEntry(thread, forumKey, crawledAt);
|
|
if (!normalized || seen[normalized.cacheKey]) {
|
|
return;
|
|
}
|
|
|
|
seen[normalized.cacheKey] = true;
|
|
normalizedThreads.push({
|
|
threadKey: normalized.threadKey,
|
|
url: normalized.url,
|
|
title: normalized.title,
|
|
magnets: normalizeMagnets(thread.magnets),
|
|
lastSeenAt: Number(thread.lastSeenAt) || crawledAt
|
|
});
|
|
});
|
|
|
|
return normalizedThreads;
|
|
}
|
|
|
|
function normalizeCloudCoveragePayload(coverage) {
|
|
if (!coverage || typeof coverage !== 'object') {
|
|
return null;
|
|
}
|
|
|
|
var forumKey = typeof coverage.forumKey === 'string' ? coverage.forumKey : '';
|
|
var startPage = Math.max(1, Number(coverage.startPage) || 1);
|
|
var endPage = Math.max(startPage, Number(coverage.endPage) || startPage);
|
|
if (!forumKey) {
|
|
return null;
|
|
}
|
|
|
|
return {
|
|
forumKey: forumKey,
|
|
startPage: startPage,
|
|
endPage: endPage,
|
|
crawledAt: Number(coverage.crawledAt) || Date.now(),
|
|
strategy: typeof coverage.strategy === 'string' && coverage.strategy ? coverage.strategy : 'full_live',
|
|
frontRefreshPages: Math.max(0, Number(coverage.frontRefreshPages) || 0),
|
|
threads: normalizeThreadsForCoverage(coverage.threads, forumKey, Number(coverage.crawledAt) || Date.now())
|
|
};
|
|
}
|
|
|
|
function normalizeCloudPageCoveragePayload(coverage) {
|
|
if (!coverage || typeof coverage !== 'object') {
|
|
return null;
|
|
}
|
|
|
|
var forumKey = typeof coverage.forumKey === 'string' ? coverage.forumKey : '';
|
|
var page = Math.max(1, Number(coverage.page) || 1);
|
|
if (!forumKey) {
|
|
return null;
|
|
}
|
|
|
|
return {
|
|
forumKey: forumKey,
|
|
page: page,
|
|
crawledAt: Number(coverage.crawledAt) || Date.now(),
|
|
threads: normalizeThreadsForCoverage(coverage.threads, forumKey, Number(coverage.crawledAt) || Date.now())
|
|
};
|
|
}
|
|
|
|
function normalizeCloudThreadMagnetsPayload(result) {
|
|
if (!result || typeof result !== 'object') {
|
|
return null;
|
|
}
|
|
|
|
var forumKey = typeof result.forumKey === 'string' ? result.forumKey : '';
|
|
var threadKey = typeof result.threadKey === 'string' ? result.threadKey : getThreadKeyFromUrl(result.url);
|
|
var url = normalizeUrl(result.url);
|
|
var magnets = normalizeMagnets(result.magnets);
|
|
if (!forumKey || !threadKey || !url || magnets.length === 0) {
|
|
return null;
|
|
}
|
|
|
|
return {
|
|
forumKey: forumKey,
|
|
threadKey: threadKey,
|
|
url: url,
|
|
title: normalizeTitle(result.title),
|
|
magnets: magnets,
|
|
lastSeenAt: Number(result.lastSeenAt) || Date.now()
|
|
};
|
|
}
|
|
|
|
async function cloudLookupThreadMagnets(payload) {
|
|
var response = await callCloudCache('/api/shared-cache/threads/lookup', {
|
|
threads: normalizeThreadsForCoverage(payload.threads, payload.forumKey, Date.now()).map(function(item) {
|
|
return {
|
|
forumKey: payload.forumKey,
|
|
threadKey: item.threadKey
|
|
};
|
|
})
|
|
});
|
|
|
|
if (!response || !response.ok || !Array.isArray(response.threads)) {
|
|
return [];
|
|
}
|
|
|
|
return response.threads.map(function(item) {
|
|
return normalizeCloudThreadMagnetsPayload({
|
|
forumKey: payload.forumKey,
|
|
threadKey: item.threadKey,
|
|
url: item.url,
|
|
title: item.title,
|
|
magnets: item.magnets,
|
|
lastSeenAt: item.lastSeenAt
|
|
});
|
|
}).filter(Boolean);
|
|
}
|
|
|
|
async function cloudUpsertThreadMagnets(payload) {
|
|
var allThreads = normalizeThreadsForCloudIndex(payload.threads, payload.forumKey, Number(payload.syncedAt) || Date.now()).map(function(item) {
|
|
return {
|
|
forumKey: payload.forumKey,
|
|
threadKey: item.threadKey,
|
|
url: item.url,
|
|
title: item.title,
|
|
magnets: item.magnets,
|
|
lastSeenAt: item.lastSeenAt
|
|
};
|
|
});
|
|
if (allThreads.length === 0) {
|
|
return { ok: true, savedCount: 0 };
|
|
}
|
|
return callCloudCache('/api/shared-cache/threads/upsert', {
|
|
threads: allThreads
|
|
}, CLOUD_CACHE_TIMEOUT, {
|
|
'x-shared-cache-write-token': CLOUD_SHARED_CACHE_WRITE_TOKEN
|
|
});
|
|
}
|
|
|
|
async function cloudLookupCoverageByStrategy(forumKey, startPage, endPage, strategy) {
|
|
var response = await callCloudCache('/api/shared-cache/coverages/lookup', {
|
|
forumKey: forumKey,
|
|
startPage: startPage,
|
|
endPage: endPage,
|
|
strategy: strategy
|
|
});
|
|
|
|
if (!response || !response.ok || !response.coverage) {
|
|
return null;
|
|
}
|
|
|
|
return normalizeCloudCoveragePayload(response.coverage);
|
|
}
|
|
|
|
async function cloudLookupCoverage(payload) {
|
|
var strategies = ['full_live', 'exact_cache', 'smart_incremental', 'assembled_pages'];
|
|
for (var index = 0; index < strategies.length; index++) {
|
|
var coverage = await cloudLookupCoverageByStrategy(payload.forumKey, payload.startPage, payload.endPage, strategies[index]);
|
|
if (coverage) {
|
|
return coverage;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function cloudLookupCoveragePlan(payload) {
|
|
var response = await callCloudCache('/api/shared-cache/coverages/plan', {
|
|
forumKey: payload.forumKey,
|
|
startPage: payload.startPage,
|
|
endPage: payload.endPage,
|
|
frontRefreshPages: payload.frontRefreshPages
|
|
});
|
|
if (!response || response.ok === false) {
|
|
return null;
|
|
}
|
|
return response;
|
|
}
|
|
|
|
async function cloudUpsertCoverage(payload) {
|
|
return callCloudCache('/api/shared-cache/coverages/upsert', {
|
|
forumKey: payload.forumKey,
|
|
startPage: payload.startPage,
|
|
endPage: payload.endPage,
|
|
strategy: payload.strategy,
|
|
crawledAt: payload.crawledAt,
|
|
threads: normalizeThreadsForCoverage(payload.threads, payload.forumKey, payload.crawledAt)
|
|
}, CLOUD_CACHE_TIMEOUT, {
|
|
'x-shared-cache-write-token': CLOUD_SHARED_CACHE_WRITE_TOKEN
|
|
});
|
|
}
|
|
|
|
async function cloudLookupPageCoverage(payload) {
|
|
var response = await callCloudCache('/api/shared-cache/pages/lookup', {
|
|
forumKey: payload.forumKey,
|
|
page: payload.page
|
|
});
|
|
|
|
if (!response || !response.ok || !response.coverage) {
|
|
return null;
|
|
}
|
|
|
|
return normalizeCloudPageCoveragePayload(response.coverage);
|
|
}
|
|
|
|
async function cloudUpsertPageCoverage(payload) {
|
|
return callCloudCache('/api/shared-cache/pages/upsert', {
|
|
forumKey: payload.forumKey,
|
|
page: payload.page,
|
|
crawledAt: payload.crawledAt,
|
|
threads: normalizeThreadsForCoverage(payload.threads, payload.forumKey, payload.crawledAt)
|
|
}, CLOUD_CACHE_TIMEOUT, {
|
|
'x-shared-cache-write-token': CLOUD_SHARED_CACHE_WRITE_TOKEN
|
|
});
|
|
}
|
|
|
|
async function getCloudAssembledPageCoverage(forumKey, startPage, endPage) {
|
|
var requests = [];
|
|
for (var page = startPage; page <= endPage; page++) {
|
|
requests.push(cloudLookupPageCoverage({ forumKey: forumKey, page: page }));
|
|
}
|
|
|
|
var results = await Promise.all(requests);
|
|
if (results.some(function(item) { return !item; })) {
|
|
return null;
|
|
}
|
|
|
|
var merged = Object.create(null);
|
|
var threads = [];
|
|
var latestCrawledAt = 0;
|
|
results.forEach(function(item) {
|
|
latestCrawledAt = Math.max(latestCrawledAt, Number(item.crawledAt || 0));
|
|
(Array.isArray(item.threads) ? item.threads : []).forEach(function(thread) {
|
|
var key = typeof thread.threadKey === 'string' && thread.threadKey ? thread.threadKey : getThreadKeyFromUrl(thread.url);
|
|
if (!key || merged[key]) {
|
|
return;
|
|
}
|
|
merged[key] = true;
|
|
threads.push({
|
|
threadKey: key,
|
|
url: normalizeUrl(thread.url),
|
|
title: normalizeTitle(thread.title)
|
|
});
|
|
});
|
|
});
|
|
|
|
return {
|
|
forumKey: forumKey,
|
|
startPage: startPage,
|
|
endPage: endPage,
|
|
crawledAt: latestCrawledAt || Date.now(),
|
|
strategy: 'assembled_pages',
|
|
frontRefreshPages: 0,
|
|
threads: threads
|
|
};
|
|
}
|
|
|
|
async function upsertThreadEntries(threadStore, forumKey, rawThreads, crawledAt) {
|
|
var seenThreadKeys = Object.create(null);
|
|
var threadKeys = [];
|
|
|
|
for (var threadIndex = 0; threadIndex < rawThreads.length; threadIndex++) {
|
|
var thread = rawThreads[threadIndex];
|
|
var normalized = normalizeThreadEntry(thread, forumKey, crawledAt);
|
|
if (!normalized || seenThreadKeys[normalized.cacheKey]) {
|
|
continue;
|
|
}
|
|
|
|
seenThreadKeys[normalized.cacheKey] = true;
|
|
threadKeys.push(normalized.cacheKey);
|
|
|
|
var existingRecord = await requestToPromise(threadStore.get(normalized.cacheKey));
|
|
var incomingMagnets = normalizeMagnets(normalized.magnets);
|
|
var mergedMagnets = incomingMagnets.length > 0
|
|
? incomingMagnets
|
|
: (existingRecord && Array.isArray(existingRecord.magnets) ? existingRecord.magnets : []);
|
|
var mergedLastMagnetSyncAt = incomingMagnets.length > 0
|
|
? (Number(normalized.lastMagnetSyncAt) || crawledAt)
|
|
: (existingRecord && Number(existingRecord.lastMagnetSyncAt) ? Number(existingRecord.lastMagnetSyncAt) : (normalized.lastMagnetSyncAt || 0));
|
|
|
|
threadStore.put({
|
|
cacheKey: normalized.cacheKey,
|
|
forumKey: normalized.forumKey,
|
|
threadKey: normalized.threadKey,
|
|
url: normalized.url,
|
|
title: normalized.title,
|
|
titleNorm: normalized.titleNorm,
|
|
firstSeenAt: existingRecord && Number(existingRecord.firstSeenAt)
|
|
? Number(existingRecord.firstSeenAt)
|
|
: normalized.firstSeenAt,
|
|
lastSeenAt: normalized.lastSeenAt,
|
|
magnets: normalizeMagnets(mergedMagnets),
|
|
magnetCount: normalizeMagnets(mergedMagnets).length,
|
|
lastMagnetSyncAt: mergedLastMagnetSyncAt
|
|
});
|
|
}
|
|
|
|
return threadKeys;
|
|
}
|
|
|
|
async function getAssembledPageCoverage(db, forumKey, startPage, endPage) {
|
|
var pageTx = db.transaction('pageCoverages', 'readonly');
|
|
var pageStore = pageTx.objectStore('pageCoverages');
|
|
var pageRequests = [];
|
|
|
|
for (var page = startPage; page <= endPage; page++) {
|
|
pageRequests.push(requestToPromise(pageStore.get(getPageCoverageKey(forumKey, page))));
|
|
}
|
|
|
|
var pageRecords = await Promise.all(pageRequests);
|
|
await transactionDone(pageTx);
|
|
|
|
if (pageRecords.some(function(record) { return !record; })) {
|
|
return null;
|
|
}
|
|
|
|
var merged = Object.create(null);
|
|
var mergedThreadKeys = [];
|
|
var latestCrawledAt = 0;
|
|
|
|
pageRecords.forEach(function(record) {
|
|
latestCrawledAt = Math.max(latestCrawledAt, Number(record.crawledAt || 0));
|
|
(Array.isArray(record.threadKeys) ? record.threadKeys : []).forEach(function(threadKey) {
|
|
if (merged[threadKey]) {
|
|
return;
|
|
}
|
|
merged[threadKey] = true;
|
|
mergedThreadKeys.push(threadKey);
|
|
});
|
|
});
|
|
|
|
return getCoverageWithThreads(db, {
|
|
forumKey: forumKey,
|
|
startPage: startPage,
|
|
endPage: endPage,
|
|
crawledAt: latestCrawledAt,
|
|
strategy: 'assembled_pages',
|
|
frontRefreshPages: 0,
|
|
threadKeys: mergedThreadKeys
|
|
});
|
|
}
|
|
|
|
async function getCachedPageCoverageBlocks(db, forumKey, startPage, endPage) {
|
|
var tx = db.transaction('pageCoverages', 'readonly');
|
|
var store = tx.objectStore('pageCoverages');
|
|
var index = store.index('forumKey');
|
|
var request = index.openCursor(IDBKeyRange.only(forumKey));
|
|
var pageRecords = [];
|
|
|
|
var records = await new Promise(function(resolve, reject) {
|
|
request.onsuccess = function(event) {
|
|
var cursor = event.target.result;
|
|
var record = null;
|
|
if (!cursor) {
|
|
resolve(pageRecords);
|
|
return;
|
|
}
|
|
record = cursor.value;
|
|
if (record && Number(record.page) >= startPage && Number(record.page) <= endPage) {
|
|
pageRecords.push(record);
|
|
}
|
|
cursor.continue();
|
|
};
|
|
request.onerror = function(event) {
|
|
reject(event.target.error || new Error('读取页缓存块失败'));
|
|
};
|
|
});
|
|
await transactionDone(tx);
|
|
|
|
if (!records.length) {
|
|
return [];
|
|
}
|
|
|
|
records.sort(function(a, b) {
|
|
return Number(a.page || 0) - Number(b.page || 0);
|
|
});
|
|
|
|
var blocks = [];
|
|
var currentBlock = null;
|
|
var latestCrawledAt = 0;
|
|
var mergedThreadKeys = null;
|
|
|
|
records.forEach(function(record) {
|
|
var page = Number(record.page || 0);
|
|
if (!currentBlock) {
|
|
currentBlock = {
|
|
forumKey: forumKey,
|
|
startPage: page,
|
|
endPage: page,
|
|
pages: [record],
|
|
crawledAt: Number(record.crawledAt || 0)
|
|
};
|
|
return;
|
|
}
|
|
|
|
if (page === currentBlock.endPage + 1) {
|
|
currentBlock.endPage = page;
|
|
currentBlock.pages.push(record);
|
|
currentBlock.crawledAt = Math.max(currentBlock.crawledAt, Number(record.crawledAt || 0));
|
|
return;
|
|
}
|
|
|
|
blocks.push(currentBlock);
|
|
currentBlock = {
|
|
forumKey: forumKey,
|
|
startPage: page,
|
|
endPage: page,
|
|
pages: [record],
|
|
crawledAt: Number(record.crawledAt || 0)
|
|
};
|
|
});
|
|
|
|
if (currentBlock) {
|
|
blocks.push(currentBlock);
|
|
}
|
|
|
|
return blocks.map(function(block) {
|
|
var merged = Object.create(null);
|
|
mergedThreadKeys = [];
|
|
latestCrawledAt = 0;
|
|
block.pages.forEach(function(record) {
|
|
latestCrawledAt = Math.max(latestCrawledAt, Number(record.crawledAt || 0));
|
|
(Array.isArray(record.threadKeys) ? record.threadKeys : []).forEach(function(threadKey) {
|
|
if (merged[threadKey]) {
|
|
return;
|
|
}
|
|
merged[threadKey] = true;
|
|
mergedThreadKeys.push(threadKey);
|
|
});
|
|
});
|
|
return {
|
|
forumKey: forumKey,
|
|
startPage: block.startPage,
|
|
endPage: block.endPage,
|
|
crawledAt: latestCrawledAt,
|
|
strategy: 'assembled_pages',
|
|
frontRefreshPages: 0,
|
|
threadKeys: mergedThreadKeys
|
|
};
|
|
});
|
|
}
|
|
|
|
async function getCoverageWithThreads(db, coverageRecord) {
|
|
if (!coverageRecord) {
|
|
return null;
|
|
}
|
|
|
|
var threadKeys = Array.isArray(coverageRecord.threadKeys) ? coverageRecord.threadKeys : [];
|
|
var threads = [];
|
|
|
|
if (threadKeys.length > 0) {
|
|
var threadTx = db.transaction('threads', 'readonly');
|
|
var threadStore = threadTx.objectStore('threads');
|
|
var threadRequests = threadKeys.map(function(threadKey) {
|
|
return requestToPromise(threadStore.get(threadKey));
|
|
});
|
|
var threadResults = await Promise.all(threadRequests);
|
|
await transactionDone(threadTx);
|
|
|
|
threads = threadResults.filter(function(record) {
|
|
return !!record;
|
|
}).map(function(record) {
|
|
return {
|
|
url: record.url,
|
|
title: record.title,
|
|
threadKey: record.threadKey
|
|
};
|
|
});
|
|
}
|
|
|
|
return {
|
|
forumKey: coverageRecord.forumKey,
|
|
startPage: coverageRecord.startPage,
|
|
endPage: coverageRecord.endPage,
|
|
crawledAt: coverageRecord.crawledAt,
|
|
strategy: coverageRecord.strategy || 'full_live',
|
|
frontRefreshPages: coverageRecord.frontRefreshPages || 0,
|
|
threads: threads
|
|
};
|
|
}
|
|
|
|
async function listForumCoverages(db, forumKey) {
|
|
return new Promise(function(resolve, reject) {
|
|
var tx = db.transaction('coverages', 'readonly');
|
|
var store = tx.objectStore('coverages');
|
|
var index = store.index('forumKey');
|
|
var request = index.openCursor(IDBKeyRange.only(forumKey));
|
|
var results = [];
|
|
|
|
request.onsuccess = function(event) {
|
|
var cursor = event.target.result;
|
|
if (!cursor) {
|
|
resolve(results);
|
|
return;
|
|
}
|
|
|
|
results.push(cursor.value);
|
|
cursor.continue();
|
|
};
|
|
|
|
request.onerror = function(event) {
|
|
reject(event.target.error || new Error('读取覆盖快照失败'));
|
|
};
|
|
});
|
|
}
|
|
|
|
async function getIntersectingCoverageBlocks(db, forumKey, startPage, endPage) {
|
|
var coverages = await listForumCoverages(db, forumKey);
|
|
var filtered = coverages.filter(function(record) {
|
|
var recordStart = Number(record.startPage || 0);
|
|
var recordEnd = Number(record.endPage || 0);
|
|
return recordStart <= endPage && recordEnd >= startPage;
|
|
}).sort(function(a, b) {
|
|
var aStart = Number(a.startPage || 0);
|
|
var bStart = Number(b.startPage || 0);
|
|
if (aStart !== bStart) {
|
|
return aStart - bStart;
|
|
}
|
|
return Number(b.endPage || 0) - Number(a.endPage || 0);
|
|
});
|
|
|
|
var result = [];
|
|
var seenRanges = Object.create(null);
|
|
var index = 0;
|
|
var record = null;
|
|
var hydrated = null;
|
|
var clippedStart = 0;
|
|
var clippedEnd = 0;
|
|
var rangeKey = '';
|
|
|
|
for (index = 0; index < filtered.length; index++) {
|
|
record = filtered[index];
|
|
hydrated = await getCoverageWithThreads(db, record);
|
|
if (!hydrated || !Array.isArray(hydrated.threads) || hydrated.threads.length === 0) {
|
|
continue;
|
|
}
|
|
clippedStart = Math.max(startPage, Number(hydrated.startPage || startPage));
|
|
clippedEnd = Math.min(endPage, Number(hydrated.endPage || endPage));
|
|
if (clippedStart > clippedEnd) {
|
|
continue;
|
|
}
|
|
rangeKey = String(clippedStart) + '-' + String(clippedEnd);
|
|
if (seenRanges[rangeKey]) {
|
|
continue;
|
|
}
|
|
seenRanges[rangeKey] = true;
|
|
result.push({
|
|
forumKey: hydrated.forumKey,
|
|
startPage: clippedStart,
|
|
endPage: clippedEnd,
|
|
crawledAt: hydrated.crawledAt,
|
|
strategy: hydrated.strategy || 'full_live',
|
|
frontRefreshPages: hydrated.frontRefreshPages || 0,
|
|
threads: hydrated.threads
|
|
});
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
async function pruneOldCoverages(db, forumKey) {
|
|
var coverages = await listForumCoverages(db, forumKey);
|
|
if (coverages.length <= COVERAGE_LIMIT_PER_FORUM) {
|
|
return;
|
|
}
|
|
|
|
coverages.sort(function(a, b) {
|
|
return Number(b.crawledAt || 0) - Number(a.crawledAt || 0);
|
|
});
|
|
|
|
var stale = coverages.slice(COVERAGE_LIMIT_PER_FORUM);
|
|
if (stale.length === 0) {
|
|
return;
|
|
}
|
|
|
|
var tx = db.transaction('coverages', 'readwrite');
|
|
var store = tx.objectStore('coverages');
|
|
stale.forEach(function(record) {
|
|
store.delete(record.coverageKey);
|
|
});
|
|
await transactionDone(tx);
|
|
}
|
|
|
|
async function saveCoverageSnapshot(payload) {
|
|
var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : '';
|
|
var startPage = Math.max(1, Number(payload.startPage) || 1);
|
|
var endPage = Math.max(startPage, Number(payload.endPage) || startPage);
|
|
var crawledAt = Number(payload.crawledAt) || Date.now();
|
|
var frontRefreshPages = Math.max(0, Number(payload.frontRefreshPages) || 0);
|
|
var strategy = typeof payload.strategy === 'string' ? payload.strategy : 'full_live';
|
|
var skipCloudSync = !!payload.skipCloudSync;
|
|
var rawThreads = Array.isArray(payload.threads) ? payload.threads : [];
|
|
var normalizedThreadsForCoverage = normalizeThreadsForCoverage(rawThreads, forumKey, crawledAt);
|
|
var intersectingBlocks = null;
|
|
var historicalThreads = [];
|
|
var mergedCoverageThreads = null;
|
|
var result = null;
|
|
|
|
var db = await openCacheDb();
|
|
try {
|
|
intersectingBlocks = await getIntersectingCoverageBlocks(db, forumKey, startPage, endPage);
|
|
historicalThreads = [];
|
|
intersectingBlocks.forEach(function(block) {
|
|
(Array.isArray(block.threads) ? block.threads : []).forEach(function(thread) {
|
|
historicalThreads.push(thread);
|
|
});
|
|
});
|
|
mergedCoverageThreads = mergeCoverageThreadLists(forumKey, crawledAt, rawThreads, historicalThreads);
|
|
normalizedThreadsForCoverage = normalizeThreadsForCoverage(mergedCoverageThreads, forumKey, crawledAt);
|
|
|
|
var tx = db.transaction(['threads', 'coverages'], 'readwrite');
|
|
var threadStore = tx.objectStore('threads');
|
|
var coverageStore = tx.objectStore('coverages');
|
|
var threadKeys = await upsertThreadEntries(threadStore, forumKey, mergedCoverageThreads, crawledAt);
|
|
|
|
coverageStore.put({
|
|
coverageKey: getCoverageKey(forumKey, startPage, endPage),
|
|
forumKey: forumKey,
|
|
startPage: startPage,
|
|
endPage: endPage,
|
|
crawledAt: crawledAt,
|
|
strategy: strategy,
|
|
frontRefreshPages: frontRefreshPages,
|
|
threadKeys: threadKeys,
|
|
threadCount: threadKeys.length
|
|
});
|
|
|
|
await transactionDone(tx);
|
|
await pruneOldCoverages(db, forumKey);
|
|
|
|
result = {
|
|
ok: true,
|
|
threadCount: threadKeys.length,
|
|
coverageKey: getCoverageKey(forumKey, startPage, endPage)
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
|
|
if (!skipCloudSync && forumKey && normalizedThreadsForCoverage.length > 0) {
|
|
try {
|
|
var coverageThreadUploadPlan = await filterThreadUploadsForCloud(
|
|
forumKey,
|
|
normalizeThreadsForCloudIndex(mergedCoverageThreads, forumKey, crawledAt),
|
|
{ ttlMs: CLOUD_THREAD_UPLOAD_TTL_MS }
|
|
);
|
|
if (coverageThreadUploadPlan.uploads.length > 0) {
|
|
var threadUpsertResult = await cloudUpsertThreadMagnets({
|
|
forumKey: forumKey,
|
|
syncedAt: crawledAt,
|
|
threads: coverageThreadUploadPlan.uploads
|
|
});
|
|
if (threadUpsertResult && threadUpsertResult.ok) {
|
|
await markUploadMetaEntries('thread', coverageThreadUploadPlan.pendingMeta);
|
|
}
|
|
}
|
|
|
|
var coverageUploadCheck = await shouldUploadCoverageToCloud({
|
|
forumKey: forumKey,
|
|
startPage: startPage,
|
|
endPage: endPage,
|
|
strategy: strategy,
|
|
crawledAt: crawledAt,
|
|
threads: normalizedThreadsForCoverage
|
|
});
|
|
if (coverageUploadCheck.shouldUpload) {
|
|
var coverageUpsertResult = await cloudUpsertCoverage({
|
|
forumKey: forumKey,
|
|
startPage: startPage,
|
|
endPage: endPage,
|
|
strategy: strategy,
|
|
crawledAt: crawledAt,
|
|
threads: normalizedThreadsForCoverage
|
|
});
|
|
if (coverageUpsertResult && coverageUpsertResult.ok) {
|
|
await markUploadMetaEntries('coverage', [coverageUploadCheck]);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.warn('[MagnetPlugin][Cloud] save coverage failed:', error && error.message ? error.message : error);
|
|
}
|
|
backfillLocalCacheToCloud({ force: false }).catch(function(backfillErr) {
|
|
console.warn('[MagnetPlugin][Cloud] post-coverage backfill failed:', backfillErr && backfillErr.message ? backfillErr.message : backfillErr);
|
|
});
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
async function savePageCoverageSnapshot(payload) {
|
|
var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : '';
|
|
var page = Math.max(1, Number(payload.page) || 1);
|
|
var crawledAt = Number(payload.crawledAt) || Date.now();
|
|
var skipCloudSync = !!payload.skipCloudSync;
|
|
var rawThreads = Array.isArray(payload.threads) ? payload.threads : [];
|
|
var normalizedThreadsForCoverage = normalizeThreadsForCoverage(rawThreads, forumKey, crawledAt);
|
|
var result = null;
|
|
|
|
var db = await openCacheDb();
|
|
try {
|
|
var tx = db.transaction(['threads', 'pageCoverages'], 'readwrite');
|
|
var threadStore = tx.objectStore('threads');
|
|
var pageCoverageStore = tx.objectStore('pageCoverages');
|
|
var threadKeys = await upsertThreadEntries(threadStore, forumKey, rawThreads, crawledAt);
|
|
|
|
pageCoverageStore.put({
|
|
coverageKey: getPageCoverageKey(forumKey, page),
|
|
forumKey: forumKey,
|
|
page: page,
|
|
crawledAt: crawledAt,
|
|
threadKeys: threadKeys,
|
|
threadCount: threadKeys.length
|
|
});
|
|
|
|
await transactionDone(tx);
|
|
|
|
result = {
|
|
ok: true,
|
|
page: page,
|
|
threadCount: threadKeys.length,
|
|
coverageKey: getPageCoverageKey(forumKey, page)
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
|
|
if (!skipCloudSync && forumKey && normalizedThreadsForCoverage.length > 0) {
|
|
try {
|
|
var pageThreadUploadPlan = await filterThreadUploadsForCloud(
|
|
forumKey,
|
|
normalizeThreadsForCloudIndex(rawThreads, forumKey, crawledAt),
|
|
{ ttlMs: CLOUD_THREAD_UPLOAD_TTL_MS }
|
|
);
|
|
if (pageThreadUploadPlan.uploads.length > 0) {
|
|
var pageThreadUpsertResult = await cloudUpsertThreadMagnets({
|
|
forumKey: forumKey,
|
|
syncedAt: crawledAt,
|
|
threads: pageThreadUploadPlan.uploads
|
|
});
|
|
if (pageThreadUpsertResult && pageThreadUpsertResult.ok) {
|
|
await markUploadMetaEntries('thread', pageThreadUploadPlan.pendingMeta);
|
|
}
|
|
}
|
|
|
|
var pageUploadCheck = await shouldUploadPageToCloud({
|
|
forumKey: forumKey,
|
|
page: page,
|
|
crawledAt: crawledAt,
|
|
threads: normalizedThreadsForCoverage
|
|
});
|
|
if (pageUploadCheck.shouldUpload) {
|
|
var pageUpsertResult = await cloudUpsertPageCoverage({
|
|
forumKey: forumKey,
|
|
page: page,
|
|
crawledAt: crawledAt,
|
|
threads: normalizedThreadsForCoverage
|
|
});
|
|
if (pageUpsertResult && pageUpsertResult.ok) {
|
|
await markUploadMetaEntries('page', [pageUploadCheck]);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.warn('[MagnetPlugin][Cloud] save page coverage failed:', error && error.message ? error.message : error);
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
async function getCachedThreadMagnets(payload) {
|
|
var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : '';
|
|
var rawThreads = Array.isArray(payload.threads) ? payload.threads : [];
|
|
var db = await openCacheDb();
|
|
var misses = [];
|
|
var resultMap = Object.create(null);
|
|
|
|
try {
|
|
var tx = db.transaction('threads', 'readonly');
|
|
var threadStore = tx.objectStore('threads');
|
|
var results = [];
|
|
|
|
for (var threadIndex = 0; threadIndex < rawThreads.length; threadIndex++) {
|
|
var thread = rawThreads[threadIndex];
|
|
var normalized = normalizeThreadEntry(thread, forumKey, Date.now());
|
|
if (!normalized) {
|
|
continue;
|
|
}
|
|
|
|
var record = await requestToPromise(threadStore.get(normalized.cacheKey));
|
|
if (!record) {
|
|
results.push({
|
|
threadKey: normalized.threadKey,
|
|
url: normalized.url,
|
|
title: normalized.title,
|
|
magnets: []
|
|
});
|
|
misses.push({
|
|
threadKey: normalized.threadKey,
|
|
url: normalized.url,
|
|
title: normalized.title
|
|
});
|
|
continue;
|
|
}
|
|
|
|
var recordMagnets = normalizeMagnets(record.magnets);
|
|
if (recordMagnets.length === 0 && !Number(record.lastMagnetSyncAt)) {
|
|
misses.push({
|
|
threadKey: record.threadKey,
|
|
url: record.url,
|
|
title: record.title
|
|
});
|
|
}
|
|
|
|
results.push({
|
|
threadKey: record.threadKey,
|
|
url: record.url,
|
|
title: record.title,
|
|
magnets: recordMagnets,
|
|
lastMagnetSyncAt: Number(record.lastMagnetSyncAt) || 0
|
|
});
|
|
}
|
|
|
|
await transactionDone(tx);
|
|
|
|
results.forEach(function(item) {
|
|
if (item && item.threadKey) {
|
|
resultMap[item.threadKey] = item;
|
|
}
|
|
});
|
|
|
|
if (misses.length > 0) {
|
|
try {
|
|
var cloudThreads = await cloudLookupThreadMagnets({
|
|
forumKey: forumKey,
|
|
threads: misses
|
|
});
|
|
if (cloudThreads.length > 0) {
|
|
await saveThreadMagnets({
|
|
forumKey: forumKey,
|
|
syncedAt: Date.now(),
|
|
threads: cloudThreads,
|
|
skipCloudSync: true
|
|
});
|
|
cloudThreads.forEach(function(item) {
|
|
if (!item || !item.threadKey) {
|
|
return;
|
|
}
|
|
resultMap[item.threadKey] = {
|
|
threadKey: item.threadKey,
|
|
url: item.url,
|
|
title: item.title,
|
|
magnets: normalizeMagnets(item.magnets),
|
|
lastMagnetSyncAt: Number(item.lastSeenAt) || Date.now()
|
|
};
|
|
});
|
|
}
|
|
} catch (error) {
|
|
console.warn('[MagnetPlugin][Cloud] lookup thread magnets failed:', error && error.message ? error.message : error);
|
|
}
|
|
}
|
|
|
|
return {
|
|
ok: true,
|
|
threads: results.map(function(item) {
|
|
return item && item.threadKey && resultMap[item.threadKey] ? resultMap[item.threadKey] : item;
|
|
})
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
async function saveThreadMagnets(payload) {
|
|
var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : '';
|
|
var rawThreads = Array.isArray(payload.threads) ? payload.threads : [];
|
|
var syncedAt = Number(payload.syncedAt) || Date.now();
|
|
var skipCloudSync = !!payload.skipCloudSync;
|
|
var cloudThreads = [];
|
|
var result = null;
|
|
var db = await openCacheDb();
|
|
|
|
try {
|
|
var tx = db.transaction('threads', 'readwrite');
|
|
var threadStore = tx.objectStore('threads');
|
|
var savedCount = 0;
|
|
|
|
for (var threadIndex = 0; threadIndex < rawThreads.length; threadIndex++) {
|
|
var thread = rawThreads[threadIndex];
|
|
var normalized = normalizeThreadEntry(thread, forumKey, syncedAt);
|
|
if (!normalized) {
|
|
continue;
|
|
}
|
|
|
|
var existingRecord = await requestToPromise(threadStore.get(normalized.cacheKey));
|
|
var normalizedMagnets = normalizeMagnets(thread.magnets);
|
|
var finalMagnets = normalizedMagnets.length > 0
|
|
? normalizedMagnets
|
|
: (existingRecord && Array.isArray(existingRecord.magnets) ? normalizeMagnets(existingRecord.magnets) : []);
|
|
|
|
threadStore.put({
|
|
cacheKey: normalized.cacheKey,
|
|
forumKey: normalized.forumKey,
|
|
threadKey: normalized.threadKey,
|
|
url: normalized.url,
|
|
title: normalized.title || (existingRecord ? existingRecord.title : ''),
|
|
titleNorm: normalizeTitle(normalized.title || (existingRecord ? existingRecord.title : '')).toLowerCase(),
|
|
firstSeenAt: existingRecord && Number(existingRecord.firstSeenAt)
|
|
? Number(existingRecord.firstSeenAt)
|
|
: normalized.firstSeenAt,
|
|
lastSeenAt: Math.max(Number(existingRecord && existingRecord.lastSeenAt || 0), Number(normalized.lastSeenAt || 0)) || Date.now(),
|
|
magnets: finalMagnets,
|
|
magnetCount: finalMagnets.length,
|
|
lastMagnetSyncAt: finalMagnets.length > 0 ? syncedAt : (existingRecord && Number(existingRecord.lastMagnetSyncAt) ? Number(existingRecord.lastMagnetSyncAt) : 0)
|
|
});
|
|
|
|
if (finalMagnets.length > 0) {
|
|
cloudThreads.push({
|
|
threadKey: normalized.threadKey,
|
|
url: normalized.url,
|
|
title: normalized.title || (existingRecord ? existingRecord.title : ''),
|
|
magnets: finalMagnets,
|
|
lastSeenAt: syncedAt
|
|
});
|
|
}
|
|
|
|
savedCount += 1;
|
|
}
|
|
|
|
await transactionDone(tx);
|
|
|
|
result = {
|
|
ok: true,
|
|
savedCount: savedCount
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
|
|
if (!skipCloudSync && forumKey && cloudThreads.length > 0) {
|
|
try {
|
|
var threadUploadPlan = await filterThreadUploadsForCloud(forumKey, cloudThreads, {
|
|
ttlMs: CLOUD_THREAD_UPLOAD_TTL_MS
|
|
});
|
|
if (threadUploadPlan.uploads.length > 0) {
|
|
var directThreadUpsertResult = await cloudUpsertThreadMagnets({
|
|
forumKey: forumKey,
|
|
syncedAt: syncedAt,
|
|
threads: threadUploadPlan.uploads
|
|
});
|
|
if (directThreadUpsertResult && directThreadUpsertResult.ok) {
|
|
await markUploadMetaEntries('thread', threadUploadPlan.pendingMeta);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.warn('[MagnetPlugin][Cloud] save thread magnets failed:', error && error.message ? error.message : error);
|
|
}
|
|
backfillLocalCacheToCloud({ force: false }).catch(function(backfillErr) {
|
|
console.warn('[MagnetPlugin][Cloud] post-thread-save backfill failed:', backfillErr && backfillErr.message ? backfillErr.message : backfillErr);
|
|
});
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
function estimateTextSize(value) {
|
|
try {
|
|
return JSON.stringify(value).length;
|
|
} catch (e) {
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
async function getStorageEstimate() {
|
|
if (!navigator.storage || !navigator.storage.estimate) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
return await navigator.storage.estimate();
|
|
} catch (e) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function readRecentThreadRecords(db, forumKey, limit) {
|
|
var tx = db.transaction('threads', 'readonly');
|
|
var store = tx.objectStore('threads');
|
|
var index = forumKey ? store.index('forumKeyLastSeenAt') : store.index('lastSeenAt');
|
|
var range = forumKey
|
|
? IDBKeyRange.bound([forumKey, 0], [forumKey, Number.MAX_SAFE_INTEGER])
|
|
: null;
|
|
var request = index.openCursor(range, 'prev');
|
|
|
|
return new Promise(function(resolve, reject) {
|
|
var items = [];
|
|
|
|
request.onsuccess = function(event) {
|
|
var cursor = event.target.result;
|
|
if (!cursor || items.length >= limit) {
|
|
resolve(items);
|
|
return;
|
|
}
|
|
|
|
items.push(cursor.value);
|
|
cursor.continue();
|
|
};
|
|
|
|
request.onerror = function(event) {
|
|
reject(event.target.error || new Error('读取最近帖子缓存失败'));
|
|
};
|
|
}).finally(function() {
|
|
return transactionDone(tx).catch(function() {
|
|
return null;
|
|
});
|
|
});
|
|
}
|
|
|
|
async function readRecentCoverageRecords(db, forumKey, limit) {
|
|
var tx = db.transaction('coverages', 'readonly');
|
|
var store = tx.objectStore('coverages');
|
|
var index = forumKey ? store.index('forumKeyCrawledAt') : store.index('crawledAt');
|
|
var range = forumKey
|
|
? IDBKeyRange.bound([forumKey, 0], [forumKey, Number.MAX_SAFE_INTEGER])
|
|
: null;
|
|
var request = index.openCursor(range, 'prev');
|
|
|
|
return new Promise(function(resolve, reject) {
|
|
var items = [];
|
|
|
|
request.onsuccess = function(event) {
|
|
var cursor = event.target.result;
|
|
if (!cursor || items.length >= limit) {
|
|
resolve(items);
|
|
return;
|
|
}
|
|
|
|
items.push(cursor.value);
|
|
cursor.continue();
|
|
};
|
|
|
|
request.onerror = function(event) {
|
|
reject(event.target.error || new Error('读取最近范围快照失败'));
|
|
};
|
|
}).finally(function() {
|
|
return transactionDone(tx).catch(function() {
|
|
return null;
|
|
});
|
|
});
|
|
}
|
|
|
|
async function readAllStoreRecords(db, storeName) {
|
|
return new Promise(function(resolve, reject) {
|
|
var tx = db.transaction(storeName, 'readonly');
|
|
var store = tx.objectStore(storeName);
|
|
var request = store.openCursor();
|
|
var items = [];
|
|
|
|
request.onsuccess = function(event) {
|
|
var cursor = event.target.result;
|
|
if (!cursor) {
|
|
resolve(items);
|
|
return;
|
|
}
|
|
items.push(cursor.value);
|
|
cursor.continue();
|
|
};
|
|
|
|
request.onerror = function(event) {
|
|
reject(event.target.error || new Error('读取 ' + storeName + ' 失败'));
|
|
};
|
|
});
|
|
}
|
|
|
|
async function getThreadsByKeys(db, threadKeys) {
|
|
var keys = Array.isArray(threadKeys) ? threadKeys.filter(Boolean) : [];
|
|
var threadTx = null;
|
|
var threadStore = null;
|
|
var threadRequests = null;
|
|
var threadResults = null;
|
|
if (keys.length === 0) {
|
|
return [];
|
|
}
|
|
threadTx = db.transaction('threads', 'readonly');
|
|
threadStore = threadTx.objectStore('threads');
|
|
threadRequests = keys.map(function(threadKey) {
|
|
return requestToPromise(threadStore.get(threadKey));
|
|
});
|
|
threadResults = await Promise.all(threadRequests);
|
|
await transactionDone(threadTx);
|
|
return threadResults.filter(Boolean).map(function(record) {
|
|
return {
|
|
threadKey: record.threadKey,
|
|
url: record.url,
|
|
title: record.title,
|
|
magnets: normalizeMagnets(record.magnets),
|
|
lastSeenAt: Number(record.lastSeenAt) || Date.now()
|
|
};
|
|
});
|
|
}
|
|
|
|
function chunkArray(items, size) {
|
|
var chunks = [];
|
|
var index = 0;
|
|
var safeSize = Math.max(1, Number(size) || 1);
|
|
for (index = 0; index < items.length; index += safeSize) {
|
|
chunks.push(items.slice(index, index + safeSize));
|
|
}
|
|
return chunks;
|
|
}
|
|
|
|
async function getCloudBackfillState() {
|
|
var result = await storageLocalGet(CLOUD_CACHE_BACKFILL_KEY);
|
|
return result[CLOUD_CACHE_BACKFILL_KEY] || null;
|
|
}
|
|
|
|
async function saveCloudBackfillState(state) {
|
|
var data = {};
|
|
data[CLOUD_CACHE_BACKFILL_KEY] = state;
|
|
await storageLocalSet(data);
|
|
}
|
|
|
|
function normalizeCloudUploadMetaState(state) {
|
|
state = state && typeof state === 'object' ? state : {};
|
|
return {
|
|
threads: state.threads && typeof state.threads === 'object' ? state.threads : {},
|
|
coverages: state.coverages && typeof state.coverages === 'object' ? state.coverages : {},
|
|
pages: state.pages && typeof state.pages === 'object' ? state.pages : {}
|
|
};
|
|
}
|
|
|
|
async function loadCloudUploadMetaState() {
|
|
var result = null;
|
|
if (cloudUploadMetaCache) {
|
|
return cloudUploadMetaCache;
|
|
}
|
|
result = await storageLocalGet(CLOUD_UPLOAD_META_KEY);
|
|
cloudUploadMetaCache = normalizeCloudUploadMetaState(result[CLOUD_UPLOAD_META_KEY]);
|
|
return cloudUploadMetaCache;
|
|
}
|
|
|
|
function pruneUploadMetaStore(store, limit) {
|
|
var keys = Object.keys(store || {});
|
|
var safeLimit = Math.max(100, Number(limit) || 1000);
|
|
var staleKeys = null;
|
|
if (keys.length <= safeLimit) {
|
|
return store;
|
|
}
|
|
staleKeys = keys.sort(function(a, b) {
|
|
return Number((store[a] && store[a].lastUploadedAt) || 0) - Number((store[b] && store[b].lastUploadedAt) || 0);
|
|
}).slice(0, keys.length - safeLimit);
|
|
staleKeys.forEach(function(key) {
|
|
delete store[key];
|
|
});
|
|
return store;
|
|
}
|
|
|
|
async function saveCloudUploadMetaState(state) {
|
|
var data = {};
|
|
var normalized = normalizeCloudUploadMetaState(state);
|
|
pruneUploadMetaStore(normalized.threads, CLOUD_UPLOAD_META_THREAD_LIMIT);
|
|
pruneUploadMetaStore(normalized.coverages, CLOUD_UPLOAD_META_OTHER_LIMIT);
|
|
pruneUploadMetaStore(normalized.pages, CLOUD_UPLOAD_META_OTHER_LIMIT);
|
|
cloudUploadMetaCache = normalized;
|
|
data[CLOUD_UPLOAD_META_KEY] = normalized;
|
|
await storageLocalSet(data);
|
|
return normalized;
|
|
}
|
|
|
|
function getUploadMetaBucket(state, type) {
|
|
if (type === 'thread') {
|
|
return state.threads;
|
|
}
|
|
if (type === 'coverage') {
|
|
return state.coverages;
|
|
}
|
|
return state.pages;
|
|
}
|
|
|
|
function buildThreadUploadMetaKey(forumKey, threadKey) {
|
|
return String(forumKey || '') + '::' + String(threadKey || '');
|
|
}
|
|
|
|
function buildCoverageUploadMetaKey(forumKey, startPage, endPage, strategy) {
|
|
return String(forumKey || '') + '::' + String(startPage || 1) + '-' + String(endPage || 1) + '::' + String(strategy || 'full_live');
|
|
}
|
|
|
|
function buildPageUploadMetaKey(forumKey, page) {
|
|
return String(forumKey || '') + '::page::' + String(page || 1);
|
|
}
|
|
|
|
function shouldSkipUploadMetaEntry(entry, payloadHash, ttlMs, now) {
|
|
return !!(
|
|
entry &&
|
|
entry.payloadHash === payloadHash &&
|
|
Number(entry.lastUploadedAt || 0) > 0 &&
|
|
now - Number(entry.lastUploadedAt || 0) < ttlMs
|
|
);
|
|
}
|
|
|
|
function sortThreadsForHash(threads) {
|
|
return (Array.isArray(threads) ? threads : []).map(function(thread) {
|
|
return {
|
|
threadKey: String(thread.threadKey || ''),
|
|
url: normalizeUrl(thread.url),
|
|
title: normalizeTitle(thread.title),
|
|
magnets: normalizeMagnets(thread.magnets).slice().sort()
|
|
};
|
|
}).sort(function(a, b) {
|
|
return String(a.threadKey).localeCompare(String(b.threadKey));
|
|
});
|
|
}
|
|
|
|
async function buildThreadUploadHash(forumKey, thread) {
|
|
return sha256Hex(JSON.stringify({
|
|
forumKey: String(forumKey || ''),
|
|
threadKey: String(thread.threadKey || ''),
|
|
url: normalizeUrl(thread.url),
|
|
title: normalizeTitle(thread.title),
|
|
magnets: normalizeMagnets(thread.magnets).slice().sort()
|
|
}));
|
|
}
|
|
|
|
async function buildCoverageUploadHash(payload) {
|
|
return sha256Hex(JSON.stringify({
|
|
forumKey: String(payload.forumKey || ''),
|
|
startPage: Number(payload.startPage || 1),
|
|
endPage: Number(payload.endPage || payload.startPage || 1),
|
|
strategy: String(payload.strategy || 'full_live'),
|
|
threads: sortThreadsForHash(payload.threads)
|
|
}));
|
|
}
|
|
|
|
async function buildPageUploadHash(payload) {
|
|
return sha256Hex(JSON.stringify({
|
|
forumKey: String(payload.forumKey || ''),
|
|
page: Number(payload.page || 1),
|
|
threads: sortThreadsForHash(payload.threads)
|
|
}));
|
|
}
|
|
|
|
async function filterThreadUploadsForCloud(forumKey, threads, options) {
|
|
var state = await loadCloudUploadMetaState();
|
|
var bucket = getUploadMetaBucket(state, 'thread');
|
|
var now = Date.now();
|
|
var ttlMs = options && Number(options.ttlMs) >= 0 ? Number(options.ttlMs) : CLOUD_THREAD_UPLOAD_TTL_MS;
|
|
var uploads = [];
|
|
var pendingMeta = [];
|
|
var index = 0;
|
|
var item = null;
|
|
var key = '';
|
|
var payloadHash = '';
|
|
|
|
for (index = 0; index < threads.length; index++) {
|
|
item = threads[index];
|
|
if (!item || !item.threadKey) {
|
|
continue;
|
|
}
|
|
key = buildThreadUploadMetaKey(forumKey, item.threadKey);
|
|
payloadHash = await buildThreadUploadHash(forumKey, item);
|
|
if (shouldSkipUploadMetaEntry(bucket[key], payloadHash, ttlMs, now)) {
|
|
continue;
|
|
}
|
|
uploads.push(item);
|
|
pendingMeta.push({ key: key, payloadHash: payloadHash, lastUploadedAt: now });
|
|
}
|
|
|
|
return { uploads: uploads, pendingMeta: pendingMeta };
|
|
}
|
|
|
|
async function shouldUploadCoverageToCloud(payload, options) {
|
|
var state = await loadCloudUploadMetaState();
|
|
var bucket = getUploadMetaBucket(state, 'coverage');
|
|
var now = Date.now();
|
|
var ttlMs = options && Number(options.ttlMs) >= 0 ? Number(options.ttlMs) : CLOUD_COVERAGE_UPLOAD_TTL_MS;
|
|
var key = buildCoverageUploadMetaKey(payload.forumKey, payload.startPage, payload.endPage, payload.strategy);
|
|
var payloadHash = await buildCoverageUploadHash(payload);
|
|
if (shouldSkipUploadMetaEntry(bucket[key], payloadHash, ttlMs, now)) {
|
|
return { shouldUpload: false, key: key, payloadHash: payloadHash, lastUploadedAt: now };
|
|
}
|
|
return { shouldUpload: true, key: key, payloadHash: payloadHash, lastUploadedAt: now };
|
|
}
|
|
|
|
async function shouldUploadPageToCloud(payload, options) {
|
|
var state = await loadCloudUploadMetaState();
|
|
var bucket = getUploadMetaBucket(state, 'page');
|
|
var now = Date.now();
|
|
var ttlMs = options && Number(options.ttlMs) >= 0 ? Number(options.ttlMs) : CLOUD_PAGE_UPLOAD_TTL_MS;
|
|
var key = buildPageUploadMetaKey(payload.forumKey, payload.page);
|
|
var payloadHash = await buildPageUploadHash(payload);
|
|
if (shouldSkipUploadMetaEntry(bucket[key], payloadHash, ttlMs, now)) {
|
|
return { shouldUpload: false, key: key, payloadHash: payloadHash, lastUploadedAt: now };
|
|
}
|
|
return { shouldUpload: true, key: key, payloadHash: payloadHash, lastUploadedAt: now };
|
|
}
|
|
|
|
async function markUploadMetaEntries(type, entries) {
|
|
var state = await loadCloudUploadMetaState();
|
|
var bucket = getUploadMetaBucket(state, type);
|
|
(Array.isArray(entries) ? entries : []).forEach(function(entry) {
|
|
if (!entry || !entry.key) {
|
|
return;
|
|
}
|
|
bucket[entry.key] = {
|
|
payloadHash: entry.payloadHash,
|
|
lastUploadedAt: Number(entry.lastUploadedAt) || Date.now()
|
|
};
|
|
});
|
|
await saveCloudUploadMetaState(state);
|
|
}
|
|
|
|
async function backfillLocalCacheToCloud(options) {
|
|
var settings = options && typeof options === 'object' ? options : {};
|
|
var force = !!settings.force;
|
|
var state = null;
|
|
var authState = null;
|
|
var db = null;
|
|
var threadRecords = [];
|
|
var coverageRecords = [];
|
|
var pageRecords = [];
|
|
var groupedThreads = Object.create(null);
|
|
var forumKeys = [];
|
|
var forumKeySeen = Object.create(null);
|
|
var coverageIndex = 0;
|
|
var pageIndex = 0;
|
|
var chunkIndex = 0;
|
|
var chunks = null;
|
|
var currentChunk = null;
|
|
var counts = {
|
|
threadCount: 0,
|
|
coverageCount: 0,
|
|
pageCount: 0
|
|
};
|
|
|
|
if (cloudBackfillPromise) {
|
|
return cloudBackfillPromise;
|
|
}
|
|
|
|
cloudBackfillPromise = (async function() {
|
|
authState = await loadCloudAuthState();
|
|
if (!authState.authenticated && !authState.token) {
|
|
return { ok: true, skipped: true, reason: 'not_authenticated' };
|
|
}
|
|
|
|
state = await getCloudBackfillState();
|
|
if (!force && state && Number(state.lastRunAt || 0) > Date.now() - 10 * 60 * 1000) {
|
|
return { ok: true, skipped: true, reason: 'recently_ran', state: state };
|
|
}
|
|
|
|
db = await openCacheDb();
|
|
try {
|
|
threadRecords = await readAllStoreRecords(db, 'threads');
|
|
coverageRecords = await readAllStoreRecords(db, 'coverages');
|
|
pageRecords = await readAllStoreRecords(db, 'pageCoverages');
|
|
|
|
threadRecords.forEach(function(record) {
|
|
if (!record || !record.forumKey || !record.threadKey) {
|
|
return;
|
|
}
|
|
var recordMagnets = normalizeMagnets(record.magnets);
|
|
if (!groupedThreads[record.forumKey]) {
|
|
groupedThreads[record.forumKey] = [];
|
|
}
|
|
groupedThreads[record.forumKey].push({
|
|
threadKey: record.threadKey,
|
|
url: record.url,
|
|
title: record.title,
|
|
magnets: recordMagnets,
|
|
lastSeenAt: Number(record.lastSeenAt) || Date.now()
|
|
});
|
|
if (!forumKeySeen[record.forumKey]) {
|
|
forumKeySeen[record.forumKey] = true;
|
|
forumKeys.push(record.forumKey);
|
|
}
|
|
});
|
|
|
|
forumKeys.forEach(function(fk) {
|
|
if (groupedThreads[fk]) {
|
|
groupedThreads[fk].sort(function(a, b) {
|
|
return (b.magnets.length > 0 ? 1 : 0) - (a.magnets.length > 0 ? 1 : 0);
|
|
});
|
|
}
|
|
});
|
|
|
|
for (chunkIndex = 0; chunkIndex < forumKeys.length; chunkIndex++) {
|
|
chunks = chunkArray(groupedThreads[forumKeys[chunkIndex]] || [], 200);
|
|
for (pageIndex = 0; pageIndex < chunks.length; pageIndex++) {
|
|
currentChunk = chunks[pageIndex];
|
|
var threadUploadPlan = null;
|
|
if (currentChunk.length === 0) {
|
|
continue;
|
|
}
|
|
threadUploadPlan = await filterThreadUploadsForCloud(forumKeys[chunkIndex], currentChunk, {
|
|
ttlMs: CLOUD_THREAD_UPLOAD_TTL_MS
|
|
});
|
|
if (threadUploadPlan.uploads.length === 0) {
|
|
continue;
|
|
}
|
|
var backfillThreadResult = await cloudUpsertThreadMagnets({
|
|
forumKey: forumKeys[chunkIndex],
|
|
syncedAt: Date.now(),
|
|
threads: threadUploadPlan.uploads
|
|
});
|
|
if (backfillThreadResult && backfillThreadResult.ok) {
|
|
await markUploadMetaEntries('thread', threadUploadPlan.pendingMeta);
|
|
counts.threadCount += threadUploadPlan.uploads.length;
|
|
}
|
|
}
|
|
}
|
|
|
|
for (coverageIndex = 0; coverageIndex < coverageRecords.length; coverageIndex++) {
|
|
var coverageRecord = coverageRecords[coverageIndex];
|
|
var coverageThreads = await getThreadsByKeys(db, coverageRecord.threadKeys);
|
|
var coverageUploadCheck = null;
|
|
if (!coverageRecord || !coverageRecord.forumKey || coverageThreads.length === 0) {
|
|
continue;
|
|
}
|
|
coverageUploadCheck = await shouldUploadCoverageToCloud({
|
|
forumKey: coverageRecord.forumKey,
|
|
startPage: Number(coverageRecord.startPage) || 1,
|
|
endPage: Number(coverageRecord.endPage) || Number(coverageRecord.startPage) || 1,
|
|
strategy: coverageRecord.strategy || 'full_live',
|
|
crawledAt: Number(coverageRecord.crawledAt) || Date.now(),
|
|
threads: coverageThreads
|
|
}, {
|
|
ttlMs: CLOUD_COVERAGE_UPLOAD_TTL_MS
|
|
});
|
|
if (!coverageUploadCheck.shouldUpload) {
|
|
continue;
|
|
}
|
|
var backfillCoverageResult = await cloudUpsertCoverage({
|
|
forumKey: coverageRecord.forumKey,
|
|
startPage: Number(coverageRecord.startPage) || 1,
|
|
endPage: Number(coverageRecord.endPage) || Number(coverageRecord.startPage) || 1,
|
|
strategy: coverageRecord.strategy || 'full_live',
|
|
crawledAt: Number(coverageRecord.crawledAt) || Date.now(),
|
|
threads: coverageThreads
|
|
});
|
|
if (backfillCoverageResult && backfillCoverageResult.ok) {
|
|
await markUploadMetaEntries('coverage', [coverageUploadCheck]);
|
|
counts.coverageCount += 1;
|
|
}
|
|
}
|
|
|
|
for (pageIndex = 0; pageIndex < pageRecords.length; pageIndex++) {
|
|
var pageRecord = pageRecords[pageIndex];
|
|
var pageThreads = await getThreadsByKeys(db, pageRecord.threadKeys);
|
|
var pageUploadCheck = null;
|
|
if (!pageRecord || !pageRecord.forumKey || pageThreads.length === 0) {
|
|
continue;
|
|
}
|
|
pageUploadCheck = await shouldUploadPageToCloud({
|
|
forumKey: pageRecord.forumKey,
|
|
page: Number(pageRecord.page) || 1,
|
|
crawledAt: Number(pageRecord.crawledAt) || Date.now(),
|
|
threads: pageThreads
|
|
}, {
|
|
ttlMs: CLOUD_PAGE_UPLOAD_TTL_MS
|
|
});
|
|
if (!pageUploadCheck.shouldUpload) {
|
|
continue;
|
|
}
|
|
var backfillPageResult = await cloudUpsertPageCoverage({
|
|
forumKey: pageRecord.forumKey,
|
|
page: Number(pageRecord.page) || 1,
|
|
crawledAt: Number(pageRecord.crawledAt) || Date.now(),
|
|
threads: pageThreads
|
|
});
|
|
if (backfillPageResult && backfillPageResult.ok) {
|
|
await markUploadMetaEntries('page', [pageUploadCheck]);
|
|
counts.pageCount += 1;
|
|
}
|
|
}
|
|
|
|
await saveCloudBackfillState({
|
|
lastRunAt: Date.now(),
|
|
counts: counts
|
|
});
|
|
await setCloudHealth(true);
|
|
return { ok: true, counts: counts };
|
|
} finally {
|
|
if (db) {
|
|
db.close();
|
|
}
|
|
}
|
|
})().catch(async function(error) {
|
|
console.warn('[MagnetPlugin][Cloud] backfill failed:', error && error.message ? error.message : error);
|
|
throw error;
|
|
}).finally(function() {
|
|
cloudBackfillPromise = null;
|
|
});
|
|
|
|
return cloudBackfillPromise;
|
|
}
|
|
|
|
async function getCacheOverview(payload) {
|
|
var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : '';
|
|
var limit = Math.max(1, Math.min(Number(payload.limit) || 12, 50));
|
|
var db = await openCacheDb();
|
|
|
|
try {
|
|
var threadTx = db.transaction('threads', 'readonly');
|
|
var coverageTx = db.transaction(['coverages', 'pageCoverages'], 'readonly');
|
|
var threadStore = threadTx.objectStore('threads');
|
|
var coverageStore = coverageTx.objectStore('coverages');
|
|
var pageCoverageStore = coverageTx.objectStore('pageCoverages');
|
|
var threadForumIndex = threadStore.index('forumKey');
|
|
var coverageForumIndex = coverageStore.index('forumKey');
|
|
var pageCoverageForumIndex = pageCoverageStore.index('forumKey');
|
|
|
|
var totals = await Promise.all([
|
|
requestToPromise(threadStore.count()),
|
|
requestToPromise(coverageStore.count()),
|
|
requestToPromise(pageCoverageStore.count()),
|
|
forumKey ? requestToPromise(threadForumIndex.count(IDBKeyRange.only(forumKey))) : Promise.resolve(0),
|
|
forumKey ? requestToPromise(coverageForumIndex.count(IDBKeyRange.only(forumKey))) : Promise.resolve(0),
|
|
forumKey ? requestToPromise(pageCoverageForumIndex.count(IDBKeyRange.only(forumKey))) : Promise.resolve(0)
|
|
]);
|
|
|
|
await Promise.all([transactionDone(threadTx), transactionDone(coverageTx)]);
|
|
|
|
var recentThreads = await readRecentThreadRecords(db, forumKey, limit);
|
|
var recentCoverages = await readRecentCoverageRecords(db, forumKey, Math.min(limit, 8));
|
|
var usedBytes = recentThreads.reduce(function(sum, item) {
|
|
return sum + estimateTextSize(item);
|
|
}, 0);
|
|
var storageEstimate = await getStorageEstimate();
|
|
var magnetCachedThreads = recentThreads.filter(function(item) {
|
|
return Number(item.magnetCount || 0) > 0;
|
|
}).length;
|
|
var magnetCount = recentThreads.reduce(function(sum, item) {
|
|
return sum + Number(item.magnetCount || 0);
|
|
}, 0);
|
|
|
|
return {
|
|
ok: true,
|
|
summary: {
|
|
forumKey: forumKey,
|
|
totalThreadCount: Number(totals[0] || 0),
|
|
totalCoverageCount: Number(totals[1] || 0),
|
|
totalPageCoverageCount: Number(totals[2] || 0),
|
|
forumThreadCount: Number(totals[3] || 0),
|
|
forumCoverageCount: Number(totals[4] || 0),
|
|
forumPageCoverageCount: Number(totals[5] || 0),
|
|
sampleBytes: usedBytes,
|
|
storageUsage: storageEstimate && Number(storageEstimate.usage) ? Number(storageEstimate.usage) : 0,
|
|
storageQuota: storageEstimate && Number(storageEstimate.quota) ? Number(storageEstimate.quota) : 0,
|
|
recentMagnetCachedThreads: magnetCachedThreads,
|
|
recentMagnetCount: magnetCount
|
|
},
|
|
recentThreads: recentThreads.map(function(item) {
|
|
return {
|
|
title: item.title,
|
|
url: item.url,
|
|
threadKey: item.threadKey,
|
|
magnetCount: Number(item.magnetCount || 0),
|
|
lastSeenAt: Number(item.lastSeenAt || 0),
|
|
lastMagnetSyncAt: Number(item.lastMagnetSyncAt || 0)
|
|
};
|
|
}),
|
|
recentCoverages: recentCoverages.map(function(item) {
|
|
return {
|
|
forumKey: item.forumKey,
|
|
startPage: Number(item.startPage || 0),
|
|
endPage: Number(item.endPage || 0),
|
|
threadCount: Number(item.threadCount || 0),
|
|
crawledAt: Number(item.crawledAt || 0),
|
|
strategy: item.strategy || 'full_live'
|
|
};
|
|
})
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
async function clearCacheData() {
|
|
var db = await openCacheDb();
|
|
|
|
try {
|
|
var tx = db.transaction(['threads', 'coverages', 'pageCoverages'], 'readwrite');
|
|
tx.objectStore('threads').clear();
|
|
tx.objectStore('coverages').clear();
|
|
tx.objectStore('pageCoverages').clear();
|
|
await transactionDone(tx);
|
|
|
|
return { ok: true };
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
async function getCachedCoveragePlan(payload) {
|
|
var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : '';
|
|
var startPage = Math.max(1, Number(payload.startPage) || 1);
|
|
var endPage = Math.max(startPage, Number(payload.endPage) || startPage);
|
|
var frontRefreshPages = Math.max(0, Number(payload.frontRefreshPages) || 0);
|
|
var cloudCoverage = null;
|
|
var cloudAssembledCoverage = null;
|
|
var cachedBlocks = [];
|
|
|
|
var db = await openCacheDb();
|
|
try {
|
|
var exactKey = getCoverageKey(forumKey, startPage, endPage);
|
|
var serverPlan = null;
|
|
var exactTx = db.transaction('coverages', 'readonly');
|
|
var exactStore = exactTx.objectStore('coverages');
|
|
var exactRecord = await requestToPromise(exactStore.get(exactKey));
|
|
await transactionDone(exactTx);
|
|
|
|
var exactCoverage = exactRecord ? await getCoverageWithThreads(db, exactRecord) : null;
|
|
if (!exactCoverage) {
|
|
exactCoverage = await getAssembledPageCoverage(db, forumKey, startPage, endPage);
|
|
}
|
|
if (!exactCoverage) {
|
|
try {
|
|
cloudCoverage = await cloudLookupCoverage({
|
|
forumKey: forumKey,
|
|
startPage: startPage,
|
|
endPage: endPage
|
|
});
|
|
if (cloudCoverage && Array.isArray(cloudCoverage.threads) && cloudCoverage.threads.length > 0) {
|
|
await saveCoverageSnapshot({
|
|
forumKey: cloudCoverage.forumKey,
|
|
startPage: cloudCoverage.startPage,
|
|
endPage: cloudCoverage.endPage,
|
|
crawledAt: cloudCoverage.crawledAt,
|
|
strategy: cloudCoverage.strategy,
|
|
frontRefreshPages: cloudCoverage.frontRefreshPages,
|
|
threads: cloudCoverage.threads,
|
|
skipCloudSync: true
|
|
});
|
|
exactCoverage = cloudCoverage;
|
|
}
|
|
} catch (error) {
|
|
console.warn('[MagnetPlugin][Cloud] lookup exact coverage failed:', error && error.message ? error.message : error);
|
|
}
|
|
}
|
|
if (!exactCoverage) {
|
|
try {
|
|
cloudAssembledCoverage = await getCloudAssembledPageCoverage(forumKey, startPage, endPage);
|
|
if (cloudAssembledCoverage && Array.isArray(cloudAssembledCoverage.threads) && cloudAssembledCoverage.threads.length > 0) {
|
|
await saveCoverageSnapshot({
|
|
forumKey: cloudAssembledCoverage.forumKey,
|
|
startPage: cloudAssembledCoverage.startPage,
|
|
endPage: cloudAssembledCoverage.endPage,
|
|
crawledAt: cloudAssembledCoverage.crawledAt,
|
|
strategy: cloudAssembledCoverage.strategy,
|
|
frontRefreshPages: cloudAssembledCoverage.frontRefreshPages,
|
|
threads: cloudAssembledCoverage.threads,
|
|
skipCloudSync: true
|
|
});
|
|
exactCoverage = cloudAssembledCoverage;
|
|
}
|
|
} catch (error) {
|
|
console.warn('[MagnetPlugin][Cloud] assemble page coverage failed:', error && error.message ? error.message : error);
|
|
}
|
|
}
|
|
if (!exactCoverage) {
|
|
try {
|
|
serverPlan = await cloudLookupCoveragePlan({
|
|
forumKey: forumKey,
|
|
startPage: startPage,
|
|
endPage: endPage,
|
|
frontRefreshPages: frontRefreshPages
|
|
});
|
|
if (serverPlan && serverPlan.exactCoverage && Array.isArray(serverPlan.exactCoverage.threads) && serverPlan.exactCoverage.threads.length > 0) {
|
|
await saveCoverageSnapshot({
|
|
forumKey: serverPlan.exactCoverage.forumKey,
|
|
startPage: serverPlan.exactCoverage.startPage,
|
|
endPage: serverPlan.exactCoverage.endPage,
|
|
crawledAt: serverPlan.exactCoverage.crawledAt,
|
|
strategy: serverPlan.exactCoverage.strategy,
|
|
frontRefreshPages: serverPlan.exactCoverage.frontRefreshPages,
|
|
threads: serverPlan.exactCoverage.threads,
|
|
skipCloudSync: true
|
|
});
|
|
exactCoverage = serverPlan.exactCoverage;
|
|
}
|
|
} catch (error) {
|
|
console.warn('[MagnetPlugin][Cloud] coverage plan failed:', error && error.message ? error.message : error);
|
|
}
|
|
}
|
|
|
|
if (exactCoverage) {
|
|
return {
|
|
ok: true,
|
|
exactCoverage: exactCoverage,
|
|
shiftedCoverage: null,
|
|
cachedBlocks: []
|
|
};
|
|
}
|
|
|
|
try {
|
|
var localBlocks = await getCachedPageCoverageBlocks(db, forumKey, startPage, endPage);
|
|
var intersectingCoverages = await getIntersectingCoverageBlocks(db, forumKey, startPage, endPage);
|
|
for (var blockIndex = 0; blockIndex < localBlocks.length; blockIndex++) {
|
|
var hydratedBlock = await getCoverageWithThreads(db, localBlocks[blockIndex]);
|
|
if (hydratedBlock && Array.isArray(hydratedBlock.threads) && hydratedBlock.threads.length > 0) {
|
|
cachedBlocks.push(hydratedBlock);
|
|
}
|
|
}
|
|
intersectingCoverages.forEach(function(block) {
|
|
cachedBlocks.push(block);
|
|
});
|
|
if (serverPlan && Array.isArray(serverPlan.cachedBlocks)) {
|
|
serverPlan.cachedBlocks.forEach(function(block) {
|
|
if (block && Array.isArray(block.threads) && block.threads.length > 0) {
|
|
cachedBlocks.push(block);
|
|
}
|
|
});
|
|
}
|
|
cachedBlocks.sort(function(a, b) {
|
|
var startDiff = Number(a.startPage || 0) - Number(b.startPage || 0);
|
|
if (startDiff !== 0) {
|
|
return startDiff;
|
|
}
|
|
return Number(b.endPage || 0) - Number(a.endPage || 0);
|
|
});
|
|
} catch (error) {
|
|
console.warn('[MagnetPlugin][Cache] build local cached blocks failed:', error && error.message ? error.message : error);
|
|
}
|
|
|
|
var shiftedCoverage = serverPlan && serverPlan.shiftedCoverage ? serverPlan.shiftedCoverage : null;
|
|
if (!shiftedCoverage && startPage === 1 && frontRefreshPages > 0) {
|
|
var coverages = await listForumCoverages(db, forumKey);
|
|
coverages = coverages.filter(function(record) {
|
|
return Number(record.startPage) === 1 && Number(record.endPage) >= 1;
|
|
}).sort(function(a, b) {
|
|
var crawlDiff = Number(b.crawledAt || 0) - Number(a.crawledAt || 0);
|
|
if (crawlDiff !== 0) {
|
|
return crawlDiff;
|
|
}
|
|
return Number(b.endPage || 0) - Number(a.endPage || 0);
|
|
});
|
|
|
|
if (coverages.length > 0) {
|
|
var bestCoverage = coverages[0];
|
|
var hydratedCoverage = await getCoverageWithThreads(db, bestCoverage);
|
|
if (hydratedCoverage && Array.isArray(hydratedCoverage.threads) && hydratedCoverage.threads.length > 0) {
|
|
shiftedCoverage = {
|
|
forumKey: hydratedCoverage.forumKey,
|
|
sourceStartPage: hydratedCoverage.startPage,
|
|
sourceEndPage: hydratedCoverage.endPage,
|
|
crawledAt: hydratedCoverage.crawledAt,
|
|
strategy: hydratedCoverage.strategy,
|
|
threads: hydratedCoverage.threads,
|
|
reusedStartPage: Math.min(endPage, startPage + frontRefreshPages),
|
|
reusedEndPage: Math.min(endPage, bestCoverage.endPage + frontRefreshPages)
|
|
};
|
|
|
|
if (shiftedCoverage.reusedStartPage > shiftedCoverage.reusedEndPage) {
|
|
shiftedCoverage = null;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
ok: true,
|
|
exactCoverage: null,
|
|
shiftedCoverage: shiftedCoverage,
|
|
cachedBlocks: cachedBlocks
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
function isProgressStateKey(key) {
|
|
return typeof key === 'string' && key.indexOf(STORAGE_KEY_PREFIX) === 0;
|
|
}
|
|
|
|
function flushSessionReadyCallbacks() {
|
|
var callbacks = sessionReadyCallbacks.slice();
|
|
sessionReadyCallbacks = [];
|
|
callbacks.forEach(function(callback) {
|
|
try {
|
|
callback();
|
|
} catch (e) {
|
|
// ignore callback errors
|
|
}
|
|
});
|
|
}
|
|
|
|
function finishSessionPrepare() {
|
|
isSessionReady = true;
|
|
isPreparingSession = false;
|
|
flushSessionReadyCallbacks();
|
|
}
|
|
|
|
function clearAllProgressStates(done) {
|
|
chrome.storage.local.get(null, function(result) {
|
|
if (chrome.runtime.lastError) {
|
|
done();
|
|
return;
|
|
}
|
|
|
|
var keys = Object.keys(result || {}).filter(isProgressStateKey);
|
|
if (keys.length === 0) {
|
|
done();
|
|
return;
|
|
}
|
|
|
|
chrome.storage.local.remove(keys, function() {
|
|
done();
|
|
});
|
|
});
|
|
}
|
|
|
|
function ensureSessionReady(callback) {
|
|
if (isSessionReady) {
|
|
callback();
|
|
return;
|
|
}
|
|
|
|
sessionReadyCallbacks.push(callback);
|
|
if (isPreparingSession) {
|
|
return;
|
|
}
|
|
|
|
isPreparingSession = true;
|
|
if (!chrome.storage || !chrome.storage.session) {
|
|
finishSessionPrepare();
|
|
return;
|
|
}
|
|
|
|
chrome.storage.session.get(SESSION_MARKER_KEY, function(result) {
|
|
if (chrome.runtime.lastError) {
|
|
finishSessionPrepare();
|
|
return;
|
|
}
|
|
|
|
if (result && result[SESSION_MARKER_KEY]) {
|
|
finishSessionPrepare();
|
|
return;
|
|
}
|
|
|
|
clearAllProgressStates(function() {
|
|
var marker = {};
|
|
marker[SESSION_MARKER_KEY] = Date.now();
|
|
chrome.storage.session.set(marker, function() {
|
|
finishSessionPrepare();
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
function clampTimeout(timeoutMs) {
|
|
var parsed = Number(timeoutMs);
|
|
if (!Number.isFinite(parsed)) return 15000;
|
|
return Math.max(3000, Math.min(parsed, 30000));
|
|
}
|
|
|
|
function getStorageKeyBySender(sender) {
|
|
if (!sender || !sender.tab || typeof sender.tab.id !== 'number') {
|
|
return null;
|
|
}
|
|
return STORAGE_KEY_PREFIX + sender.tab.id;
|
|
}
|
|
|
|
function normalizeProgressState(state) {
|
|
if (!state || typeof state !== 'object') {
|
|
return null;
|
|
}
|
|
|
|
var speed = typeof state.speedMode === 'string' ? state.speedMode : 'fast';
|
|
if (speed !== 'slow' && speed !== 'medium' && speed !== 'fast' && speed !== 'ultrafast') {
|
|
speed = 'fast';
|
|
}
|
|
|
|
var startPage = Number(state.startPage);
|
|
var endPage = Number(state.endPage);
|
|
if (!Number.isFinite(startPage) || startPage < 1) startPage = 1;
|
|
if (!Number.isFinite(endPage) || endPage < 1) endPage = startPage;
|
|
|
|
var resumeFromPage = Number(state.resumeFromPage);
|
|
if (!Number.isFinite(resumeFromPage) || resumeFromPage < 1) {
|
|
resumeFromPage = startPage;
|
|
}
|
|
|
|
var isRunning = !!state.isRunning;
|
|
var stoppedByUser = !!state.stoppedByUser;
|
|
|
|
var linkSet = new Set();
|
|
var links = [];
|
|
if (Array.isArray(state.links)) {
|
|
state.links.forEach(function(link) {
|
|
if (typeof link !== 'string' || !link) return;
|
|
if (linkSet.has(link)) return;
|
|
linkSet.add(link);
|
|
links.push(link);
|
|
});
|
|
}
|
|
|
|
var items = [];
|
|
if (Array.isArray(state.items)) {
|
|
state.items.forEach(function(item) {
|
|
if (!item || typeof item !== 'object') return;
|
|
var link = typeof item.link === 'string' ? item.link : '';
|
|
if (!link) return;
|
|
var title = typeof item.title === 'string' ? item.title : '恢复记录';
|
|
if (!linkSet.has(link)) {
|
|
linkSet.add(link);
|
|
links.push(link);
|
|
}
|
|
items.push({
|
|
title: title.slice(0, 200),
|
|
link: link
|
|
});
|
|
});
|
|
}
|
|
|
|
if (links.length > MAX_PROGRESS_ITEMS) {
|
|
links = links.slice(links.length - MAX_PROGRESS_ITEMS);
|
|
}
|
|
if (items.length > MAX_PROGRESS_ITEMS) {
|
|
items = items.slice(items.length - MAX_PROGRESS_ITEMS);
|
|
}
|
|
|
|
return {
|
|
links: links,
|
|
items: items,
|
|
keyword: typeof state.keyword === 'string' ? state.keyword.slice(0, 200) : '',
|
|
speedMode: speed,
|
|
startPage: startPage,
|
|
endPage: endPage,
|
|
resumeFromPage: resumeFromPage,
|
|
isRunning: isRunning,
|
|
stoppedByUser: stoppedByUser,
|
|
statusText: typeof state.statusText === 'string' ? state.statusText.slice(0, 300) : '',
|
|
statusType: typeof state.statusType === 'string' ? state.statusType.slice(0, 30) : '',
|
|
pageUrl: typeof state.pageUrl === 'string' ? state.pageUrl.slice(0, 1000) : '',
|
|
updatedAt: Date.now()
|
|
};
|
|
}
|
|
|
|
function fetchPageHtml(url, timeoutMs) {
|
|
var controller = new AbortController();
|
|
var realTimeout = clampTimeout(timeoutMs);
|
|
var timer = setTimeout(function() {
|
|
controller.abort();
|
|
}, realTimeout);
|
|
|
|
return fetch(url, { signal: controller.signal })
|
|
.then(function(response) {
|
|
if (!response.ok) {
|
|
throw new Error('HTTP ' + response.status + ' ' + response.statusText);
|
|
}
|
|
return response.text();
|
|
})
|
|
.catch(function(err) {
|
|
if (err && err.name === 'AbortError') {
|
|
throw new Error('请求超时(' + realTimeout + 'ms)');
|
|
}
|
|
throw err;
|
|
})
|
|
.finally(function() {
|
|
clearTimeout(timer);
|
|
});
|
|
}
|
|
|
|
function extractMagnetsFromHtml(html) {
|
|
const magnetPattern = /magnet:\?xt=urn:btih:[a-fA-F0-9]{32,40}/gi;
|
|
const magnets = html.match(magnetPattern) || [];
|
|
return [...new Set(magnets)];
|
|
}
|
|
|
|
chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
|
|
if (request.action === 'openCloudSyncPage') {
|
|
chrome.tabs.create({ url: chrome.runtime.getURL('popup.html?view=cloud') }, function(tab) {
|
|
if (chrome.runtime.lastError) {
|
|
sendResponse({ ok: false, error: chrome.runtime.lastError.message });
|
|
return;
|
|
}
|
|
sendResponse({ ok: true, tabId: tab && typeof tab.id === 'number' ? tab.id : null });
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cloudRegister') {
|
|
cloudRegisterAccount(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cloudLogin') {
|
|
cloudLoginAccount(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cloudLogout') {
|
|
cloudLogoutAccount()
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cloudGetSyncStatus') {
|
|
cloudGetSyncStatus()
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, authenticated: false, healthy: false, color: 'red', text: '云同步异常', error: err && err.message ? err.message : String(err) });
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cloudPullVaultItems') {
|
|
cloudPullVaultItems(request.itemTypes)
|
|
.then(function(items) {
|
|
sendResponse({ ok: true, items: items, status: buildCloudStatus(cloudAuthStateCache) });
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, items: [], error: err && err.message ? err.message : String(err) });
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cloudPushVaultItems') {
|
|
cloudPushVaultItems(request)
|
|
.then(function(status) {
|
|
sendResponse({ ok: true, status: status });
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cloudBackfillLocalCache') {
|
|
backfillLocalCacheToCloud({ force: !!request.force })
|
|
.then(function(result) {
|
|
sendResponse({ ok: true, result: result, status: buildCloudStatus(cloudAuthStateCache) });
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cloudGetCacheStats') {
|
|
cloudGetCacheStats()
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'localGetCacheStats') {
|
|
getLocalCacheStats()
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheSaveCoverage') {
|
|
saveCoverageSnapshot(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheGetPageThreadKeys') {
|
|
(async function() {
|
|
var forumKey = typeof request.forumKey === 'string' ? request.forumKey : '';
|
|
var page = Math.max(1, Number(request.page) || 1);
|
|
if (!forumKey) {
|
|
return { ok: true, threadKeys: [], crawledAt: 0 };
|
|
}
|
|
var db = await openCacheDb();
|
|
try {
|
|
var tx = db.transaction('pageCoverages', 'readonly');
|
|
var store = tx.objectStore('pageCoverages');
|
|
var record = await requestToPromise(store.get(getPageCoverageKey(forumKey, page)));
|
|
await transactionDone(tx);
|
|
if (!record || !Array.isArray(record.threadKeys)) {
|
|
return { ok: true, threadKeys: [], crawledAt: 0 };
|
|
}
|
|
return {
|
|
ok: true,
|
|
threadKeys: record.threadKeys,
|
|
crawledAt: Number(record.crawledAt || 0)
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
})()
|
|
.then(function(result) { sendResponse(result); })
|
|
.catch(function(err) { sendResponse({ ok: false, threadKeys: [], error: err && err.message ? err.message : String(err) }); });
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheGetCoveragePlan') {
|
|
getCachedCoveragePlan(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheGetThreadMagnets') {
|
|
getCachedThreadMagnets(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheSaveThreadMagnets') {
|
|
saveThreadMagnets(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheSavePageCoverage') {
|
|
savePageCoverageSnapshot(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheGetOverview') {
|
|
getCacheOverview(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheClearAll') {
|
|
clearCacheData()
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'saveProgressState') {
|
|
ensureSessionReady(function() {
|
|
const saveKey = getStorageKeyBySender(sender);
|
|
if (!saveKey) {
|
|
sendResponse({ ok: false, error: '无法识别标签页' });
|
|
return;
|
|
}
|
|
|
|
const normalizedState = normalizeProgressState(request.state);
|
|
if (!normalizedState) {
|
|
sendResponse({ ok: false, error: '状态数据无效' });
|
|
return;
|
|
}
|
|
|
|
chrome.storage.local.set({ [saveKey]: normalizedState }, function() {
|
|
if (chrome.runtime.lastError) {
|
|
sendResponse({ ok: false, error: chrome.runtime.lastError.message });
|
|
return;
|
|
}
|
|
cloudPushVaultItems({
|
|
items: [{
|
|
itemType: 'progress_state',
|
|
itemKey: 'latest',
|
|
data: normalizedState
|
|
}]
|
|
}).catch(function() {
|
|
return null;
|
|
});
|
|
sendResponse({ ok: true });
|
|
});
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'loadProgressState') {
|
|
ensureSessionReady(function() {
|
|
const loadKey = getStorageKeyBySender(sender);
|
|
if (!loadKey) {
|
|
sendResponse({ ok: false, error: '无法识别标签页', state: null });
|
|
return;
|
|
}
|
|
|
|
chrome.storage.local.get(loadKey, function(result) {
|
|
if (chrome.runtime.lastError) {
|
|
sendResponse({ ok: false, error: chrome.runtime.lastError.message, state: null });
|
|
return;
|
|
}
|
|
sendResponse({ ok: true, state: result && result[loadKey] ? result[loadKey] : null });
|
|
});
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'clearProgressState') {
|
|
ensureSessionReady(function() {
|
|
const clearKey = getStorageKeyBySender(sender);
|
|
if (!clearKey) {
|
|
sendResponse({ ok: false, error: '无法识别标签页' });
|
|
return;
|
|
}
|
|
|
|
chrome.storage.local.remove(clearKey, function() {
|
|
if (chrome.runtime.lastError) {
|
|
sendResponse({ ok: false, error: chrome.runtime.lastError.message });
|
|
return;
|
|
}
|
|
sendResponse({ ok: true });
|
|
});
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'fetchHtml') {
|
|
fetchPageHtml(request.url, request.timeoutMs)
|
|
.then(html => {
|
|
sendResponse({ html: html });
|
|
})
|
|
.catch(err => {
|
|
sendResponse({ html: '', error: err.message });
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'openAndFetch') {
|
|
fetchPageHtml(request.url, request.timeoutMs)
|
|
.then(html => {
|
|
sendResponse({ magnets: extractMagnetsFromHtml(html) });
|
|
})
|
|
.catch(err => {
|
|
sendResponse({ magnets: [], error: err.message });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
});
|
|
|
|
if (chrome.tabs && chrome.tabs.onRemoved) {
|
|
chrome.tabs.onRemoved.addListener(function(tabId) {
|
|
var key = STORAGE_KEY_PREFIX + tabId;
|
|
chrome.storage.local.remove(key, function() {
|
|
// ignore cleanup errors
|
|
});
|
|
});
|
|
}
|