1166 lines
35 KiB
JavaScript
1166 lines
35 KiB
JavaScript
var STORAGE_KEY_PREFIX = 'magnet-progress-state:';
|
|
var MAX_PROGRESS_ITEMS = 2000;
|
|
var SESSION_MARKER_KEY = 'magnet-progress-session-marker-v1';
|
|
var CACHE_DB_NAME = 'magnet-thread-cache';
|
|
var CACHE_DB_VERSION = 3;
|
|
var COVERAGE_LIMIT_PER_FORUM = 12;
|
|
var isSessionReady = false;
|
|
var isPreparingSession = false;
|
|
var sessionReadyCallbacks = [];
|
|
|
|
function requestToPromise(request) {
|
|
return new Promise(function(resolve, reject) {
|
|
request.onsuccess = function(event) {
|
|
resolve(event.target.result);
|
|
};
|
|
request.onerror = function(event) {
|
|
reject(event.target.error || new Error('IndexedDB 请求失败'));
|
|
};
|
|
});
|
|
}
|
|
|
|
function transactionDone(transaction) {
|
|
return new Promise(function(resolve, reject) {
|
|
transaction.oncomplete = function() {
|
|
resolve();
|
|
};
|
|
transaction.onerror = function() {
|
|
reject(transaction.error || new Error('IndexedDB 事务失败'));
|
|
};
|
|
transaction.onabort = function() {
|
|
reject(transaction.error || new Error('IndexedDB 事务已中止'));
|
|
};
|
|
});
|
|
}
|
|
|
|
function openCacheDb() {
|
|
return new Promise(function(resolve, reject) {
|
|
var request = indexedDB.open(CACHE_DB_NAME, CACHE_DB_VERSION);
|
|
|
|
request.onupgradeneeded = function(event) {
|
|
var db = event.target.result;
|
|
|
|
if (!db.objectStoreNames.contains('threads')) {
|
|
var threadStore = db.createObjectStore('threads', { keyPath: 'cacheKey' });
|
|
threadStore.createIndex('forumKey', 'forumKey', { unique: false });
|
|
threadStore.createIndex('lastSeenAt', 'lastSeenAt', { unique: false });
|
|
threadStore.createIndex('forumKeyLastSeenAt', ['forumKey', 'lastSeenAt'], { unique: false });
|
|
} else {
|
|
var existingThreadStore = request.transaction.objectStore('threads');
|
|
if (!existingThreadStore.indexNames.contains('forumKeyLastSeenAt')) {
|
|
existingThreadStore.createIndex('forumKeyLastSeenAt', ['forumKey', 'lastSeenAt'], { unique: false });
|
|
}
|
|
}
|
|
|
|
if (!db.objectStoreNames.contains('coverages')) {
|
|
var coverageStore = db.createObjectStore('coverages', { keyPath: 'coverageKey' });
|
|
coverageStore.createIndex('forumKey', 'forumKey', { unique: false });
|
|
coverageStore.createIndex('crawledAt', 'crawledAt', { unique: false });
|
|
coverageStore.createIndex('forumKeyCrawledAt', ['forumKey', 'crawledAt'], { unique: false });
|
|
} else {
|
|
var existingCoverageStore = request.transaction.objectStore('coverages');
|
|
if (!existingCoverageStore.indexNames.contains('forumKeyCrawledAt')) {
|
|
existingCoverageStore.createIndex('forumKeyCrawledAt', ['forumKey', 'crawledAt'], { unique: false });
|
|
}
|
|
}
|
|
|
|
if (!db.objectStoreNames.contains('pageCoverages')) {
|
|
var pageCoverageStore = db.createObjectStore('pageCoverages', { keyPath: 'coverageKey' });
|
|
pageCoverageStore.createIndex('forumKey', 'forumKey', { unique: false });
|
|
pageCoverageStore.createIndex('crawledAt', 'crawledAt', { unique: false });
|
|
pageCoverageStore.createIndex('forumKeyCrawledAt', ['forumKey', 'crawledAt'], { unique: false });
|
|
} else {
|
|
var existingPageCoverageStore = request.transaction.objectStore('pageCoverages');
|
|
if (!existingPageCoverageStore.indexNames.contains('forumKeyCrawledAt')) {
|
|
existingPageCoverageStore.createIndex('forumKeyCrawledAt', ['forumKey', 'crawledAt'], { unique: false });
|
|
}
|
|
}
|
|
};
|
|
|
|
request.onsuccess = function(event) {
|
|
resolve(event.target.result);
|
|
};
|
|
|
|
request.onerror = function(event) {
|
|
reject(event.target.error || new Error('无法打开缓存数据库'));
|
|
};
|
|
});
|
|
}
|
|
|
|
function normalizeUrl(url) {
|
|
if (typeof url !== 'string' || !url) {
|
|
return '';
|
|
}
|
|
|
|
try {
|
|
var parsed = new URL(url);
|
|
parsed.hash = '';
|
|
return parsed.href;
|
|
} catch (e) {
|
|
return url;
|
|
}
|
|
}
|
|
|
|
function getThreadKeyFromUrl(url) {
|
|
var normalized = normalizeUrl(url);
|
|
if (!normalized) {
|
|
return '';
|
|
}
|
|
|
|
var match = normalized.match(/thread-(\d+)-/i) || normalized.match(/[?&]tid=(\d+)/i);
|
|
return match ? match[1] : normalized;
|
|
}
|
|
|
|
function normalizeTitle(title) {
|
|
return String(title || '').replace(/\s+/g, ' ').trim();
|
|
}
|
|
|
|
function normalizeMagnets(magnets) {
|
|
var seen = Object.create(null);
|
|
var result = [];
|
|
|
|
if (!Array.isArray(magnets)) {
|
|
return result;
|
|
}
|
|
|
|
magnets.forEach(function(magnet) {
|
|
if (typeof magnet !== 'string' || !magnet) {
|
|
return;
|
|
}
|
|
|
|
if (seen[magnet]) {
|
|
return;
|
|
}
|
|
|
|
seen[magnet] = true;
|
|
result.push(magnet);
|
|
});
|
|
|
|
return result;
|
|
}
|
|
|
|
function normalizeThreadEntry(thread, forumKey, crawledAt) {
|
|
if (!thread || typeof thread !== 'object') {
|
|
return null;
|
|
}
|
|
|
|
var normalizedUrl = normalizeUrl(thread.url);
|
|
var normalizedTitle = normalizeTitle(thread.title);
|
|
var threadKey = typeof thread.threadKey === 'string' && thread.threadKey
|
|
? thread.threadKey
|
|
: getThreadKeyFromUrl(normalizedUrl);
|
|
|
|
if (!forumKey || !normalizedUrl || !threadKey) {
|
|
return null;
|
|
}
|
|
|
|
return {
|
|
cacheKey: forumKey + '::' + threadKey,
|
|
forumKey: forumKey,
|
|
threadKey: threadKey,
|
|
url: normalizedUrl,
|
|
title: normalizedTitle,
|
|
titleNorm: normalizedTitle.toLowerCase(),
|
|
lastSeenAt: crawledAt,
|
|
firstSeenAt: crawledAt,
|
|
magnets: normalizeMagnets(thread.magnets),
|
|
magnetCount: Array.isArray(thread.magnets) ? normalizeMagnets(thread.magnets).length : 0,
|
|
lastMagnetSyncAt: Array.isArray(thread.magnets) && thread.magnets.length > 0 ? crawledAt : 0
|
|
};
|
|
}
|
|
|
|
function getCoverageKey(forumKey, startPage, endPage) {
|
|
return forumKey + '::' + startPage + '-' + endPage;
|
|
}
|
|
|
|
function getPageCoverageKey(forumKey, page) {
|
|
return forumKey + '::page::' + page;
|
|
}
|
|
|
|
async function upsertThreadEntries(threadStore, forumKey, rawThreads, crawledAt) {
|
|
var seenThreadKeys = Object.create(null);
|
|
var threadKeys = [];
|
|
|
|
for (var threadIndex = 0; threadIndex < rawThreads.length; threadIndex++) {
|
|
var thread = rawThreads[threadIndex];
|
|
var normalized = normalizeThreadEntry(thread, forumKey, crawledAt);
|
|
if (!normalized || seenThreadKeys[normalized.cacheKey]) {
|
|
continue;
|
|
}
|
|
|
|
seenThreadKeys[normalized.cacheKey] = true;
|
|
threadKeys.push(normalized.cacheKey);
|
|
|
|
var existingRecord = await requestToPromise(threadStore.get(normalized.cacheKey));
|
|
var mergedMagnets = existingRecord && Array.isArray(existingRecord.magnets)
|
|
? existingRecord.magnets
|
|
: normalized.magnets;
|
|
var mergedLastMagnetSyncAt = existingRecord && Number(existingRecord.lastMagnetSyncAt)
|
|
? Number(existingRecord.lastMagnetSyncAt)
|
|
: normalized.lastMagnetSyncAt;
|
|
|
|
threadStore.put({
|
|
cacheKey: normalized.cacheKey,
|
|
forumKey: normalized.forumKey,
|
|
threadKey: normalized.threadKey,
|
|
url: normalized.url,
|
|
title: normalized.title,
|
|
titleNorm: normalized.titleNorm,
|
|
firstSeenAt: existingRecord && Number(existingRecord.firstSeenAt)
|
|
? Number(existingRecord.firstSeenAt)
|
|
: normalized.firstSeenAt,
|
|
lastSeenAt: normalized.lastSeenAt,
|
|
magnets: normalizeMagnets(mergedMagnets),
|
|
magnetCount: normalizeMagnets(mergedMagnets).length,
|
|
lastMagnetSyncAt: mergedLastMagnetSyncAt
|
|
});
|
|
}
|
|
|
|
return threadKeys;
|
|
}
|
|
|
|
async function getAssembledPageCoverage(db, forumKey, startPage, endPage) {
|
|
var pageTx = db.transaction('pageCoverages', 'readonly');
|
|
var pageStore = pageTx.objectStore('pageCoverages');
|
|
var pageRequests = [];
|
|
|
|
for (var page = startPage; page <= endPage; page++) {
|
|
pageRequests.push(requestToPromise(pageStore.get(getPageCoverageKey(forumKey, page))));
|
|
}
|
|
|
|
var pageRecords = await Promise.all(pageRequests);
|
|
await transactionDone(pageTx);
|
|
|
|
if (pageRecords.some(function(record) { return !record; })) {
|
|
return null;
|
|
}
|
|
|
|
var merged = Object.create(null);
|
|
var mergedThreadKeys = [];
|
|
var latestCrawledAt = 0;
|
|
|
|
pageRecords.forEach(function(record) {
|
|
latestCrawledAt = Math.max(latestCrawledAt, Number(record.crawledAt || 0));
|
|
(Array.isArray(record.threadKeys) ? record.threadKeys : []).forEach(function(threadKey) {
|
|
if (merged[threadKey]) {
|
|
return;
|
|
}
|
|
merged[threadKey] = true;
|
|
mergedThreadKeys.push(threadKey);
|
|
});
|
|
});
|
|
|
|
return getCoverageWithThreads(db, {
|
|
forumKey: forumKey,
|
|
startPage: startPage,
|
|
endPage: endPage,
|
|
crawledAt: latestCrawledAt,
|
|
strategy: 'assembled_pages',
|
|
frontRefreshPages: 0,
|
|
threadKeys: mergedThreadKeys
|
|
});
|
|
}
|
|
|
|
async function getCoverageWithThreads(db, coverageRecord) {
|
|
if (!coverageRecord) {
|
|
return null;
|
|
}
|
|
|
|
var threadKeys = Array.isArray(coverageRecord.threadKeys) ? coverageRecord.threadKeys : [];
|
|
var threads = [];
|
|
|
|
if (threadKeys.length > 0) {
|
|
var threadTx = db.transaction('threads', 'readonly');
|
|
var threadStore = threadTx.objectStore('threads');
|
|
var threadRequests = threadKeys.map(function(threadKey) {
|
|
return requestToPromise(threadStore.get(threadKey));
|
|
});
|
|
var threadResults = await Promise.all(threadRequests);
|
|
await transactionDone(threadTx);
|
|
|
|
threads = threadResults.filter(function(record) {
|
|
return !!record;
|
|
}).map(function(record) {
|
|
return {
|
|
url: record.url,
|
|
title: record.title,
|
|
threadKey: record.threadKey
|
|
};
|
|
});
|
|
}
|
|
|
|
return {
|
|
forumKey: coverageRecord.forumKey,
|
|
startPage: coverageRecord.startPage,
|
|
endPage: coverageRecord.endPage,
|
|
crawledAt: coverageRecord.crawledAt,
|
|
strategy: coverageRecord.strategy || 'full_live',
|
|
frontRefreshPages: coverageRecord.frontRefreshPages || 0,
|
|
threads: threads
|
|
};
|
|
}
|
|
|
|
async function listForumCoverages(db, forumKey) {
|
|
return new Promise(function(resolve, reject) {
|
|
var tx = db.transaction('coverages', 'readonly');
|
|
var store = tx.objectStore('coverages');
|
|
var index = store.index('forumKey');
|
|
var request = index.openCursor(IDBKeyRange.only(forumKey));
|
|
var results = [];
|
|
|
|
request.onsuccess = function(event) {
|
|
var cursor = event.target.result;
|
|
if (!cursor) {
|
|
resolve(results);
|
|
return;
|
|
}
|
|
|
|
results.push(cursor.value);
|
|
cursor.continue();
|
|
};
|
|
|
|
request.onerror = function(event) {
|
|
reject(event.target.error || new Error('读取覆盖快照失败'));
|
|
};
|
|
});
|
|
}
|
|
|
|
async function pruneOldCoverages(db, forumKey) {
|
|
var coverages = await listForumCoverages(db, forumKey);
|
|
if (coverages.length <= COVERAGE_LIMIT_PER_FORUM) {
|
|
return;
|
|
}
|
|
|
|
coverages.sort(function(a, b) {
|
|
return Number(b.crawledAt || 0) - Number(a.crawledAt || 0);
|
|
});
|
|
|
|
var stale = coverages.slice(COVERAGE_LIMIT_PER_FORUM);
|
|
if (stale.length === 0) {
|
|
return;
|
|
}
|
|
|
|
var tx = db.transaction('coverages', 'readwrite');
|
|
var store = tx.objectStore('coverages');
|
|
stale.forEach(function(record) {
|
|
store.delete(record.coverageKey);
|
|
});
|
|
await transactionDone(tx);
|
|
}
|
|
|
|
async function saveCoverageSnapshot(payload) {
|
|
var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : '';
|
|
var startPage = Math.max(1, Number(payload.startPage) || 1);
|
|
var endPage = Math.max(startPage, Number(payload.endPage) || startPage);
|
|
var crawledAt = Number(payload.crawledAt) || Date.now();
|
|
var frontRefreshPages = Math.max(0, Number(payload.frontRefreshPages) || 0);
|
|
var strategy = typeof payload.strategy === 'string' ? payload.strategy : 'full_live';
|
|
var rawThreads = Array.isArray(payload.threads) ? payload.threads : [];
|
|
|
|
var db = await openCacheDb();
|
|
try {
|
|
var tx = db.transaction(['threads', 'coverages'], 'readwrite');
|
|
var threadStore = tx.objectStore('threads');
|
|
var coverageStore = tx.objectStore('coverages');
|
|
var threadKeys = await upsertThreadEntries(threadStore, forumKey, rawThreads, crawledAt);
|
|
|
|
coverageStore.put({
|
|
coverageKey: getCoverageKey(forumKey, startPage, endPage),
|
|
forumKey: forumKey,
|
|
startPage: startPage,
|
|
endPage: endPage,
|
|
crawledAt: crawledAt,
|
|
strategy: strategy,
|
|
frontRefreshPages: frontRefreshPages,
|
|
threadKeys: threadKeys,
|
|
threadCount: threadKeys.length
|
|
});
|
|
|
|
await transactionDone(tx);
|
|
await pruneOldCoverages(db, forumKey);
|
|
|
|
return {
|
|
ok: true,
|
|
threadCount: threadKeys.length,
|
|
coverageKey: getCoverageKey(forumKey, startPage, endPage)
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
async function savePageCoverageSnapshot(payload) {
|
|
var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : '';
|
|
var page = Math.max(1, Number(payload.page) || 1);
|
|
var crawledAt = Number(payload.crawledAt) || Date.now();
|
|
var rawThreads = Array.isArray(payload.threads) ? payload.threads : [];
|
|
|
|
var db = await openCacheDb();
|
|
try {
|
|
var tx = db.transaction(['threads', 'pageCoverages'], 'readwrite');
|
|
var threadStore = tx.objectStore('threads');
|
|
var pageCoverageStore = tx.objectStore('pageCoverages');
|
|
var threadKeys = await upsertThreadEntries(threadStore, forumKey, rawThreads, crawledAt);
|
|
|
|
pageCoverageStore.put({
|
|
coverageKey: getPageCoverageKey(forumKey, page),
|
|
forumKey: forumKey,
|
|
page: page,
|
|
crawledAt: crawledAt,
|
|
threadKeys: threadKeys,
|
|
threadCount: threadKeys.length
|
|
});
|
|
|
|
await transactionDone(tx);
|
|
|
|
return {
|
|
ok: true,
|
|
page: page,
|
|
threadCount: threadKeys.length,
|
|
coverageKey: getPageCoverageKey(forumKey, page)
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
async function getCachedThreadMagnets(payload) {
|
|
var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : '';
|
|
var rawThreads = Array.isArray(payload.threads) ? payload.threads : [];
|
|
var db = await openCacheDb();
|
|
|
|
try {
|
|
var tx = db.transaction('threads', 'readonly');
|
|
var threadStore = tx.objectStore('threads');
|
|
var results = [];
|
|
|
|
for (var threadIndex = 0; threadIndex < rawThreads.length; threadIndex++) {
|
|
var thread = rawThreads[threadIndex];
|
|
var normalized = normalizeThreadEntry(thread, forumKey, Date.now());
|
|
if (!normalized) {
|
|
continue;
|
|
}
|
|
|
|
var record = await requestToPromise(threadStore.get(normalized.cacheKey));
|
|
if (!record) {
|
|
results.push({
|
|
threadKey: normalized.threadKey,
|
|
url: normalized.url,
|
|
title: normalized.title,
|
|
magnets: []
|
|
});
|
|
continue;
|
|
}
|
|
|
|
results.push({
|
|
threadKey: record.threadKey,
|
|
url: record.url,
|
|
title: record.title,
|
|
magnets: normalizeMagnets(record.magnets),
|
|
lastMagnetSyncAt: Number(record.lastMagnetSyncAt) || 0
|
|
});
|
|
}
|
|
|
|
await transactionDone(tx);
|
|
|
|
return {
|
|
ok: true,
|
|
threads: results
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
async function saveThreadMagnets(payload) {
|
|
var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : '';
|
|
var rawThreads = Array.isArray(payload.threads) ? payload.threads : [];
|
|
var syncedAt = Number(payload.syncedAt) || Date.now();
|
|
var db = await openCacheDb();
|
|
|
|
try {
|
|
var tx = db.transaction('threads', 'readwrite');
|
|
var threadStore = tx.objectStore('threads');
|
|
var savedCount = 0;
|
|
|
|
for (var threadIndex = 0; threadIndex < rawThreads.length; threadIndex++) {
|
|
var thread = rawThreads[threadIndex];
|
|
var normalized = normalizeThreadEntry(thread, forumKey, syncedAt);
|
|
if (!normalized) {
|
|
continue;
|
|
}
|
|
|
|
var existingRecord = await requestToPromise(threadStore.get(normalized.cacheKey));
|
|
var normalizedMagnets = normalizeMagnets(thread.magnets);
|
|
var finalMagnets = normalizedMagnets.length > 0
|
|
? normalizedMagnets
|
|
: (existingRecord && Array.isArray(existingRecord.magnets) ? normalizeMagnets(existingRecord.magnets) : []);
|
|
|
|
threadStore.put({
|
|
cacheKey: normalized.cacheKey,
|
|
forumKey: normalized.forumKey,
|
|
threadKey: normalized.threadKey,
|
|
url: normalized.url,
|
|
title: normalized.title || (existingRecord ? existingRecord.title : ''),
|
|
titleNorm: normalizeTitle(normalized.title || (existingRecord ? existingRecord.title : '')).toLowerCase(),
|
|
firstSeenAt: existingRecord && Number(existingRecord.firstSeenAt)
|
|
? Number(existingRecord.firstSeenAt)
|
|
: normalized.firstSeenAt,
|
|
lastSeenAt: existingRecord && Number(existingRecord.lastSeenAt)
|
|
? Number(existingRecord.lastSeenAt)
|
|
: normalized.lastSeenAt,
|
|
magnets: finalMagnets,
|
|
magnetCount: finalMagnets.length,
|
|
lastMagnetSyncAt: finalMagnets.length > 0 ? syncedAt : (existingRecord && Number(existingRecord.lastMagnetSyncAt) ? Number(existingRecord.lastMagnetSyncAt) : 0)
|
|
});
|
|
|
|
savedCount += 1;
|
|
}
|
|
|
|
await transactionDone(tx);
|
|
|
|
return {
|
|
ok: true,
|
|
savedCount: savedCount
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
function estimateTextSize(value) {
|
|
try {
|
|
return JSON.stringify(value).length;
|
|
} catch (e) {
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
async function getStorageEstimate() {
|
|
if (!navigator.storage || !navigator.storage.estimate) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
return await navigator.storage.estimate();
|
|
} catch (e) {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function readRecentThreadRecords(db, forumKey, limit) {
|
|
var tx = db.transaction('threads', 'readonly');
|
|
var store = tx.objectStore('threads');
|
|
var index = forumKey ? store.index('forumKeyLastSeenAt') : store.index('lastSeenAt');
|
|
var range = forumKey
|
|
? IDBKeyRange.bound([forumKey, 0], [forumKey, Number.MAX_SAFE_INTEGER])
|
|
: null;
|
|
var request = index.openCursor(range, 'prev');
|
|
|
|
return new Promise(function(resolve, reject) {
|
|
var items = [];
|
|
|
|
request.onsuccess = function(event) {
|
|
var cursor = event.target.result;
|
|
if (!cursor || items.length >= limit) {
|
|
resolve(items);
|
|
return;
|
|
}
|
|
|
|
items.push(cursor.value);
|
|
cursor.continue();
|
|
};
|
|
|
|
request.onerror = function(event) {
|
|
reject(event.target.error || new Error('读取最近帖子缓存失败'));
|
|
};
|
|
}).finally(function() {
|
|
return transactionDone(tx).catch(function() {
|
|
return null;
|
|
});
|
|
});
|
|
}
|
|
|
|
async function readRecentCoverageRecords(db, forumKey, limit) {
|
|
var tx = db.transaction('coverages', 'readonly');
|
|
var store = tx.objectStore('coverages');
|
|
var index = forumKey ? store.index('forumKeyCrawledAt') : store.index('crawledAt');
|
|
var range = forumKey
|
|
? IDBKeyRange.bound([forumKey, 0], [forumKey, Number.MAX_SAFE_INTEGER])
|
|
: null;
|
|
var request = index.openCursor(range, 'prev');
|
|
|
|
return new Promise(function(resolve, reject) {
|
|
var items = [];
|
|
|
|
request.onsuccess = function(event) {
|
|
var cursor = event.target.result;
|
|
if (!cursor || items.length >= limit) {
|
|
resolve(items);
|
|
return;
|
|
}
|
|
|
|
items.push(cursor.value);
|
|
cursor.continue();
|
|
};
|
|
|
|
request.onerror = function(event) {
|
|
reject(event.target.error || new Error('读取最近范围快照失败'));
|
|
};
|
|
}).finally(function() {
|
|
return transactionDone(tx).catch(function() {
|
|
return null;
|
|
});
|
|
});
|
|
}
|
|
|
|
async function getCacheOverview(payload) {
|
|
var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : '';
|
|
var limit = Math.max(1, Math.min(Number(payload.limit) || 12, 50));
|
|
var db = await openCacheDb();
|
|
|
|
try {
|
|
var threadTx = db.transaction('threads', 'readonly');
|
|
var coverageTx = db.transaction(['coverages', 'pageCoverages'], 'readonly');
|
|
var threadStore = threadTx.objectStore('threads');
|
|
var coverageStore = coverageTx.objectStore('coverages');
|
|
var pageCoverageStore = coverageTx.objectStore('pageCoverages');
|
|
var threadForumIndex = threadStore.index('forumKey');
|
|
var coverageForumIndex = coverageStore.index('forumKey');
|
|
var pageCoverageForumIndex = pageCoverageStore.index('forumKey');
|
|
|
|
var totals = await Promise.all([
|
|
requestToPromise(threadStore.count()),
|
|
requestToPromise(coverageStore.count()),
|
|
requestToPromise(pageCoverageStore.count()),
|
|
forumKey ? requestToPromise(threadForumIndex.count(IDBKeyRange.only(forumKey))) : Promise.resolve(0),
|
|
forumKey ? requestToPromise(coverageForumIndex.count(IDBKeyRange.only(forumKey))) : Promise.resolve(0),
|
|
forumKey ? requestToPromise(pageCoverageForumIndex.count(IDBKeyRange.only(forumKey))) : Promise.resolve(0)
|
|
]);
|
|
|
|
await Promise.all([transactionDone(threadTx), transactionDone(coverageTx)]);
|
|
|
|
var recentThreads = await readRecentThreadRecords(db, forumKey, limit);
|
|
var recentCoverages = await readRecentCoverageRecords(db, forumKey, Math.min(limit, 8));
|
|
var usedBytes = recentThreads.reduce(function(sum, item) {
|
|
return sum + estimateTextSize(item);
|
|
}, 0);
|
|
var storageEstimate = await getStorageEstimate();
|
|
var magnetCachedThreads = recentThreads.filter(function(item) {
|
|
return Number(item.magnetCount || 0) > 0;
|
|
}).length;
|
|
var magnetCount = recentThreads.reduce(function(sum, item) {
|
|
return sum + Number(item.magnetCount || 0);
|
|
}, 0);
|
|
|
|
return {
|
|
ok: true,
|
|
summary: {
|
|
forumKey: forumKey,
|
|
totalThreadCount: Number(totals[0] || 0),
|
|
totalCoverageCount: Number(totals[1] || 0),
|
|
totalPageCoverageCount: Number(totals[2] || 0),
|
|
forumThreadCount: Number(totals[3] || 0),
|
|
forumCoverageCount: Number(totals[4] || 0),
|
|
forumPageCoverageCount: Number(totals[5] || 0),
|
|
sampleBytes: usedBytes,
|
|
storageUsage: storageEstimate && Number(storageEstimate.usage) ? Number(storageEstimate.usage) : 0,
|
|
storageQuota: storageEstimate && Number(storageEstimate.quota) ? Number(storageEstimate.quota) : 0,
|
|
recentMagnetCachedThreads: magnetCachedThreads,
|
|
recentMagnetCount: magnetCount
|
|
},
|
|
recentThreads: recentThreads.map(function(item) {
|
|
return {
|
|
title: item.title,
|
|
url: item.url,
|
|
threadKey: item.threadKey,
|
|
magnetCount: Number(item.magnetCount || 0),
|
|
lastSeenAt: Number(item.lastSeenAt || 0),
|
|
lastMagnetSyncAt: Number(item.lastMagnetSyncAt || 0)
|
|
};
|
|
}),
|
|
recentCoverages: recentCoverages.map(function(item) {
|
|
return {
|
|
forumKey: item.forumKey,
|
|
startPage: Number(item.startPage || 0),
|
|
endPage: Number(item.endPage || 0),
|
|
threadCount: Number(item.threadCount || 0),
|
|
crawledAt: Number(item.crawledAt || 0),
|
|
strategy: item.strategy || 'full_live'
|
|
};
|
|
})
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
async function clearCacheData() {
|
|
var db = await openCacheDb();
|
|
|
|
try {
|
|
var tx = db.transaction(['threads', 'coverages', 'pageCoverages'], 'readwrite');
|
|
tx.objectStore('threads').clear();
|
|
tx.objectStore('coverages').clear();
|
|
tx.objectStore('pageCoverages').clear();
|
|
await transactionDone(tx);
|
|
|
|
return { ok: true };
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
async function getCachedCoveragePlan(payload) {
|
|
var forumKey = typeof payload.forumKey === 'string' ? payload.forumKey : '';
|
|
var startPage = Math.max(1, Number(payload.startPage) || 1);
|
|
var endPage = Math.max(startPage, Number(payload.endPage) || startPage);
|
|
var frontRefreshPages = Math.max(0, Number(payload.frontRefreshPages) || 0);
|
|
|
|
var db = await openCacheDb();
|
|
try {
|
|
var exactKey = getCoverageKey(forumKey, startPage, endPage);
|
|
var exactTx = db.transaction('coverages', 'readonly');
|
|
var exactStore = exactTx.objectStore('coverages');
|
|
var exactRecord = await requestToPromise(exactStore.get(exactKey));
|
|
await transactionDone(exactTx);
|
|
|
|
var exactCoverage = exactRecord ? await getCoverageWithThreads(db, exactRecord) : null;
|
|
if (!exactCoverage) {
|
|
exactCoverage = await getAssembledPageCoverage(db, forumKey, startPage, endPage);
|
|
}
|
|
if (exactCoverage) {
|
|
return {
|
|
ok: true,
|
|
exactCoverage: exactCoverage,
|
|
shiftedCoverage: null
|
|
};
|
|
}
|
|
|
|
var shiftedCoverage = null;
|
|
if (startPage === 1 && frontRefreshPages > 0) {
|
|
var coverages = await listForumCoverages(db, forumKey);
|
|
coverages = coverages.filter(function(record) {
|
|
return Number(record.startPage) === 1 && Number(record.endPage) >= 1;
|
|
}).sort(function(a, b) {
|
|
var crawlDiff = Number(b.crawledAt || 0) - Number(a.crawledAt || 0);
|
|
if (crawlDiff !== 0) {
|
|
return crawlDiff;
|
|
}
|
|
return Number(b.endPage || 0) - Number(a.endPage || 0);
|
|
});
|
|
|
|
if (coverages.length > 0) {
|
|
var bestCoverage = coverages[0];
|
|
var hydratedCoverage = await getCoverageWithThreads(db, bestCoverage);
|
|
if (hydratedCoverage && Array.isArray(hydratedCoverage.threads) && hydratedCoverage.threads.length > 0) {
|
|
shiftedCoverage = {
|
|
forumKey: hydratedCoverage.forumKey,
|
|
sourceStartPage: hydratedCoverage.startPage,
|
|
sourceEndPage: hydratedCoverage.endPage,
|
|
crawledAt: hydratedCoverage.crawledAt,
|
|
strategy: hydratedCoverage.strategy,
|
|
threads: hydratedCoverage.threads,
|
|
reusedStartPage: Math.min(endPage, startPage + frontRefreshPages),
|
|
reusedEndPage: Math.min(endPage, bestCoverage.endPage + frontRefreshPages)
|
|
};
|
|
|
|
if (shiftedCoverage.reusedStartPage > shiftedCoverage.reusedEndPage) {
|
|
shiftedCoverage = null;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
ok: true,
|
|
exactCoverage: null,
|
|
shiftedCoverage: shiftedCoverage
|
|
};
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
function isProgressStateKey(key) {
|
|
return typeof key === 'string' && key.indexOf(STORAGE_KEY_PREFIX) === 0;
|
|
}
|
|
|
|
function flushSessionReadyCallbacks() {
|
|
var callbacks = sessionReadyCallbacks.slice();
|
|
sessionReadyCallbacks = [];
|
|
callbacks.forEach(function(callback) {
|
|
try {
|
|
callback();
|
|
} catch (e) {
|
|
// ignore callback errors
|
|
}
|
|
});
|
|
}
|
|
|
|
function finishSessionPrepare() {
|
|
isSessionReady = true;
|
|
isPreparingSession = false;
|
|
flushSessionReadyCallbacks();
|
|
}
|
|
|
|
function clearAllProgressStates(done) {
|
|
chrome.storage.local.get(null, function(result) {
|
|
if (chrome.runtime.lastError) {
|
|
done();
|
|
return;
|
|
}
|
|
|
|
var keys = Object.keys(result || {}).filter(isProgressStateKey);
|
|
if (keys.length === 0) {
|
|
done();
|
|
return;
|
|
}
|
|
|
|
chrome.storage.local.remove(keys, function() {
|
|
done();
|
|
});
|
|
});
|
|
}
|
|
|
|
function ensureSessionReady(callback) {
|
|
if (isSessionReady) {
|
|
callback();
|
|
return;
|
|
}
|
|
|
|
sessionReadyCallbacks.push(callback);
|
|
if (isPreparingSession) {
|
|
return;
|
|
}
|
|
|
|
isPreparingSession = true;
|
|
if (!chrome.storage || !chrome.storage.session) {
|
|
finishSessionPrepare();
|
|
return;
|
|
}
|
|
|
|
chrome.storage.session.get(SESSION_MARKER_KEY, function(result) {
|
|
if (chrome.runtime.lastError) {
|
|
finishSessionPrepare();
|
|
return;
|
|
}
|
|
|
|
if (result && result[SESSION_MARKER_KEY]) {
|
|
finishSessionPrepare();
|
|
return;
|
|
}
|
|
|
|
clearAllProgressStates(function() {
|
|
var marker = {};
|
|
marker[SESSION_MARKER_KEY] = Date.now();
|
|
chrome.storage.session.set(marker, function() {
|
|
finishSessionPrepare();
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
function clampTimeout(timeoutMs) {
|
|
var parsed = Number(timeoutMs);
|
|
if (!Number.isFinite(parsed)) return 15000;
|
|
return Math.max(3000, Math.min(parsed, 30000));
|
|
}
|
|
|
|
function getStorageKeyBySender(sender) {
|
|
if (!sender || !sender.tab || typeof sender.tab.id !== 'number') {
|
|
return null;
|
|
}
|
|
return STORAGE_KEY_PREFIX + sender.tab.id;
|
|
}
|
|
|
|
function normalizeProgressState(state) {
|
|
if (!state || typeof state !== 'object') {
|
|
return null;
|
|
}
|
|
|
|
var speed = typeof state.speedMode === 'string' ? state.speedMode : 'fast';
|
|
if (speed !== 'slow' && speed !== 'medium' && speed !== 'fast' && speed !== 'ultrafast') {
|
|
speed = 'fast';
|
|
}
|
|
|
|
var startPage = Number(state.startPage);
|
|
var endPage = Number(state.endPage);
|
|
if (!Number.isFinite(startPage) || startPage < 1) startPage = 1;
|
|
if (!Number.isFinite(endPage) || endPage < 1) endPage = startPage;
|
|
|
|
var resumeFromPage = Number(state.resumeFromPage);
|
|
if (!Number.isFinite(resumeFromPage) || resumeFromPage < 1) {
|
|
resumeFromPage = startPage;
|
|
}
|
|
|
|
var isRunning = !!state.isRunning;
|
|
var stoppedByUser = !!state.stoppedByUser;
|
|
|
|
var linkSet = new Set();
|
|
var links = [];
|
|
if (Array.isArray(state.links)) {
|
|
state.links.forEach(function(link) {
|
|
if (typeof link !== 'string' || !link) return;
|
|
if (linkSet.has(link)) return;
|
|
linkSet.add(link);
|
|
links.push(link);
|
|
});
|
|
}
|
|
|
|
var items = [];
|
|
if (Array.isArray(state.items)) {
|
|
state.items.forEach(function(item) {
|
|
if (!item || typeof item !== 'object') return;
|
|
var link = typeof item.link === 'string' ? item.link : '';
|
|
if (!link) return;
|
|
var title = typeof item.title === 'string' ? item.title : '恢复记录';
|
|
if (!linkSet.has(link)) {
|
|
linkSet.add(link);
|
|
links.push(link);
|
|
}
|
|
items.push({
|
|
title: title.slice(0, 200),
|
|
link: link
|
|
});
|
|
});
|
|
}
|
|
|
|
if (links.length > MAX_PROGRESS_ITEMS) {
|
|
links = links.slice(links.length - MAX_PROGRESS_ITEMS);
|
|
}
|
|
if (items.length > MAX_PROGRESS_ITEMS) {
|
|
items = items.slice(items.length - MAX_PROGRESS_ITEMS);
|
|
}
|
|
|
|
return {
|
|
links: links,
|
|
items: items,
|
|
keyword: typeof state.keyword === 'string' ? state.keyword.slice(0, 200) : '',
|
|
speedMode: speed,
|
|
startPage: startPage,
|
|
endPage: endPage,
|
|
resumeFromPage: resumeFromPage,
|
|
isRunning: isRunning,
|
|
stoppedByUser: stoppedByUser,
|
|
statusText: typeof state.statusText === 'string' ? state.statusText.slice(0, 300) : '',
|
|
statusType: typeof state.statusType === 'string' ? state.statusType.slice(0, 30) : '',
|
|
pageUrl: typeof state.pageUrl === 'string' ? state.pageUrl.slice(0, 1000) : '',
|
|
updatedAt: Date.now()
|
|
};
|
|
}
|
|
|
|
function fetchPageHtml(url, timeoutMs) {
|
|
var controller = new AbortController();
|
|
var realTimeout = clampTimeout(timeoutMs);
|
|
var timer = setTimeout(function() {
|
|
controller.abort();
|
|
}, realTimeout);
|
|
|
|
return fetch(url, { signal: controller.signal })
|
|
.then(function(response) {
|
|
if (!response.ok) {
|
|
throw new Error('HTTP ' + response.status + ' ' + response.statusText);
|
|
}
|
|
return response.text();
|
|
})
|
|
.catch(function(err) {
|
|
if (err && err.name === 'AbortError') {
|
|
throw new Error('请求超时(' + realTimeout + 'ms)');
|
|
}
|
|
throw err;
|
|
})
|
|
.finally(function() {
|
|
clearTimeout(timer);
|
|
});
|
|
}
|
|
|
|
function extractMagnetsFromHtml(html) {
|
|
const magnetPattern = /magnet:\?xt=urn:btih:[a-fA-F0-9]{32,40}/gi;
|
|
const magnets = html.match(magnetPattern) || [];
|
|
return [...new Set(magnets)];
|
|
}
|
|
|
|
chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
|
|
if (request.action === 'cacheSaveCoverage') {
|
|
saveCoverageSnapshot(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheGetCoveragePlan') {
|
|
getCachedCoveragePlan(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheGetThreadMagnets') {
|
|
getCachedThreadMagnets(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheSaveThreadMagnets') {
|
|
saveThreadMagnets(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheSavePageCoverage') {
|
|
savePageCoverageSnapshot(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheGetOverview') {
|
|
getCacheOverview(request)
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'cacheClearAll') {
|
|
clearCacheData()
|
|
.then(function(result) {
|
|
sendResponse(result);
|
|
})
|
|
.catch(function(err) {
|
|
sendResponse({ ok: false, error: err && err.message ? err.message : String(err) });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'saveProgressState') {
|
|
ensureSessionReady(function() {
|
|
const saveKey = getStorageKeyBySender(sender);
|
|
if (!saveKey) {
|
|
sendResponse({ ok: false, error: '无法识别标签页' });
|
|
return;
|
|
}
|
|
|
|
const normalizedState = normalizeProgressState(request.state);
|
|
if (!normalizedState) {
|
|
sendResponse({ ok: false, error: '状态数据无效' });
|
|
return;
|
|
}
|
|
|
|
chrome.storage.local.set({ [saveKey]: normalizedState }, function() {
|
|
if (chrome.runtime.lastError) {
|
|
sendResponse({ ok: false, error: chrome.runtime.lastError.message });
|
|
return;
|
|
}
|
|
sendResponse({ ok: true });
|
|
});
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'loadProgressState') {
|
|
ensureSessionReady(function() {
|
|
const loadKey = getStorageKeyBySender(sender);
|
|
if (!loadKey) {
|
|
sendResponse({ ok: false, error: '无法识别标签页', state: null });
|
|
return;
|
|
}
|
|
|
|
chrome.storage.local.get(loadKey, function(result) {
|
|
if (chrome.runtime.lastError) {
|
|
sendResponse({ ok: false, error: chrome.runtime.lastError.message, state: null });
|
|
return;
|
|
}
|
|
sendResponse({ ok: true, state: result && result[loadKey] ? result[loadKey] : null });
|
|
});
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'clearProgressState') {
|
|
ensureSessionReady(function() {
|
|
const clearKey = getStorageKeyBySender(sender);
|
|
if (!clearKey) {
|
|
sendResponse({ ok: false, error: '无法识别标签页' });
|
|
return;
|
|
}
|
|
|
|
chrome.storage.local.remove(clearKey, function() {
|
|
if (chrome.runtime.lastError) {
|
|
sendResponse({ ok: false, error: chrome.runtime.lastError.message });
|
|
return;
|
|
}
|
|
sendResponse({ ok: true });
|
|
});
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'fetchHtml') {
|
|
fetchPageHtml(request.url, request.timeoutMs)
|
|
.then(html => {
|
|
sendResponse({ html: html });
|
|
})
|
|
.catch(err => {
|
|
sendResponse({ html: '', error: err.message });
|
|
});
|
|
return true;
|
|
}
|
|
|
|
if (request.action === 'openAndFetch') {
|
|
fetchPageHtml(request.url, request.timeoutMs)
|
|
.then(html => {
|
|
sendResponse({ magnets: extractMagnetsFromHtml(html) });
|
|
})
|
|
.catch(err => {
|
|
sendResponse({ magnets: [], error: err.message });
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
});
|
|
|
|
if (chrome.tabs && chrome.tabs.onRemoved) {
|
|
chrome.tabs.onRemoved.addListener(function(tabId) {
|
|
var key = STORAGE_KEY_PREFIX + tabId;
|
|
chrome.storage.local.remove(key, function() {
|
|
// ignore cleanup errors
|
|
});
|
|
});
|
|
}
|