lucasw89/KefinTweaks
1
0
Fork 0
mirror of https://github.com/ranaldsgift/KefinTweaks.git synced 2026-11-05 15:52:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

Improve loading from sections with local caches

Browse Source
This commit is contained in:
HighImKevin
2026-09-25 05:03:31 -04:00
parent 356da23ce1
commit e4ddc9e645
6 changed files with 213 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
helpers/dataHelper.js
+5 -5
View File
@@ -46,11 +46,11 @@
async function unwrapGetQueryResult(result) {
if (!result) return null;
if (typeof result.ensureData === 'function' || result.dataPromise != null) {
if (result.data != null) return result.data;
return await (typeof result.ensureData === 'function'
? result.ensureData()
: result.dataPromise);
if (typeof result.ensureData === 'function') {
return await result.ensureData();
}
if (result.dataPromise != null) {
return await result.dataPromise;
}
return result;
}
scripts/apiHelper.js
+47 -12
View File
@@ -385,7 +385,7 @@
* @param {Object} options - Cache options
* @param {boolean} options.useCache - Whether to return cached data first (default: false)
* @param {number} options.ttl - Cache time to live in milliseconds (default: 300000)
* @returns {Promise<Object>|Object} - If useCache is true, returns { data, dataPromise, isStalePromise }. Otherwise returns Promise<data>.
* @returns {Object|Promise<Object>} - If useCache is true, returns progressive stub { data, ensureData, isStalePromise } synchronously. Otherwise returns Promise<data>.
*/
getQuery: function(url, options = {}) {
ensureApiClient();
@@ -616,14 +616,49 @@
return result.ensureData();
}
});
return (async () => {
// Initial check for cached data
const entry = await cachePromise;
result.data = entry ? entry.payload : null;
result.isStale = forceRefresh || !entry || (Date.now() - entry.timestamp > ttl);
return result;
})();
// Sync stub: do not await IndexedDB before return (home first paint).
// cachePromise / isStalePromise hydrate result.data in the background.
// Thenable waits for hydrate but must NOT fulfill with `result` itself
// (Promise assimilation would recurse on result.then forever).
result.data = { Items: [] };
result.isStale = true;
const readyPromise = cachePromise.then((entry) => {
if (entry) {
result.data = entry.payload;
result.isStale = forceRefresh || (Date.now() - entry.timestamp > ttl);
} else {
// Cache miss: clear placeholder so data?.Items cannot fake a hit
result.data = null;
result.isStale = true;
}
}).catch(() => {
result.data = null;
result.isStale = true;
});
function toSettledView() {
return {
data: result.data,
isStale: result.isStale,
isStalePromise: result.isStalePromise,
ensureData: result.ensureData,
get dataPromise() {
return result.ensureData();
}
};
}
result.then = function (onFulfilled, onRejected) {
return readyPromise.then(
() => (typeof onFulfilled === 'function' ? onFulfilled(toSettledView()) : toSettledView()),
onRejected
);
};
result.catch = function (onRejected) {
return result.then(undefined, onRejected);
};
return result;
},
invalidateCache: function(url) {
@@ -721,7 +756,7 @@
throw new Error('Invalid watchlist query URL');
}
const queryResult = await this.getQuery(url, {
const queryResult = this.getQuery(url, {
useCache: true,
ttl,
forceRefresh
@@ -744,8 +779,8 @@
};
const result = {
data: normalize(queryResult.data),
isStale: queryResult.isStale === true,
data: { Items: [] },
isStale: true,
isStalePromise: queryResult.isStalePromise || Promise.resolve(false),
ensureData,
urls: [url]
scripts/libraryCache.js
+10 -10
View File
@@ -12,14 +12,14 @@
return type;
}
function isComplete(type) {
async function isComplete(type) {
const t = normalizeType(type);
if (t === 'Movie') return window.MoviesCache?.isComplete?.() === true;
if (t === 'Movie') return (await window.MoviesCache?.isComplete?.()) === true;
if (t === 'Series') return window.SeriesCache?.isComplete?.() === true;
return false;
}
function isReady(type) {
async function isReady(type) {
return isComplete(type);
}
@@ -38,11 +38,11 @@
/**
* Whether a dataSource string depends on an incomplete Movies/Series/People cache.
*/
function dataSourceDependsOnIncompleteCache(dataSource) {
async function dataSourceDependsOnIncompleteCache(dataSource) {
const ds = typeof dataSource === 'string' ? dataSource : '';
if (!ds) return false;
if (ds.startsWith('MoviesCache.') && !isComplete('Movie')) return true;
if (ds.startsWith('SeriesCache.') && !isComplete('Series')) return true;
if (ds.startsWith('MoviesCache.') && !(await isComplete('Movie'))) return true;
if (ds.startsWith('SeriesCache.') && !(await isComplete('Series'))) return true;
if (ds.startsWith('PeopleCache.') && window.PeopleCache?.isComplete && !window.PeopleCache.isComplete()) {
return true;
}
@@ -52,17 +52,17 @@
/**
* Whether a home section depends on a library/people/external cache that is not ready.
*/
function sectionDependsOnIncompleteCache(section) {
async function sectionDependsOnIncompleteCache(section) {
if (!section) return false;
if (Array.isArray(section.externalListUrls) && section.externalListUrls.length) {
return !(isComplete('Movie') && isComplete('Series'));
return !((await isComplete('Movie')) && (await isComplete('Series')));
}
if (dataSourceDependsOnIncompleteCache(section.discoverySourceQuery?.dataSource)) {
if (await dataSourceDependsOnIncompleteCache(section.discoverySourceQuery?.dataSource)) {
return true;
}
const queries = section.queries || [];
for (const q of queries) {
if (dataSourceDependsOnIncompleteCache(q?.dataSource)) return true;
if (await dataSourceDependsOnIncompleteCache(q?.dataSource)) return true;
}
return false;
}
scripts/libraryCacheUtils.js
+1
View File
@@ -76,6 +76,7 @@
Chapters: Array.isArray(item.Chapters) ? item.Chapters.map(stripChapter).filter(Boolean) : [],
Trickplay: item.Trickplay || undefined,
DateCreated: item.DateCreated,
DateLastMediaAdded: item.DateLastMediaAdded,
DateLastSaved: item.DateLastSaved,
UserData: item.UserData || {}
};
scripts/moviesCache.js
+38 -7
View File
@@ -13,7 +13,7 @@
const IMDB_TOP_250_CACHE_TTL = 24 * 60 * 60 * 1000;
const MOVIE_FIELDS = [
'ProviderIds', 'People', 'Studios', 'Taglines', 'Genres', 'Overview', 'PrimaryImageAspectRatio',
'DateCreated'
'DateCreated', 'DateLastMediaAdded'
].join(',');
let movies = null;
@@ -55,6 +55,24 @@
}
}
/** Max DateLastMediaAdded || DateCreated among movies; null if none usable. */
function watermarkFromMovies(list) {
let best = null;
for (const m of list || []) {
const raw = m?.DateLastMediaAdded || m?.DateCreated;
if (!raw) continue;
const t = new Date(raw).getTime();
if (!Number.isFinite(t)) continue;
if (best == null || t > best) best = t;
}
return best != null ? new Date(best).toISOString() : null;
}
function setLastDateFetchedFromMovies(list) {
const iso = watermarkFromMovies(list);
if (iso) setLastDateFetched(iso);
}
function getApiHelper() {
return window.apiHelper || window.ApiHelper;
}
@@ -158,7 +176,7 @@
rebuildProviderIndex(movies);
await cache.set(CACHE_NAME, movies, userId, getTtl());
await cache.clear(PARTIAL_CACHE_NAME, userId);
setLastDateFetched(new Date().toISOString());
setLastDateFetchedFromMovies(movies);
LOG(`Movie crawl complete: ${movies.length} items`);
return movies;
} catch (err) {
@@ -286,7 +304,7 @@
rebuildProviderIndex(movies);
await cache.set(CACHE_NAME, movies, userId, getTtl());
setLastDateFetched(new Date().toISOString());
setLastDateFetchedFromMovies(movies);
// Notify people cache of deltas when available
if ((incoming.length || removedIds.length) && window.PeopleCache?.applyMovieDelta) {
try {
@@ -314,8 +332,21 @@
return fetchPromise;
}
function isCacheComplete() {
return isComplete === true && Array.isArray(movies);
async function isCacheComplete() {
if (isComplete === true && Array.isArray(movies)) return true;
const cache = window.IndexedDBCache;
const userId = window.ApiClient?.getCurrentUserId?.();
if (!cache || !userId) return false;
const complete = await cache.get(CACHE_NAME, userId);
if (complete && Array.isArray(complete)) {
if (!movies) {
movies = complete;
rebuildProviderIndex(movies);
}
isComplete = true;
return true;
}
return false;
}
async function getMoviesByProviderIds(ids) {
@@ -372,7 +403,7 @@
if (cached) return cached;
if (!imdbTop250Movies) {
if (!isCacheComplete()) {
if (!(await isCacheComplete())) {
LOG('IMDb Top 250 waiting for movie cache…');
}
const imdbIds = await fetchImdbTop250Data();
@@ -389,7 +420,7 @@
async function initialize() {
await getMovies();
if (isCacheComplete() && getLastDateFetched()) {
if ((await isCacheComplete()) && getLastDateFetched()) {
await syncMoviesCache();
}
return movies || [];
scripts/sectionHelper.js
+112 -81
View File
@@ -56,29 +56,39 @@
discoveryState.pairResolvePromises.clear();
}
/**
* One-shot: whether this section's dependent Movies/Series/People cache is already complete.
* Incomplete → skip (do not poll/wait). Missing LibraryCache → treat as ready.
*/
async function isSectionCacheReady(sectionConfig) {
if (!window.LibraryCache?.sectionDependsOnIncompleteCache) return true;
try {
return (await window.LibraryCache.sectionDependsOnIncompleteCache(sectionConfig)) !== true;
} catch (_) {
return true;
}
}
function scheduleViewMoreUrl(sectionConfig) {
resolveViewMoreUrl(sectionConfig).then((viewMoreUrl) => {
if (viewMoreUrl) sectionConfig.viewMoreUrl = viewMoreUrl;
}).catch((err) => {
WARN(`Failed to resolve viewMoreUrl for ${sectionConfig?.id}:`, err);
});
}
async function loadSectionForRendering(sectionConfig) {
LOG(`Loading Section for Rendering: ${sectionConfig.id} (${sectionConfig.name})`);
const loadSectionTimerStart = performance.now();
// Omit sections that would wait on incomplete Movies/Series/People crawls
// (static-item sections never depend on those caches)
const hasStaticItems = sectionConfig.items && sectionConfig.items.length > 0;
if (!hasStaticItems && window.LibraryCache?.sectionDependsOnIncompleteCache?.(sectionConfig) === true) {
LOG(`Skipping section ${sectionConfig.id}: depends on incomplete library cache`);
return null;
}
// Prepare section (template replacements, view more URL, etc.)
const viewMoreUrl = await resolveViewMoreUrl(sectionConfig);
// Sync prep only (viewMore deferred for non-static so it does not block first paint)
const cardFormat = resolveCardFormat(sectionConfig);
const sectionName = fillTemplate(sectionConfig.name, sectionConfig.metadata);
sectionConfig.overflowCard = true;
if (viewMoreUrl) {
sectionConfig.viewMoreUrl = viewMoreUrl;
}
if (cardFormat) {
sectionConfig.cardFormat = cardFormat;
}
@@ -102,6 +112,11 @@
// Static items: normalize and return immediately (no queries, no dataPromise)
if (hasStaticItems) {
const viewMoreUrl = await resolveViewMoreUrl(sectionConfig);
if (viewMoreUrl) {
sectionConfig.viewMoreUrl = viewMoreUrl;
}
const kefinTweaksRoot = window.KefinTweaksConfig?.kefinTweaksRoot || '';
const serverId = ApiClient.serverId();
const normalizeTemplate = (value) => (value || '')
@@ -147,6 +162,14 @@
};
}
// Cache-backed sections only mount once Movies/Series/People cache is complete
if (!(await isSectionCacheReady(sectionConfig))) {
LOG(`Skipping section ${sectionConfig.id}: dependent library cache not complete`);
return null;
}
scheduleViewMoreUrl(sectionConfig);
// External list (MDBList) — progressive for network fetch when caches are already ready
if (Array.isArray(sectionConfig.externalListUrls) && sectionConfig.externalListUrls.length > 0) {
if (!window.KefinExternalList?.resolveExternalListItems) {
@@ -158,6 +181,9 @@
const ensureData = () => {
if (!mappedDataPromise) {
mappedDataPromise = (async () => {
if (!(await isSectionCacheReady(sectionConfig))) {
return [];
}
const listResult = await window.KefinExternalList.resolveExternalListItems(sectionConfig);
let items = listResult?.Items || [];
if (window.cardBuilder.postProcessItems) {
@@ -174,6 +200,7 @@
const result = {
data: { Items: [] },
isStale: true,
ensureData
};
Object.defineProperty(result, 'dataPromise', {
@@ -198,7 +225,6 @@
const userId = ApiClient.getCurrentUserId();
const serverUrl = ApiClient.serverAddress();
const results = [];
const resolveQueries = window.cardBuilder?.resolveQueriesToLoad;
const queriesToLoad = typeof resolveQueries === 'function'
@@ -220,74 +246,76 @@
LOG(`Section ${sectionConfig.id}: useRandomQuery — selected query index ${index}`);
}
// Process each query in the queries array
for (const query of queriesToLoad) {
let queryResult;
if (query.dataSource) {
// Handle cache-based data sources
queryResult = await ApiHelper.fetchFromDataSource(query.dataSource, query.queryOptions || {});
} else {
// Build and execute query
const queryUrl = ApiHelper.buildQueryFromSection(query, userId, serverUrl, sectionConfig.renderMode === 'Spotlight', { sectionType: sectionConfig.type });
if (typeof queryUrl === 'string') {
// Standard query
const cacheCfg = Config().CACHE || {};
let sectionTtl = cacheCfg.DEFAULT_TTL;
if (Number(sectionConfig.ttl) >= 0) {
sectionTtl = Number(sectionConfig.ttl);
}
queryResult = await ApiHelper.getQuery(queryUrl, {
useCache: true,
ttl: sectionTtl
});
} else {
WARN(`Invalid query URL for section ${sectionConfig.id}`);
continue;
}
}
results.push(queryResult);
}
// If multiple queries, merge results using section-level sortBy/sortOrder
if (results.length > 1) {
const loadSectionTimerEnd = performance.now();
const loadSectionDuration = loadSectionTimerEnd - loadSectionTimerStart;
LOG(`Section ${sectionConfig.id} loaded for rendering in time: ${loadSectionDuration.toFixed(2)}ms`);
return ApiHelper.mergeMultiQueryResults(results, sectionConfig);
}
const loadSectionTimerEnd = performance.now();
const loadSectionDuration = loadSectionTimerEnd - loadSectionTimerStart;
LOG(`Section ${sectionConfig.id} loaded for rendering in time: ${loadSectionDuration.toFixed(2)}ms`);
const postProcess = (sectionConfig, items) => {
const postProcess = (cfg, items) => {
let postProcessedItems = items;
if (window.cardBuilder.postProcessItems) {
postProcessedItems = window.cardBuilder.postProcessItems(sectionConfig, items);
postProcessedItems = window.cardBuilder.postProcessItems(cfg, items);
}
return sortItemsByConfiguredIds(sectionConfig, postProcessedItems);
return sortItemsByConfiguredIds(cfg, postProcessedItems);
};
const queryResult = results[0];
let mappedDataPromise = null;
const ensureData = () => {
if (!mappedDataPromise) {
const raw = typeof queryResult.ensureData === 'function'
? queryResult.ensureData()
: queryResult.dataPromise;
mappedDataPromise = Promise.resolve(raw).then((data) => postProcess(sectionConfig, data));
mappedDataPromise = (async () => {
if (!(await isSectionCacheReady(sectionConfig))) {
return [];
}
const results = [];
for (const query of queriesToLoad) {
let queryResult;
if (query.dataSource) {
queryResult = await ApiHelper.fetchFromDataSource(query.dataSource, query.queryOptions || {});
} else {
const queryUrl = ApiHelper.buildQueryFromSection(query, userId, serverUrl, sectionConfig.renderMode === 'Spotlight', { sectionType: sectionConfig.type });
if (typeof queryUrl !== 'string') {
WARN(`Invalid query URL for section ${sectionConfig.id}`);
continue;
}
const cacheCfg = Config().CACHE || {};
let sectionTtl = cacheCfg.DEFAULT_TTL;
if (Number(sectionConfig.ttl) >= 0) {
sectionTtl = Number(sectionConfig.ttl);
}
queryResult = ApiHelper.getQuery(queryUrl, {
useCache: true,
ttl: sectionTtl
});
}
if (queryResult) results.push(queryResult);
}
if (!results.length) {
return [];
}
if (results.length > 1) {
const merged = ApiHelper.mergeMultiQueryResults(results, sectionConfig);
const raw = typeof merged.result?.ensureData === 'function'
? await merged.result.ensureData()
: await merged.result?.dataPromise;
return postProcess(sectionConfig, raw);
}
const queryResult = results[0];
const raw = typeof queryResult.ensureData === 'function'
? await queryResult.ensureData()
: await queryResult.dataPromise;
return postProcess(sectionConfig, raw);
})();
}
return mappedDataPromise;
};
const result = {
data: postProcess(sectionConfig, queryResult.data),
isStale: queryResult.isStale === true,
isStalePromise: queryResult.isStalePromise,
data: { Items: [] },
isStale: true,
ensureData
};
Object.defineProperty(result, 'dataPromise', {
@@ -298,7 +326,9 @@
}
});
// Single query result
const loadSectionTimerEnd = performance.now();
LOG(`Section ${sectionConfig.id} progressive stub in ${(loadSectionTimerEnd - loadSectionTimerStart).toFixed(2)}ms`);
return {
config: sectionConfig,
queryUrl: null,
@@ -582,15 +612,17 @@
if (!result) return [];
if (Array.isArray(result)) return result;
if (Array.isArray(result.Items)) return result.Items;
if (Array.isArray(result.data?.Items)) return result.data.Items;
if (typeof result.ensureData === 'function') {
const fresh = await result.ensureData();
if (Array.isArray(fresh)) return fresh;
return fresh?.Items || [];
}
if (result.dataPromise) {
const fresh = await result.dataPromise;
if (Array.isArray(fresh)) return fresh;
return fresh?.Items || [];
}
if (Array.isArray(result.data?.Items)) return result.data.Items;
return [];
}
@@ -602,10 +634,10 @@
return [];
}
if (window.LibraryCache?.sectionDependsOnIncompleteCache?.({
if ((await window.LibraryCache?.sectionDependsOnIncompleteCache?.({
discoverySourceQuery: sourceQuery,
queries: []
}) === true) {
})) === true) {
LOG(`Discovery section ${config.id}: skipping pool fetch — incomplete library cache`);
return [];
}
@@ -622,8 +654,9 @@
}
if (typeof built !== 'string') return [];
const result = await ApiHelper.getQuery(built, { useCache: true, ttl });
return unwrapQueryItems(result);
const stub = ApiHelper.getQuery(built, { useCache: true, ttl });
const raw = typeof stub.ensureData === 'function' ? await stub.ensureData() : await stub;
return unwrapQueryItems(raw);
}
function shuffleArray(arr) {
@@ -833,8 +866,8 @@
* @param {number} [options.stubOrder] - Stable order from progressive stub
*/
async function buildDiscoverySectionInstance(template, options = {}) {
if (window.LibraryCache?.sectionDependsOnIncompleteCache?.(template) === true) {
LOG(`Skipping discovery instance ${template?.id}: depends on incomplete library cache`);
if (!(await isSectionCacheReady(template))) {
LOG(`Skipping discovery ${template?.id}: dependent library cache not complete`);
return null;
}
@@ -1052,11 +1085,9 @@
* Progressive discovery: return a stub config + lazy-loading result immediately.
* Concrete resolve + load runs on first ensureData / dataPromise / isStalePromise access.
*/
function buildDiscoverySectionPromise(template, options = {}) {
if (window.LibraryCache?.sectionDependsOnIncompleteCache?.(template) === true) {
LOG(`Skipping discovery stub ${template?.id}: depends on incomplete library cache`);
return null;
}
async function buildDiscoverySectionPromise(template, options = {}) {
// Incomplete library cache: still return a progressive stub; loadConcrete /
// loadSectionForRendering ensureData waits until caches are ready.
const pairSpotlight = options.pairSpotlight !== false;
const order = options.order;