From e4ddc9e645324b4825d4fb40fd29eeda11d76f8a Mon Sep 17 00:00:00 2001 From: HighImKevin Date: Fri, 25 Sep 2026 05:03:31 -0400 Subject: [PATCH] Improve loading from sections with local caches --- helpers/dataHelper.js | 10 +- scripts/apiHelper.js | 59 ++++++++--- scripts/libraryCache.js | 20 ++-- scripts/libraryCacheUtils.js | 1 + scripts/moviesCache.js | 45 ++++++-- scripts/sectionHelper.js | 193 ++++++++++++++++++++--------------- 6 files changed, 213 insertions(+), 115 deletions(-) diff --git a/helpers/dataHelper.js b/helpers/dataHelper.js index 54b3821..dea5cbe 100644 --- a/helpers/dataHelper.js +++ b/helpers/dataHelper.js @@ -46,11 +46,11 @@ async function unwrapGetQueryResult(result) { if (!result) return null; - if (typeof result.ensureData === 'function' || result.dataPromise != null) { - if (result.data != null) return result.data; - return await (typeof result.ensureData === 'function' - ? result.ensureData() - : result.dataPromise); + if (typeof result.ensureData === 'function') { + return await result.ensureData(); + } + if (result.dataPromise != null) { + return await result.dataPromise; } return result; } diff --git a/scripts/apiHelper.js b/scripts/apiHelper.js index 7213e65..9017e93 100644 --- a/scripts/apiHelper.js +++ b/scripts/apiHelper.js @@ -385,7 +385,7 @@ * @param {Object} options - Cache options * @param {boolean} options.useCache - Whether to return cached data first (default: false) * @param {number} options.ttl - Cache time to live in milliseconds (default: 300000) - * @returns {Promise|Object} - If useCache is true, returns { data, dataPromise, isStalePromise }. Otherwise returns Promise. + * @returns {Object|Promise} - If useCache is true, returns progressive stub { data, ensureData, isStalePromise } synchronously. Otherwise returns Promise. */ getQuery: function(url, options = {}) { ensureApiClient(); @@ -616,14 +616,49 @@ return result.ensureData(); } }); - - return (async () => { - // Initial check for cached data - const entry = await cachePromise; - result.data = entry ? entry.payload : null; - result.isStale = forceRefresh || !entry || (Date.now() - entry.timestamp > ttl); - return result; - })(); + + // Sync stub: do not await IndexedDB before return (home first paint). + // cachePromise / isStalePromise hydrate result.data in the background. + // Thenable waits for hydrate but must NOT fulfill with `result` itself + // (Promise assimilation would recurse on result.then forever). + result.data = { Items: [] }; + result.isStale = true; + const readyPromise = cachePromise.then((entry) => { + if (entry) { + result.data = entry.payload; + result.isStale = forceRefresh || (Date.now() - entry.timestamp > ttl); + } else { + // Cache miss: clear placeholder so data?.Items cannot fake a hit + result.data = null; + result.isStale = true; + } + }).catch(() => { + result.data = null; + result.isStale = true; + }); + + function toSettledView() { + return { + data: result.data, + isStale: result.isStale, + isStalePromise: result.isStalePromise, + ensureData: result.ensureData, + get dataPromise() { + return result.ensureData(); + } + }; + } + + result.then = function (onFulfilled, onRejected) { + return readyPromise.then( + () => (typeof onFulfilled === 'function' ? onFulfilled(toSettledView()) : toSettledView()), + onRejected + ); + }; + result.catch = function (onRejected) { + return result.then(undefined, onRejected); + }; + return result; }, invalidateCache: function(url) { @@ -721,7 +756,7 @@ throw new Error('Invalid watchlist query URL'); } - const queryResult = await this.getQuery(url, { + const queryResult = this.getQuery(url, { useCache: true, ttl, forceRefresh @@ -744,8 +779,8 @@ }; const result = { - data: normalize(queryResult.data), - isStale: queryResult.isStale === true, + data: { Items: [] }, + isStale: true, isStalePromise: queryResult.isStalePromise || Promise.resolve(false), ensureData, urls: [url] diff --git a/scripts/libraryCache.js b/scripts/libraryCache.js index 7fe0d23..a6d2f5b 100644 --- a/scripts/libraryCache.js +++ b/scripts/libraryCache.js @@ -12,14 +12,14 @@ return type; } - function isComplete(type) { + async function isComplete(type) { const t = normalizeType(type); - if (t === 'Movie') return window.MoviesCache?.isComplete?.() === true; + if (t === 'Movie') return (await window.MoviesCache?.isComplete?.()) === true; if (t === 'Series') return window.SeriesCache?.isComplete?.() === true; return false; } - function isReady(type) { + async function isReady(type) { return isComplete(type); } @@ -38,11 +38,11 @@ /** * Whether a dataSource string depends on an incomplete Movies/Series/People cache. */ - function dataSourceDependsOnIncompleteCache(dataSource) { + async function dataSourceDependsOnIncompleteCache(dataSource) { const ds = typeof dataSource === 'string' ? dataSource : ''; if (!ds) return false; - if (ds.startsWith('MoviesCache.') && !isComplete('Movie')) return true; - if (ds.startsWith('SeriesCache.') && !isComplete('Series')) return true; + if (ds.startsWith('MoviesCache.') && !(await isComplete('Movie'))) return true; + if (ds.startsWith('SeriesCache.') && !(await isComplete('Series'))) return true; if (ds.startsWith('PeopleCache.') && window.PeopleCache?.isComplete && !window.PeopleCache.isComplete()) { return true; } @@ -52,17 +52,17 @@ /** * Whether a home section depends on a library/people/external cache that is not ready. */ - function sectionDependsOnIncompleteCache(section) { + async function sectionDependsOnIncompleteCache(section) { if (!section) return false; if (Array.isArray(section.externalListUrls) && section.externalListUrls.length) { - return !(isComplete('Movie') && isComplete('Series')); + return !((await isComplete('Movie')) && (await isComplete('Series'))); } - if (dataSourceDependsOnIncompleteCache(section.discoverySourceQuery?.dataSource)) { + if (await dataSourceDependsOnIncompleteCache(section.discoverySourceQuery?.dataSource)) { return true; } const queries = section.queries || []; for (const q of queries) { - if (dataSourceDependsOnIncompleteCache(q?.dataSource)) return true; + if (await dataSourceDependsOnIncompleteCache(q?.dataSource)) return true; } return false; } diff --git a/scripts/libraryCacheUtils.js b/scripts/libraryCacheUtils.js index a38c73e..3cc2276 100644 --- a/scripts/libraryCacheUtils.js +++ b/scripts/libraryCacheUtils.js @@ -76,6 +76,7 @@ Chapters: Array.isArray(item.Chapters) ? item.Chapters.map(stripChapter).filter(Boolean) : [], Trickplay: item.Trickplay || undefined, DateCreated: item.DateCreated, + DateLastMediaAdded: item.DateLastMediaAdded, DateLastSaved: item.DateLastSaved, UserData: item.UserData || {} }; diff --git a/scripts/moviesCache.js b/scripts/moviesCache.js index f88ad7b..ee77fce 100644 --- a/scripts/moviesCache.js +++ b/scripts/moviesCache.js @@ -13,7 +13,7 @@ const IMDB_TOP_250_CACHE_TTL = 24 * 60 * 60 * 1000; const MOVIE_FIELDS = [ 'ProviderIds', 'People', 'Studios', 'Taglines', 'Genres', 'Overview', 'PrimaryImageAspectRatio', - 'DateCreated' + 'DateCreated', 'DateLastMediaAdded' ].join(','); let movies = null; @@ -55,6 +55,24 @@ } } + /** Max DateLastMediaAdded || DateCreated among movies; null if none usable. */ + function watermarkFromMovies(list) { + let best = null; + for (const m of list || []) { + const raw = m?.DateLastMediaAdded || m?.DateCreated; + if (!raw) continue; + const t = new Date(raw).getTime(); + if (!Number.isFinite(t)) continue; + if (best == null || t > best) best = t; + } + return best != null ? new Date(best).toISOString() : null; + } + + function setLastDateFetchedFromMovies(list) { + const iso = watermarkFromMovies(list); + if (iso) setLastDateFetched(iso); + } + function getApiHelper() { return window.apiHelper || window.ApiHelper; } @@ -158,7 +176,7 @@ rebuildProviderIndex(movies); await cache.set(CACHE_NAME, movies, userId, getTtl()); await cache.clear(PARTIAL_CACHE_NAME, userId); - setLastDateFetched(new Date().toISOString()); + setLastDateFetchedFromMovies(movies); LOG(`Movie crawl complete: ${movies.length} items`); return movies; } catch (err) { @@ -286,7 +304,7 @@ rebuildProviderIndex(movies); await cache.set(CACHE_NAME, movies, userId, getTtl()); - setLastDateFetched(new Date().toISOString()); + setLastDateFetchedFromMovies(movies); // Notify people cache of deltas when available if ((incoming.length || removedIds.length) && window.PeopleCache?.applyMovieDelta) { try { @@ -314,8 +332,21 @@ return fetchPromise; } - function isCacheComplete() { - return isComplete === true && Array.isArray(movies); + async function isCacheComplete() { + if (isComplete === true && Array.isArray(movies)) return true; + const cache = window.IndexedDBCache; + const userId = window.ApiClient?.getCurrentUserId?.(); + if (!cache || !userId) return false; + const complete = await cache.get(CACHE_NAME, userId); + if (complete && Array.isArray(complete)) { + if (!movies) { + movies = complete; + rebuildProviderIndex(movies); + } + isComplete = true; + return true; + } + return false; } async function getMoviesByProviderIds(ids) { @@ -372,7 +403,7 @@ if (cached) return cached; if (!imdbTop250Movies) { - if (!isCacheComplete()) { + if (!(await isCacheComplete())) { LOG('IMDb Top 250 waiting for movie cache…'); } const imdbIds = await fetchImdbTop250Data(); @@ -389,7 +420,7 @@ async function initialize() { await getMovies(); - if (isCacheComplete() && getLastDateFetched()) { + if ((await isCacheComplete()) && getLastDateFetched()) { await syncMoviesCache(); } return movies || []; diff --git a/scripts/sectionHelper.js b/scripts/sectionHelper.js index 05c7241..ba84228 100644 --- a/scripts/sectionHelper.js +++ b/scripts/sectionHelper.js @@ -56,29 +56,39 @@ discoveryState.pairResolvePromises.clear(); } + /** + * One-shot: whether this section's dependent Movies/Series/People cache is already complete. + * Incomplete → skip (do not poll/wait). Missing LibraryCache → treat as ready. + */ + async function isSectionCacheReady(sectionConfig) { + if (!window.LibraryCache?.sectionDependsOnIncompleteCache) return true; + try { + return (await window.LibraryCache.sectionDependsOnIncompleteCache(sectionConfig)) !== true; + } catch (_) { + return true; + } + } + + function scheduleViewMoreUrl(sectionConfig) { + resolveViewMoreUrl(sectionConfig).then((viewMoreUrl) => { + if (viewMoreUrl) sectionConfig.viewMoreUrl = viewMoreUrl; + }).catch((err) => { + WARN(`Failed to resolve viewMoreUrl for ${sectionConfig?.id}:`, err); + }); + } + async function loadSectionForRendering(sectionConfig) { LOG(`Loading Section for Rendering: ${sectionConfig.id} (${sectionConfig.name})`); const loadSectionTimerStart = performance.now(); - - // Omit sections that would wait on incomplete Movies/Series/People crawls - // (static-item sections never depend on those caches) const hasStaticItems = sectionConfig.items && sectionConfig.items.length > 0; - if (!hasStaticItems && window.LibraryCache?.sectionDependsOnIncompleteCache?.(sectionConfig) === true) { - LOG(`Skipping section ${sectionConfig.id}: depends on incomplete library cache`); - return null; - } - // Prepare section (template replacements, view more URL, etc.) - const viewMoreUrl = await resolveViewMoreUrl(sectionConfig); + // Sync prep only (viewMore deferred for non-static so it does not block first paint) const cardFormat = resolveCardFormat(sectionConfig); const sectionName = fillTemplate(sectionConfig.name, sectionConfig.metadata); sectionConfig.overflowCard = true; - if (viewMoreUrl) { - sectionConfig.viewMoreUrl = viewMoreUrl; - } if (cardFormat) { sectionConfig.cardFormat = cardFormat; } @@ -102,6 +112,11 @@ // Static items: normalize and return immediately (no queries, no dataPromise) if (hasStaticItems) { + const viewMoreUrl = await resolveViewMoreUrl(sectionConfig); + if (viewMoreUrl) { + sectionConfig.viewMoreUrl = viewMoreUrl; + } + const kefinTweaksRoot = window.KefinTweaksConfig?.kefinTweaksRoot || ''; const serverId = ApiClient.serverId(); const normalizeTemplate = (value) => (value || '') @@ -147,6 +162,14 @@ }; } + // Cache-backed sections only mount once Movies/Series/People cache is complete + if (!(await isSectionCacheReady(sectionConfig))) { + LOG(`Skipping section ${sectionConfig.id}: dependent library cache not complete`); + return null; + } + + scheduleViewMoreUrl(sectionConfig); + // External list (MDBList) — progressive for network fetch when caches are already ready if (Array.isArray(sectionConfig.externalListUrls) && sectionConfig.externalListUrls.length > 0) { if (!window.KefinExternalList?.resolveExternalListItems) { @@ -158,6 +181,9 @@ const ensureData = () => { if (!mappedDataPromise) { mappedDataPromise = (async () => { + if (!(await isSectionCacheReady(sectionConfig))) { + return []; + } const listResult = await window.KefinExternalList.resolveExternalListItems(sectionConfig); let items = listResult?.Items || []; if (window.cardBuilder.postProcessItems) { @@ -174,6 +200,7 @@ const result = { data: { Items: [] }, + isStale: true, ensureData }; Object.defineProperty(result, 'dataPromise', { @@ -198,7 +225,6 @@ const userId = ApiClient.getCurrentUserId(); const serverUrl = ApiClient.serverAddress(); - const results = []; const resolveQueries = window.cardBuilder?.resolveQueriesToLoad; const queriesToLoad = typeof resolveQueries === 'function' @@ -220,74 +246,76 @@ LOG(`Section ${sectionConfig.id}: useRandomQuery — selected query index ${index}`); } - // Process each query in the queries array - for (const query of queriesToLoad) { - let queryResult; - - if (query.dataSource) { - // Handle cache-based data sources - queryResult = await ApiHelper.fetchFromDataSource(query.dataSource, query.queryOptions || {}); - } else { - // Build and execute query - const queryUrl = ApiHelper.buildQueryFromSection(query, userId, serverUrl, sectionConfig.renderMode === 'Spotlight', { sectionType: sectionConfig.type }); - - if (typeof queryUrl === 'string') { - // Standard query - const cacheCfg = Config().CACHE || {}; - let sectionTtl = cacheCfg.DEFAULT_TTL; - if (Number(sectionConfig.ttl) >= 0) { - sectionTtl = Number(sectionConfig.ttl); - } - - queryResult = await ApiHelper.getQuery(queryUrl, { - useCache: true, - ttl: sectionTtl - }); - } else { - WARN(`Invalid query URL for section ${sectionConfig.id}`); - continue; - } - } - - results.push(queryResult); - } - - // If multiple queries, merge results using section-level sortBy/sortOrder - if (results.length > 1) { - const loadSectionTimerEnd = performance.now(); - const loadSectionDuration = loadSectionTimerEnd - loadSectionTimerStart; - LOG(`Section ${sectionConfig.id} loaded for rendering in time: ${loadSectionDuration.toFixed(2)}ms`); - return ApiHelper.mergeMultiQueryResults(results, sectionConfig); - } - - const loadSectionTimerEnd = performance.now(); - const loadSectionDuration = loadSectionTimerEnd - loadSectionTimerStart; - LOG(`Section ${sectionConfig.id} loaded for rendering in time: ${loadSectionDuration.toFixed(2)}ms`); - - const postProcess = (sectionConfig, items) => { + const postProcess = (cfg, items) => { let postProcessedItems = items; if (window.cardBuilder.postProcessItems) { - postProcessedItems = window.cardBuilder.postProcessItems(sectionConfig, items); + postProcessedItems = window.cardBuilder.postProcessItems(cfg, items); } - return sortItemsByConfiguredIds(sectionConfig, postProcessedItems); + return sortItemsByConfiguredIds(cfg, postProcessedItems); }; - const queryResult = results[0]; let mappedDataPromise = null; const ensureData = () => { if (!mappedDataPromise) { - const raw = typeof queryResult.ensureData === 'function' - ? queryResult.ensureData() - : queryResult.dataPromise; - mappedDataPromise = Promise.resolve(raw).then((data) => postProcess(sectionConfig, data)); + mappedDataPromise = (async () => { + if (!(await isSectionCacheReady(sectionConfig))) { + return []; + } + + const results = []; + for (const query of queriesToLoad) { + let queryResult; + + if (query.dataSource) { + queryResult = await ApiHelper.fetchFromDataSource(query.dataSource, query.queryOptions || {}); + } else { + const queryUrl = ApiHelper.buildQueryFromSection(query, userId, serverUrl, sectionConfig.renderMode === 'Spotlight', { sectionType: sectionConfig.type }); + + if (typeof queryUrl !== 'string') { + WARN(`Invalid query URL for section ${sectionConfig.id}`); + continue; + } + + const cacheCfg = Config().CACHE || {}; + let sectionTtl = cacheCfg.DEFAULT_TTL; + if (Number(sectionConfig.ttl) >= 0) { + sectionTtl = Number(sectionConfig.ttl); + } + + queryResult = ApiHelper.getQuery(queryUrl, { + useCache: true, + ttl: sectionTtl + }); + } + + if (queryResult) results.push(queryResult); + } + + if (!results.length) { + return []; + } + + if (results.length > 1) { + const merged = ApiHelper.mergeMultiQueryResults(results, sectionConfig); + const raw = typeof merged.result?.ensureData === 'function' + ? await merged.result.ensureData() + : await merged.result?.dataPromise; + return postProcess(sectionConfig, raw); + } + + const queryResult = results[0]; + const raw = typeof queryResult.ensureData === 'function' + ? await queryResult.ensureData() + : await queryResult.dataPromise; + return postProcess(sectionConfig, raw); + })(); } return mappedDataPromise; }; const result = { - data: postProcess(sectionConfig, queryResult.data), - isStale: queryResult.isStale === true, - isStalePromise: queryResult.isStalePromise, + data: { Items: [] }, + isStale: true, ensureData }; Object.defineProperty(result, 'dataPromise', { @@ -298,7 +326,9 @@ } }); - // Single query result + const loadSectionTimerEnd = performance.now(); + LOG(`Section ${sectionConfig.id} progressive stub in ${(loadSectionTimerEnd - loadSectionTimerStart).toFixed(2)}ms`); + return { config: sectionConfig, queryUrl: null, @@ -582,15 +612,17 @@ if (!result) return []; if (Array.isArray(result)) return result; if (Array.isArray(result.Items)) return result.Items; - if (Array.isArray(result.data?.Items)) return result.data.Items; if (typeof result.ensureData === 'function') { const fresh = await result.ensureData(); + if (Array.isArray(fresh)) return fresh; return fresh?.Items || []; } if (result.dataPromise) { const fresh = await result.dataPromise; + if (Array.isArray(fresh)) return fresh; return fresh?.Items || []; } + if (Array.isArray(result.data?.Items)) return result.data.Items; return []; } @@ -602,10 +634,10 @@ return []; } - if (window.LibraryCache?.sectionDependsOnIncompleteCache?.({ + if ((await window.LibraryCache?.sectionDependsOnIncompleteCache?.({ discoverySourceQuery: sourceQuery, queries: [] - }) === true) { + })) === true) { LOG(`Discovery section ${config.id}: skipping pool fetch — incomplete library cache`); return []; } @@ -622,8 +654,9 @@ } if (typeof built !== 'string') return []; - const result = await ApiHelper.getQuery(built, { useCache: true, ttl }); - return unwrapQueryItems(result); + const stub = ApiHelper.getQuery(built, { useCache: true, ttl }); + const raw = typeof stub.ensureData === 'function' ? await stub.ensureData() : await stub; + return unwrapQueryItems(raw); } function shuffleArray(arr) { @@ -833,8 +866,8 @@ * @param {number} [options.stubOrder] - Stable order from progressive stub */ async function buildDiscoverySectionInstance(template, options = {}) { - if (window.LibraryCache?.sectionDependsOnIncompleteCache?.(template) === true) { - LOG(`Skipping discovery instance ${template?.id}: depends on incomplete library cache`); + if (!(await isSectionCacheReady(template))) { + LOG(`Skipping discovery ${template?.id}: dependent library cache not complete`); return null; } @@ -1052,11 +1085,9 @@ * Progressive discovery: return a stub config + lazy-loading result immediately. * Concrete resolve + load runs on first ensureData / dataPromise / isStalePromise access. */ - function buildDiscoverySectionPromise(template, options = {}) { - if (window.LibraryCache?.sectionDependsOnIncompleteCache?.(template) === true) { - LOG(`Skipping discovery stub ${template?.id}: depends on incomplete library cache`); - return null; - } + async function buildDiscoverySectionPromise(template, options = {}) { + // Incomplete library cache: still return a progressive stub; loadConcrete / + // loadSectionForRendering ensureData waits until caches are ready. const pairSpotlight = options.pairSpotlight !== false; const order = options.order;