feat: add wiki.gg as a second Endfield source
First non-Game8 parser, and the first source that states region-scoped ends: wiki.gg emits ISO timestamps with one timer per server region, so Endfield events now carry exact times and separate Asia / Americas-Europe ends. That gap is up to 13 hours, which is what regionEnds was built for. Two sources for one game also exercised merge for real. It caught both overlaps — including "Bedazzling Dawnstar" against Game8's "Bedazzling Dawnstar Sign-In" — and flagged that the two disagree on the end date by 70 hours rather than averaging them into a date neither states. Matching an appended qualifier needed a prefix test, not a subset test: a subset would also fuse "Gold Clash" with "Gold Rush Clash Royale". Also fixes build-feed picking fixtures by game rather than by source, which handed the wiki.gg page to the Game8 parser once Endfield had two. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5
Claude Opus 5
parent
f4b133274a
commit
dd596b5a43
@@ -50,6 +50,15 @@ const SOURCES: SourceSpec[] = [
|
||||
url: "https://game8.co/games/Arknights-Endfield/archives/535443",
|
||||
parserId: "game8",
|
||||
},
|
||||
{
|
||||
id: "endfield-wikigg-events",
|
||||
game: "endfield",
|
||||
url: "https://endfield.wiki.gg/wiki/Event",
|
||||
parserId: "wikigg",
|
||||
// Exact timestamps and per-region ends beat Game8's day-precision prose,
|
||||
// so this source wins when the two disagree.
|
||||
priority: 10,
|
||||
},
|
||||
{
|
||||
id: "nte-game8-events",
|
||||
game: "nte",
|
||||
|
||||
+34
-8
@@ -123,12 +123,36 @@ function isSameEvent(
|
||||
toleranceHours: number,
|
||||
): boolean {
|
||||
if (a.id === b.id) return true;
|
||||
if (titleSimilarity(a.title, b.title) < titleThreshold) return false;
|
||||
|
||||
// Overlap alone misses a source that appends a qualifier: "Bedazzling
|
||||
// Dawnstar" vs "Bedazzling Dawnstar Sign-In" scores 0.67, well under any
|
||||
// safe threshold, yet is plainly one event.
|
||||
const similar =
|
||||
titleSimilarity(a.title, b.title) >= titleThreshold ||
|
||||
titleExtends(a.title, b.title);
|
||||
if (!similar) return false;
|
||||
|
||||
// Similar titles are not enough — reruns reuse names. Require the start dates
|
||||
// to be close before treating two entries as one event.
|
||||
return hoursBetween(a.startsAt, b.startsAt) <= toleranceHours;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when one title is the other with words appended — the shape a source
|
||||
* qualifier actually takes ("Bedazzling Dawnstar" → "… Sign-In").
|
||||
*
|
||||
* Deliberately a prefix test, not a subset test. Subset matching would also
|
||||
* fuse "Gold Clash" with "Gold Rush Clash Royale", which are different events.
|
||||
* Two words is the floor: a one-word title would swallow half the calendar.
|
||||
*/
|
||||
export function titleExtends(a: string, b: string): boolean {
|
||||
const ta = tokenList(a);
|
||||
const tb = tokenList(b);
|
||||
const [small, large] = ta.length <= tb.length ? [ta, tb] : [tb, ta];
|
||||
if (small.length < 2 || small.length === large.length) return false;
|
||||
return small.every((word, i) => large[i] === word);
|
||||
}
|
||||
|
||||
function findConflict(
|
||||
a: GachaEvent,
|
||||
b: GachaEvent,
|
||||
@@ -163,12 +187,14 @@ export function titleSimilarity(a: string, b: string): number {
|
||||
return (2 * shared) / (ta.size + tb.size);
|
||||
}
|
||||
|
||||
function tokenList(title: string): string[] {
|
||||
return title
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s]/g, " ")
|
||||
.split(/\s+/)
|
||||
.filter((w) => w.length > 0);
|
||||
}
|
||||
|
||||
function tokens(title: string): Set<string> {
|
||||
return new Set(
|
||||
title
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s]/g, " ")
|
||||
.split(/\s+/)
|
||||
.filter((w) => w.length > 0),
|
||||
);
|
||||
return new Set(tokenList(title));
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { game8Parser } from "./game8.ts";
|
||||
import { wikiGgParser } from "./wikigg.ts";
|
||||
import type { SourceParser } from "./types.ts";
|
||||
|
||||
/**
|
||||
@@ -6,7 +7,7 @@ import type { SourceParser } from "./types.ts";
|
||||
* an entry in `adapters/index.ts`; adding a new *site* means a parser module
|
||||
* here and one line below.
|
||||
*/
|
||||
export const PARSERS: SourceParser[] = [game8Parser];
|
||||
export const PARSERS: SourceParser[] = [game8Parser, wikiGgParser];
|
||||
|
||||
export function parserById(id: string): SourceParser | undefined {
|
||||
return PARSERS.find((p) => p.id === id);
|
||||
@@ -14,3 +15,4 @@ export function parserById(id: string): SourceParser | undefined {
|
||||
|
||||
export type { SourceParser } from "./types.ts";
|
||||
export { game8Parser } from "./game8.ts";
|
||||
export { wikiGgParser } from "./wikigg.ts";
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
import {
|
||||
eventId,
|
||||
Region,
|
||||
type GachaEvent,
|
||||
type Precision,
|
||||
} from "../../shared/schema.ts";
|
||||
import { text } from "../html.ts";
|
||||
import type { ParseContext } from "../adapters/types.ts";
|
||||
import { inferType } from "./game8.ts";
|
||||
import type { SourceParser } from "./types.ts";
|
||||
|
||||
/**
|
||||
* wiki.gg event pages (MediaWiki with the `mp-event` template).
|
||||
*
|
||||
* A markedly better source than a prose wiki: each event carries machine
|
||||
* readable ISO timestamps, one timer per server region:
|
||||
*
|
||||
* <div class="mp-event">
|
||||
* <div class="mp-event-header">[<span class="mp-event-name">TITLE</span>]
|
||||
* <span class="mp-event-type">Challenge Event</span></div>
|
||||
* <div class="mp-event-image"><a href="/wiki/SLUG" …></div>
|
||||
* <ul class="mp-event-timers">
|
||||
* <li data-start="…Z" data-end="…Z"><span class="mp-server-name">Asia:</span>…
|
||||
*
|
||||
* This is the first source that states region-scoped ends, which is what
|
||||
* `regionScoped` / `regionEnds` exist for — the difference is up to 13 hours.
|
||||
*/
|
||||
|
||||
const EVENT_BLOCK = /<div class="mp-event">([\s\S]*?)<ul class="mp-event-timers">([\s\S]*?)<\/ul>/gi;
|
||||
const NAME = /<span class="mp-event-name">([\s\S]*?)<\/span>/i;
|
||||
const TYPE = /<span class="mp-event-type">([\s\S]*?)<\/span>/i;
|
||||
const LINK = /<a href="(\/wiki\/[^"]+)"/i;
|
||||
const TIMER =
|
||||
/<li[^>]*class="[^"]*mp-event-timer[^"]*"[^>]*data-start="([^"]+)"[^>]*data-end="([^"]+)"[\s\S]*?<span class="[^"]*mp-server-name[^"]*">([\s\S]*?)<\/span>/gi;
|
||||
|
||||
/**
|
||||
* Server labels vary in wording ("Americas / Europe:", "Asia:"), so match on
|
||||
* the words present rather than expecting one label per region. A single timer
|
||||
* can legitimately cover two regions.
|
||||
*/
|
||||
function regionsFor(label: string): Region[] {
|
||||
const l = label.toLowerCase();
|
||||
const out: Region[] = [];
|
||||
if (/asia/.test(l)) out.push("asia");
|
||||
if (/america/.test(l)) out.push("america");
|
||||
if (/europe|eu\b/.test(l)) out.push("europe");
|
||||
return out;
|
||||
}
|
||||
|
||||
interface Timer {
|
||||
regions: Region[];
|
||||
start: number;
|
||||
end: number;
|
||||
}
|
||||
|
||||
export function parseWikiGgEventsPage(
|
||||
html: string,
|
||||
ctx: ParseContext,
|
||||
): GachaEvent[] {
|
||||
const flat = html.replace(/\s+/g, " ");
|
||||
const out: GachaEvent[] = [];
|
||||
|
||||
for (const block of flat.matchAll(EVENT_BLOCK)) {
|
||||
const head = block[1] ?? "";
|
||||
const timersHtml = block[2] ?? "";
|
||||
|
||||
const title = text(NAME.exec(head)?.[1] ?? "");
|
||||
if (title.length === 0) continue;
|
||||
|
||||
const timers: Timer[] = [];
|
||||
for (const t of timersHtml.matchAll(TIMER)) {
|
||||
const start = Date.parse(t[1] ?? "");
|
||||
const end = Date.parse(t[2] ?? "");
|
||||
if (Number.isNaN(start) || Number.isNaN(end) || end <= start) continue;
|
||||
timers.push({ regions: regionsFor(text(t[3] ?? "")), start, end });
|
||||
}
|
||||
if (timers.length === 0) continue;
|
||||
|
||||
const regionEnds: Partial<Record<Region, string>> = {};
|
||||
for (const timer of timers) {
|
||||
for (const region of timer.regions) {
|
||||
regionEnds[region] = new Date(timer.end).toISOString();
|
||||
}
|
||||
}
|
||||
|
||||
const covered = Object.keys(regionEnds).length;
|
||||
// Region-scoped only when the source actually distinguishes regions and
|
||||
// they genuinely differ. One timer for everyone is a global instant.
|
||||
const distinctEnds = new Set(Object.values(regionEnds)).size;
|
||||
const regionScoped = covered > 0 && distinctEnds > 1;
|
||||
|
||||
// Canonical start is the earliest any region sees it; the canonical end is
|
||||
// the earliest any region loses it. `endsAt` is only a fallback for a
|
||||
// region the source did not list, so the conservative choice is right.
|
||||
const startsAt = new Date(
|
||||
Math.min(...timers.map((t) => t.start)),
|
||||
).toISOString();
|
||||
const endsAt = new Date(Math.min(...timers.map((t) => t.end))).toISOString();
|
||||
|
||||
const href = LINK.exec(block[0] ?? "")?.[1];
|
||||
const typeLabel = text(TYPE.exec(head)?.[1] ?? "");
|
||||
const precision: Precision = "exact";
|
||||
|
||||
out.push({
|
||||
id: eventId(ctx.game, title, startsAt),
|
||||
game: ctx.game,
|
||||
title,
|
||||
type: inferType(`${title} ${typeLabel}`),
|
||||
summary: typeLabel.length > 0 ? typeLabel : null,
|
||||
startsAt,
|
||||
startPrecision: precision,
|
||||
endsAt,
|
||||
endPrecision: precision,
|
||||
regionScoped,
|
||||
regionEnds: regionScoped ? (regionEnds as Record<Region, string>) : null,
|
||||
sourceUrl: href === undefined ? ctx.sourceUrl : new URL(href, ctx.sourceUrl).toString(),
|
||||
sourceId: ctx.sourceId,
|
||||
status: "published",
|
||||
// Exact timestamps straight from the source, no inference anywhere.
|
||||
confidence: 0.95,
|
||||
extractionMethod: "parser",
|
||||
version: 1,
|
||||
firstSeenAt: ctx.now,
|
||||
updatedAt: ctx.now,
|
||||
});
|
||||
}
|
||||
|
||||
return out.sort((a, b) =>
|
||||
a.startsAt === b.startsAt
|
||||
? a.id.localeCompare(b.id)
|
||||
: a.startsAt.localeCompare(b.startsAt),
|
||||
);
|
||||
}
|
||||
|
||||
export const wikiGgParser: SourceParser = {
|
||||
id: "wikigg",
|
||||
label: "wiki.gg",
|
||||
canParse(html: string): boolean {
|
||||
return /class="mp-event"/.test(html) && /mp-event-timer/.test(html);
|
||||
},
|
||||
parse: parseWikiGgEventsPage,
|
||||
};
|
||||
Reference in New Issue
Block a user