From 715e36713722cc44678fd8c9ae53a248ceb017ac Mon Sep 17 00:00:00 2001 From: Katalina Okano Date: Tue, 20 Dec 2022 18:56:19 -0500 Subject: [PATCH] rebuild caching layer --- src/cache.ts | 30 ++++++----- src/handlers.ts | 77 ++++++++++++++++++---------- src/index.ts | 18 +++++-- src/sources/fisu.ts | 111 ++++++++++++++++++++++++++-------------- src/sources/honu.ts | 37 ++++++++++++-- src/sources/saerro.ts | 47 +++++++++++++---- src/sources/voidwell.ts | 78 ++++++++++++++++++++-------- src/types.ts | 11 ++++ wrangler.toml | 4 ++ 9 files changed, 295 insertions(+), 118 deletions(-) diff --git a/src/cache.ts b/src/cache.ts index a13498e..1262ee6 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -1,26 +1,32 @@ -import { DebugPayload, OnePayload } from "./types"; - -type WorldObject = { - world: OnePayload | null; - debug: DebugPayload; -}; - -export class WorldCache { +export class Cache { + private cache: Map = new Map(); constructor(public kv: KVNamespace, public disableCache: boolean = false) {} - async get(id: string): Promise { + async get(id: string): Promise { if (this.disableCache) { return null; } - const world = await this.kv.get(id, "json"); - return world; + + // console.log("cache get", id); + let item = this.cache.get(id); + if (!item) { + // console.log("remote cache get", id); + item = await this.kv.get(id, "json"); + if (item) { + // console.log("local cache miss, remote cache hit", id); + this.cache.set(id, item); + } + } + return item; } - async put(id: string, world: WorldObject): Promise { + async put(id: string, world: T): Promise { if (this.disableCache) { return world; } + this.cache.set(id, world); + await this.kv.put(id, JSON.stringify(world), { expirationTtl: 60 * 3, }); diff --git a/src/handlers.ts b/src/handlers.ts index ee780da..29c1fa3 100644 --- a/src/handlers.ts +++ b/src/handlers.ts @@ -4,8 +4,8 @@ import { fisuFetchWorld } from "./sources/fisu"; import { honuFetchWorld } from "./sources/honu"; import { voidwellFetchWorld } from "./sources/voidwell"; import { noData } from "./errors"; -import { DebugPayload, Flags, OnePayload } from "./types"; -import { WorldCache } from "./cache"; +import { DebugPayload, Flags, OnePayload, ServiceResponse } from "./types"; +import { Cache } from "./cache"; const avgOf = (arr: number[]) => Math.floor(arr.reduce((a, b) => a + b, 0) / arr.length); @@ -13,35 +13,43 @@ const avgOf = (arr: number[]) => const flatMapBy = (arr: any[], key: string) => arr.reduce((a, b) => [...a, b[key]], []); -const defaultServiceResponse = { +const defaultServiceResponse: ServiceResponse = { population: { total: -1, - nc: null, - tr: null, - vs: null, + nc: -1, + tr: -1, + vs: -1, }, raw: null, - cachedAt: undefined, + cachedAt: new Date(), }; -export const getWorld = async (id: string, cache: WorldCache, flags: Flags) => { - const cached = await cache.get(id); +type World = { + world: OnePayload | null; + debug: DebugPayload; +}; + +export const getWorld = async (id: string, cache: Cache, flags: Flags) => { + const cached = await cache.get(id); if (cached) { return cached; } const [saerro, fisu, honu, voidwell] = await Promise.all([ !flags.disableSaerro - ? saerroFetchWorld(id).catch(() => defaultServiceResponse) + ? saerroFetchWorld(id, cache).catch((e) => { + console.error("SAERRO ERROR:", e); + return defaultServiceResponse; + }) : defaultServiceResponse, !flags.disableFisu - ? fisuFetchWorld(id).catch(() => defaultServiceResponse) + ? fisuFetchWorld(id, cache).catch(() => defaultServiceResponse) : defaultServiceResponse, !flags.disableHonu - ? honuFetchWorld(id).catch(() => defaultServiceResponse) + ? honuFetchWorld(id, cache).catch(() => defaultServiceResponse) : defaultServiceResponse, !flags.disableVoidwell - ? voidwellFetchWorld(id).catch(() => defaultServiceResponse) + ? voidwellFetchWorld(id, cache).catch(() => defaultServiceResponse) : defaultServiceResponse, ]); @@ -52,6 +60,12 @@ export const getWorld = async (id: string, cache: WorldCache, flags: Flags) => { honu: honu.raw, voidwell: voidwell.raw, }, + timings: { + saerro: saerro?.timings || null, + fisu: fisu?.timings || null, + honu: honu?.timings || null, + voidwell: voidwell?.timings || null, + }, lastFetchTimes: { saerro: saerro.cachedAt, fisu: fisu.cachedAt, @@ -68,7 +82,7 @@ export const getWorld = async (id: string, cache: WorldCache, flags: Flags) => { ].filter((x) => x > 0); if (totalPopulations.length === 0) { - return await cache.put(id, { + return await cache.put(id, { world: id !== "19" ? null @@ -121,10 +135,10 @@ export const handleOne = async ( { params: { id }, query: { debug: debugParam } }: IRequest, _1: unknown, _2: unknown, - worldCache: WorldCache, + Cache: Cache, flags: Flags ) => { - const { world, debug } = await getWorld(id, worldCache, flags); + const { world, debug } = await getWorld(id, Cache, flags); if (world === null) { return noData(); @@ -144,22 +158,29 @@ export const handleOne = async ( }; export const handleAll = async ( - _1: unknown, + { query: { debug } }: IRequest, _2: unknown, _3: unknown, - worldCache: WorldCache, + Cache: Cache, flags: Flags ): Promise => { const worlds = ["1", "10", "13", "17", "19", "40", "1000", "2000"]; - const worldData = await Promise.all( - worlds.map((x) => - getWorld(x, worldCache, flags).catch(() => { - error: "World data is missing. Is it down?"; - }) - ) - ); - const worldPayloads = worldData.map((x) => x?.world || x); + const worldData = []; + + for (const world of worlds) { + worldData.push(await getWorld(world, Cache, flags)); + } + + if (debug === "1") { + return new Response(JSON.stringify(worldData), { + headers: { + "content-type": "application/json", + }, + }); + } + + const worldPayloads = worldData.map((x: any) => x.world || x); return new Response(JSON.stringify(worldPayloads), { headers: { @@ -258,9 +279,11 @@ GET /all - Get all worlds } ] + -- This also has a debug query parameter, but it's extremely verbose. It's good for debugging extreme async issues with the platform. + ## Caching and usage limits -This service cached on a world basis for 3 minutes.`; +This service cached on a world basis for 3 minutes. Debug data is cached alongside world data too.`; return new Response(body, { headers: { diff --git a/src/index.ts b/src/index.ts index 1a8b9b2..182d297 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,7 @@ import { Route, Router, RouterType } from "itty-router"; import { handleAll, handleOne, index } from "./handlers"; import { Env, Flags } from "./types"; -import { WorldCache } from "./cache"; +import { Cache } from "./cache"; interface BasicRouter extends RouterType { all: Route; @@ -11,9 +11,14 @@ interface BasicRouter extends RouterType { const router = Router(); router - .get("/", index) - .get("/all", handleAll) - .get("/:id", handleOne) + .get( + "/", + () => + new Response(null, { status: 303, headers: { location: "/population/" } }) + ) + .get("/population/", index) + .get("/population/all", handleAll) + .get("/population/:id", handleOne) .all("*", () => { return new Response("Not found", { headers: { "content-type": "text/plain" }, @@ -22,7 +27,7 @@ router export default { fetch: async (request: Request, env: Env, ctx: ExecutionContext) => { - const worldCache = new WorldCache(env.CACHE, env.DISABLE_CACHE === "1"); + const worldCache = new Cache(env.CACHE, env.DISABLE_CACHE === "1"); const flags: Flags = { disableFisu: env.DISABLE_FISU === "1", @@ -31,6 +36,8 @@ export default { disableVoidwell: env.DISABLE_VOIDWELL === "1", }; + const start = Date.now(); + return router .handle(request as any, env, ctx, worldCache, flags) .then((response) => { @@ -39,6 +46,7 @@ export default { "access-control-allow-method", "GET, HEAD, OPTIONS" ); + response.headers.set("x-timing", `${Date.now() - start}ms`); return response; }); }, diff --git a/src/sources/fisu.ts b/src/sources/fisu.ts index 9f4f1b3..ab3289d 100644 --- a/src/sources/fisu.ts +++ b/src/sources/fisu.ts @@ -1,50 +1,80 @@ -import { Population, ServiceResponse } from "../types"; - -const subdomain = (worldID: string) => { - switch (worldID) { - case "1000": - return "ps4us.ps2"; - case "2000": - return "ps4eu.ps2"; - default: - return "ps2"; - } -}; +import { Cache } from "../cache"; +import { ServiceResponse } from "../types"; interface FisuResponse { - config: { - world: string[]; - }; - result: { - worldId: number; - vs: number; - nc: number; - tr: number; - ns: number; - }[]; - timing: { - "start-ms": number; - "query-ms": number; - "total-ms": number; - "process-ms": number; - }; + result: Record< + string, + { + worldId: number; + vs: number; + nc: number; + tr: number; + ns: number; + }[] + >; } +const fisuFetchAllWorlds = async (cache: Cache): Promise => { + const cached = await cache.get("fisu"); + if (cached) { + // console.log("FISU data cached", cached); + return cached; + } + + const [pc, ps4us, ps4eu] = await Promise.all([ + fetch(`https://ps2.fisu.pw/api/population/?world=1,10,13,17,19,40`) + .then((res) => res.json()) + .catch((e) => { + console.error("FISU PC ERROR", e); + return { result: {} } as FisuResponse; + }), + fetch(`https://ps4us.ps2.fisu.pw/api/population/?world=1000`) + .then((res) => res.json()) + .catch((e) => { + console.error("FISU PS4US ERROR", e); + return { result: {} } as FisuResponse; + }), + fetch(`https://ps4eu.ps2.fisu.pw/api/population/?world=2000`) + .then((res) => res.json()) + .catch((e) => { + console.error("FISU PS4EU ERROR", e); + return { result: {} } as FisuResponse; + }), + ]).catch((e) => { + console.error("FISU ERROR", e); + return [{ result: {} }, { result: {} }, { result: {} }] as FisuResponse[]; + }); + + // console.log("FISU data fetched", JSON.stringify({ pc, ps4us, ps4eu })); + const response: FisuResponse = { + result: { + ...pc.result, + "1000": [ps4us.result[0] as any], + "2000": [ps4eu.result[0] as any], + }, + }; + + return await cache.put("fisu", response); +}; + export const fisuFetchWorld = async ( - worldID: string -): Promise> => { - const url = `https://${subdomain( - worldID - )}.fisu.pw/api/population/?world=${worldID}`; + worldID: string, + cache: Cache +): Promise> => { + const start = Date.now(); + const data: FisuResponse = await fisuFetchAllWorlds(cache); + const end = Date.now(); - const res = await fetch(url); + const world = data.result[worldID]; + if (!world) { + console.error(`fisu: World ${worldID} not found`); + throw new Error(`fisu: World ${worldID} not found`); + } - const data: FisuResponse = await res.json(); - - const { vs, nc, tr, ns } = data.result[0]; + const { nc, tr, vs, ns } = world[0]; return { - raw: data, + raw: world[0], population: { total: vs + nc + tr + ns, nc, @@ -52,5 +82,10 @@ export const fisuFetchWorld = async ( vs, }, cachedAt: new Date(), + timings: { + enter: start, + exit: end, + upstream: end - start, + }, }; }; diff --git a/src/sources/honu.ts b/src/sources/honu.ts index ddf93ba..7e71d34 100644 --- a/src/sources/honu.ts +++ b/src/sources/honu.ts @@ -1,6 +1,7 @@ +import { Cache } from "../cache"; import { ServiceResponse } from "../types"; -interface HonuResponse { +type HonuResponse = { worldID: number; timestamp: string; cachedUntil: string; @@ -12,13 +13,34 @@ interface HonuResponse { ns_tr: number; ns_nc: number; nsOther: number; -} +}[]; + +const honuFetchAllWorlds = async (cache: Cache): Promise => { + const cached = await cache.get("honu"); + if (cached) { + return cached; + } + + const req = await fetch( + `https://wt.honu.pw/api/population/multiple?worldID=1&worldID=10&worldID=13&worldID=17&worldID=19&worldID=40&worldID=1000&worldID=2000` + ); + + return await cache.put("honu", await req.json()); +}; export const honuFetchWorld = async ( - worldID: string + worldID: string, + cache: Cache ): Promise> => { - const res = await fetch(`https://wt.honu.pw/api/population/${worldID}`); - const data: HonuResponse = await res.json(); + const start = Date.now(); + const resp = await honuFetchAllWorlds(cache); + const end = Date.now(); + + const data = resp.find((w) => w.worldID === Number(worldID)); + + if (!data) { + throw new Error(`honu: World ${worldID} not found`); + } return { population: { @@ -29,5 +51,10 @@ export const honuFetchWorld = async ( }, raw: data, cachedAt: new Date(), + timings: { + enter: start, + exit: end, + upstream: end - start, + }, }; }; diff --git a/src/sources/saerro.ts b/src/sources/saerro.ts index 1bef7da..2b56f75 100644 --- a/src/sources/saerro.ts +++ b/src/sources/saerro.ts @@ -1,17 +1,21 @@ +import { Cache } from "../cache"; import { Population, ServiceResponse } from "../types"; -interface OneResponse { +interface SaerroResponse { data: { - world: { - id: string; + allWorlds: { + id: number; population: Population; - }; + }[]; }; } -export const saerroFetchWorld = async ( - id: string -): Promise> => { +const saerroFetchAllWorlds = async (cache: Cache): Promise => { + const cached = await cache.get("saerro"); + if (cached) { + return cached; + } + const req = await fetch(`https://saerro.ps2.live/graphql`, { method: "POST", headers: { @@ -19,7 +23,7 @@ export const saerroFetchWorld = async ( }, body: JSON.stringify({ query: `{ - world(by: {id: ${id}}) { + allWorlds { id population { total @@ -32,11 +36,32 @@ export const saerroFetchWorld = async ( }), }); - const json: OneResponse = await req.json(); + return await cache.put("saerro", await req.json()); +}; + +export const saerroFetchWorld = async ( + id: string, + cache: Cache +): Promise> => { + const start = Date.now(); + + const json: SaerroResponse = await saerroFetchAllWorlds(cache); + const end = Date.now(); + + const world = json.data.allWorlds.find((w) => w.id === Number(id)); + + if (!world) { + throw new Error(`World ${id} not found`); + } return { - population: json.data.world.population, - raw: json, + population: world.population, + raw: world, cachedAt: new Date(), + timings: { + enter: start, + exit: end, + upstream: end - start, + }, }; }; diff --git a/src/sources/voidwell.ts b/src/sources/voidwell.ts index 73f33ba..375d412 100644 --- a/src/sources/voidwell.ts +++ b/src/sources/voidwell.ts @@ -1,6 +1,7 @@ +import { Cache } from "../cache"; import { ServiceResponse } from "../types"; -interface VoidwellResponse { +type VoidwellResponse = Array<{ id: number; name: string; isOnline: boolean; @@ -22,39 +23,76 @@ interface VoidwellResponse { ns: number; }[]; }; -} +}>; -const platform = (worldID: string) => { - switch (worldID) { - case "1000": - return "ps4us"; - case "2000": - return "ps4eu"; - default: - return "pc"; +const voidwellFetchAllWorlds = async ( + cache: Cache +): Promise => { + const cached = await cache.get("voidwell"); + if (cached) { + return cached; } + + const [pc, ps4us, ps4eu] = await Promise.all([ + fetch(`https://api.voidwell.com/ps2/worldstate/?platform=pc`) + .then((res) => res.json()) + .catch((e) => { + console.error("voidwell PC ERROR", e); + return [] as VoidwellResponse; + }), + fetch(`https://api.voidwell.com/ps2/worldstate/?platform=ps4us`) + .then((res) => res.json()) + .catch((e) => { + console.error("voidwell PS4US ERROR", e); + return [] as VoidwellResponse; + }), + fetch(`https://api.voidwell.com/ps2/worldstate/?platform=ps4eu`) + .then((res) => res.json()) + .catch((e) => { + console.error("voidwell PS4EU ERROR", e); + return [] as VoidwellResponse; + }), + ]); + + // console.log("voidwell data fetched", JSON.stringify({ pc, ps4us, ps4eu })); + const response: VoidwellResponse = [ + ...pc, + ...ps4us, + ...ps4eu, + ] as VoidwellResponse; + + return await cache.put("voidwell", response); }; // Voidwell is missing Oshur, and since zoneStates are the only way we can get a faction-specific population count, // we're stuck with not counting faction populations. export const voidwellFetchWorld = async ( - worldID: string -): Promise> => { - const res = await fetch( - `https://api.voidwell.com/ps2/worldstate/${worldID}?platform=${platform( - worldID - )}` - ); - const data: VoidwellResponse = await res.json(); + worldID: string, + cache: Cache +): Promise> => { + const start = Date.now(); + const data = await voidwellFetchAllWorlds(cache); + const end = Date.now(); + + const world = data.find((w) => w.id === Number(worldID)); + + if (!world) { + throw new Error(`voidwell: World ${worldID} not found`); + } return { - raw: data, + raw: world, population: { - total: data.onlineCharacters, + total: world.onlineCharacters, nc: undefined, tr: undefined, vs: undefined, }, cachedAt: new Date(), + timings: { + enter: start, + exit: end, + upstream: end - start, + }, }; }; diff --git a/src/types.ts b/src/types.ts index d30c543..6d2452d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -9,6 +9,11 @@ export interface ServiceResponse { population: Population; raw: Raw; cachedAt: Date; + timings?: { + enter: number; + upstream: number; + exit: number; + }; } export interface Env { @@ -43,6 +48,12 @@ export type DebugPayload = { honu: any; voidwell: any; }; + timings: { + saerro: any; + fisu: any; + honu: any; + voidwell: any; + }; lastFetchTimes: { saerro?: Date; fisu?: Date; diff --git a/wrangler.toml b/wrangler.toml index f6be851..3c394b4 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -2,6 +2,10 @@ name = "agg-population" main = "src/index.ts" compatibility_date = "2022-12-19" +[route] +pattern = "agg.ps2.live/population*" +zone_name = "ps2.live" + [vars] DISABLE_HONU = "0" DISABLE_FISU = "0"