From ac05a12ca1c29ecd7285f1f5ecfccfabdf2aaa02 Mon Sep 17 00:00:00 2001 From: Katalina Okano Date: Tue, 20 Dec 2022 19:52:44 -0500 Subject: [PATCH] refactor, add ~/health --- src/fetcher.ts | 134 +++++++++++++++++++++++++ src/handlers.ts | 256 ++++-------------------------------------------- src/index.ts | 24 ++++- src/landing.ts | 106 ++++++++++++++++++++ 4 files changed, 281 insertions(+), 239 deletions(-) create mode 100644 src/fetcher.ts create mode 100644 src/landing.ts diff --git a/src/fetcher.ts b/src/fetcher.ts new file mode 100644 index 0000000..50e1966 --- /dev/null +++ b/src/fetcher.ts @@ -0,0 +1,134 @@ +import { Cache } from "./cache"; +import { fisuFetchWorld } from "./sources/fisu"; +import { honuFetchWorld } from "./sources/honu"; +import { saerroFetchWorld } from "./sources/saerro"; +import { voidwellFetchWorld } from "./sources/voidwell"; +import { DebugPayload, Flags, OnePayload, ServiceResponse } from "./types"; + +const avgOf = (arr: number[]) => + Math.floor(arr.reduce((a, b) => a + b, 0) / arr.length); + +const flatMapBy = (arr: any[], key: string) => + arr.reduce((a, b) => [...a, b[key]], []); + +const defaultServiceResponse: ServiceResponse = { + population: { + total: -1, + nc: -1, + tr: -1, + vs: -1, + }, + raw: null, + cachedAt: new Date(), +}; + +type World = { + world: OnePayload | null; + debug: DebugPayload; +}; + +export const getWorld = async (id: string, cache: Cache, flags: Flags) => { + const cached = await cache.get(id); + if (cached) { + return cached; + } + + const [saerro, fisu, honu, voidwell] = await Promise.all([ + !flags.disableSaerro + ? saerroFetchWorld(id, cache).catch((e) => { + console.error("SAERRO ERROR:", e); + return defaultServiceResponse; + }) + : defaultServiceResponse, + !flags.disableFisu + ? fisuFetchWorld(id, cache, flags.fisuUsePS4EU).catch( + () => defaultServiceResponse + ) + : defaultServiceResponse, + !flags.disableHonu + ? honuFetchWorld(id, cache).catch(() => defaultServiceResponse) + : defaultServiceResponse, + !flags.disableVoidwell + ? voidwellFetchWorld(id, cache, flags.voidwellUsePS4).catch( + () => defaultServiceResponse + ) + : defaultServiceResponse, + ]); + + const debug: DebugPayload = { + raw: { + saerro: saerro.raw, + fisu: fisu.raw, + honu: honu.raw, + voidwell: voidwell.raw, + }, + timings: { + saerro: saerro?.timings || null, + fisu: fisu?.timings || null, + honu: honu?.timings || null, + voidwell: voidwell?.timings || null, + }, + lastFetchTimes: { + saerro: saerro.cachedAt, + fisu: fisu.cachedAt, + honu: honu.cachedAt, + voidwell: voidwell.cachedAt, + }, + }; + + const totalPopulations = [ + saerro.population.total, + fisu.population.total, + honu.population.total, + voidwell.population.total, + ].filter((x) => x > 0); + + if (totalPopulations.length === 0) { + return await cache.put(id, { + world: + id !== "19" + ? null + : { + // Jaeger gets a special case, we assume it's always up, but empty. + id: 19, + average: 0, + factions: { + nc: 0, + tr: 0, + vs: 0, + }, + services: { + saerro: 0, + fisu: 0, + honu: 0, + voidwell: 0, + }, + }, + debug, + }); + } + + const factionPopulations = [ + saerro.population, + fisu.population, + honu.population, + ].filter((x) => x.total > 0); + + const payload: OnePayload = { + id: Number(id), + average: avgOf(totalPopulations), + factions: { + nc: avgOf(flatMapBy(factionPopulations, "nc")), + tr: avgOf(flatMapBy(factionPopulations, "tr")), + vs: avgOf(flatMapBy(factionPopulations, "vs")), + }, + services: { + saerro: saerro.population.total, + fisu: fisu.population.total, + honu: honu.population.total, + voidwell: voidwell.population.total, + }, + }; + + return await cache.put(id, { world: payload, debug }); +}; diff --git a/src/handlers.ts b/src/handlers.ts index 086ee9d..be1a02a 100644 --- a/src/handlers.ts +++ b/src/handlers.ts @@ -1,139 +1,8 @@ import { IRequest } from "itty-router"; -import { saerroFetchWorld } from "./sources/saerro"; -import { fisuFetchWorld } from "./sources/fisu"; -import { honuFetchWorld } from "./sources/honu"; -import { voidwellFetchWorld } from "./sources/voidwell"; import { noData } from "./errors"; import { DebugPayload, Flags, OnePayload, ServiceResponse } from "./types"; import { Cache } from "./cache"; - -const avgOf = (arr: number[]) => - Math.floor(arr.reduce((a, b) => a + b, 0) / arr.length); - -const flatMapBy = (arr: any[], key: string) => - arr.reduce((a, b) => [...a, b[key]], []); - -const defaultServiceResponse: ServiceResponse = { - population: { - total: -1, - nc: -1, - tr: -1, - vs: -1, - }, - raw: null, - cachedAt: new Date(), -}; - -type World = { - world: OnePayload | null; - debug: DebugPayload; -}; - -export const getWorld = async (id: string, cache: Cache, flags: Flags) => { - const cached = await cache.get(id); - if (cached) { - return cached; - } - - const [saerro, fisu, honu, voidwell] = await Promise.all([ - !flags.disableSaerro - ? saerroFetchWorld(id, cache).catch((e) => { - console.error("SAERRO ERROR:", e); - return defaultServiceResponse; - }) - : defaultServiceResponse, - !flags.disableFisu - ? fisuFetchWorld(id, cache, flags.fisuUsePS4EU).catch( - () => defaultServiceResponse - ) - : defaultServiceResponse, - !flags.disableHonu - ? honuFetchWorld(id, cache).catch(() => defaultServiceResponse) - : defaultServiceResponse, - !flags.disableVoidwell - ? voidwellFetchWorld(id, cache, flags.voidwellUsePS4).catch( - () => defaultServiceResponse - ) - : defaultServiceResponse, - ]); - - const debug: DebugPayload = { - raw: { - saerro: saerro.raw, - fisu: fisu.raw, - honu: honu.raw, - voidwell: voidwell.raw, - }, - timings: { - saerro: saerro?.timings || null, - fisu: fisu?.timings || null, - honu: honu?.timings || null, - voidwell: voidwell?.timings || null, - }, - lastFetchTimes: { - saerro: saerro.cachedAt, - fisu: fisu.cachedAt, - honu: honu.cachedAt, - voidwell: voidwell.cachedAt, - }, - }; - - const totalPopulations = [ - saerro.population.total, - fisu.population.total, - honu.population.total, - voidwell.population.total, - ].filter((x) => x > 0); - - if (totalPopulations.length === 0) { - return await cache.put(id, { - world: - id !== "19" - ? null - : { - // Jaeger gets a special case, we assume it's always up, but empty. - id: 19, - average: 0, - factions: { - nc: 0, - tr: 0, - vs: 0, - }, - services: { - saerro: 0, - fisu: 0, - honu: 0, - voidwell: 0, - }, - }, - debug, - }); - } - - const factionPopulations = [ - saerro.population, - fisu.population, - honu.population, - ].filter((x) => x.total > 0); - - const payload: OnePayload = { - id: Number(id), - average: avgOf(totalPopulations), - factions: { - nc: avgOf(flatMapBy(factionPopulations, "nc")), - tr: avgOf(flatMapBy(factionPopulations, "tr")), - vs: avgOf(flatMapBy(factionPopulations, "vs")), - }, - services: { - saerro: saerro.population.total, - fisu: fisu.population.total, - honu: honu.population.total, - voidwell: voidwell.population.total, - }, - }; - - return await cache.put(id, { world: payload, debug }); -}; +import { getWorld } from "./fetcher"; export const handleOne = async ( { params: { id }, query: { debug: debugParam } }: IRequest, @@ -165,17 +34,29 @@ export const handleAll = async ( { query: { debug } }: IRequest, _2: unknown, _3: unknown, - Cache: Cache, + cache: Cache, flags: Flags ): Promise => { + const cached = await cache.get("all"); + if (cached) { + return new Response(JSON.stringify(cached), { + headers: { + "content-type": "application/json", + }, + }); + } + const worlds = ["1", "10", "13", "17", "19", "40", "1000", "2000"]; - const worldData = []; + const worldTasks = []; for (const world of worlds) { - worldData.push(await getWorld(world, Cache, flags)); + worldTasks.push(getWorld(world, cache, flags)); } + await worldTasks[0]; // Force the first one to cache for the rest + const worldData = await Promise.all(worldTasks.slice(1)); + if (debug === "1") { return new Response(JSON.stringify(worldData), { headers: { @@ -186,112 +67,11 @@ export const handleAll = async ( const worldPayloads = worldData.map((x: any) => x.world || x); + await cache.put("all", worldPayloads); + return new Response(JSON.stringify(worldPayloads), { headers: { "content-type": "application/json", }, }); }; - -export const index = (): Response => { - const body = `Aggregate Planetside 2 World Population - -GitHub: https://github.com/genudine/agg-population -Production: https://agg.ps2.live/population - -Need help with this data? - -## Methodology - -This service aggregates the population data from the following sources: -- https://saerro.ps2.live/ -- https://ps2.fisu.pw/ -- https://wt.honu.pw/ -- https://voidwell.com/ (caveat: no factions, non-standard counting method) - -## Routes - -GET /:id - Get one world by ID - - { - "id": 17, - "average": 285, - "factions": { - "nc": 91, - "tr": 92, - "vs": 91 - }, - "services": { - "saerro": 282, - "fisu": 271, - "honu": 292, - "voidwell": 298 - } - } - - Query Parameters: - - ?debug=1 - Adds these fields to the response: - { - /// ... other fields - "raw": { - "saerro": { ... }, - "fisu": { ... }, - "honu": { ... }, - "voidwell": { ... } - }, - "lastFetchTimes": { - "saerro": "2020-10-10T00:00:00.000Z", - "fisu": "2020-10-10T00:00:00.000Z", - "honu": "2020-10-10T00:00:00.000Z", - "voidwell": "2020-10-10T00:00:00.000Z" - } - } - -GET /all - Get all worlds - - [ - { - "id": 17, - "average": 285, - "factions": { - "nc": 91, - "tr": 92, - "vs": 91 - }, - "services": { - "saerro": 282, - "fisu": 271, - "honu": 292, - "voidwell": 298 - } - }, - { - "id": 1, - "average": 83, - "factions": { - "nc": 30, - "tr": 15, - "vs": 29 - }, - "services": { - "saerro": 95, - "fisu": 48, - "honu": 91, - "voidwell": 99 - } - } - ] - - -- This also has a debug query parameter, but it's extremely verbose. It's good for debugging extreme async issues with the platform. - -## Caching and usage limits - -This service cached on a world basis for 3 minutes. Debug data is cached alongside world data too.`; - - return new Response(body, { - headers: { - "content-type": "text/plain", - }, - }); -}; diff --git a/src/index.ts b/src/index.ts index 6486d7b..d6219cd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,8 @@ import { Route, Router, RouterType } from "itty-router"; -import { handleAll, handleOne, index } from "./handlers"; +import { handleAll, handleOne } from "./handlers"; import { Env, Flags } from "./types"; import { Cache } from "./cache"; +import { index } from "./landing"; interface BasicRouter extends RouterType { all: Route; @@ -33,6 +34,27 @@ router ); } ) + .get("/population~/health", async () => { + const [saerro, voidwell, honu, fisu] = await Promise.all([ + fetch("https://saerro.ps2.live/health").then((r) => r.status === 200), + fetch("https://voidwell.com/").then((r) => r.status === 200), + fetch("https://wt.honu.pw/api/health").then((r) => r.status === 200), + fetch("https://ps2.fisu.pw").then((r) => r.status === 200), + ]); + + return new Response( + JSON.stringify({ + saerro, + voidwell, + honu, + fisu, + }), + { + headers: { "content-type": "application/json" }, + status: saerro && voidwell && honu && fisu ? 200 : 502, + } + ); + }) .all("*", () => { return new Response("Not found", { headers: { "content-type": "text/plain" }, diff --git a/src/landing.ts b/src/landing.ts new file mode 100644 index 0000000..8e1f964 --- /dev/null +++ b/src/landing.ts @@ -0,0 +1,106 @@ +export const index = (): Response => { + const body = `Aggregate Planetside 2 World Population + +GitHub: https://github.com/genudine/agg-population +Production: https://agg.ps2.live/population + +Need help with this data? + +## Methodology + +This service aggregates the population data from the following sources: +- https://saerro.ps2.live/ +- https://ps2.fisu.pw/ +- https://wt.honu.pw/ +- https://voidwell.com/ (caveat: no factions, non-standard counting method) + +## Routes + +GET /:id - Get one world by ID + + { + "id": 17, + "average": 285, + "factions": { + "nc": 91, + "tr": 92, + "vs": 91 + }, + "services": { + "saerro": 282, + "fisu": 271, + "honu": 292, + "voidwell": 298 + } + } + + Query Parameters: + + ?debug=1 - Adds these fields to the response: + { + /// ... other fields + "raw": { + "saerro": { ... }, + "fisu": { ... }, + "honu": { ... }, + "voidwell": { ... } + }, + "lastFetchTimes": { + "saerro": "2020-10-10T00:00:00.000Z", + "fisu": "2020-10-10T00:00:00.000Z", + "honu": "2020-10-10T00:00:00.000Z", + "voidwell": "2020-10-10T00:00:00.000Z" + } + } + +GET /all - Get all worlds + + [ + { + "id": 17, + "average": 285, + "factions": { + "nc": 91, + "tr": 92, + "vs": 91 + }, + "services": { + "saerro": 282, + "fisu": 271, + "honu": 292, + "voidwell": 298 + } + }, + { + "id": 1, + "average": 83, + "factions": { + "nc": 30, + "tr": 15, + "vs": 29 + }, + "services": { + "saerro": 95, + "fisu": 48, + "honu": 91, + "voidwell": 99 + } + } + ] + + -- This also has a debug query parameter, but it's extremely verbose. It's good for debugging extreme async issues with the platform. + +GET ~/flags - Get the current feature flags. These wiggle knobs that affect request timings, caching, and other things. + +GET ~/health - Gets health of this and upstream services. + +## Caching and usage limits + +This service cached on a world basis for 3 minutes. Debug data is cached alongside world data too.`; + + return new Response(body, { + headers: { + "content-type": "text/plain", + }, + }); +};