rebuild caching layer

src/cache.ts

@@ -1,26 +1,32 @@
-import { DebugPayload, OnePayload } from "./types";
+export class Cache {
-
+  private cache: Map<string, any> = new Map();
-type WorldObject = {
-  world: OnePayload | null;
-  debug: DebugPayload;
-};
-
-export class WorldCache {
   constructor(public kv: KVNamespace, public disableCache: boolean = false) {}
 
-  async get(id: string): Promise<WorldObject | null> {
+  async get<T>(id: string): Promise<T | null> {
     if (this.disableCache) {
       return null;
     }
-    const world = await this.kv.get<WorldObject>(id, "json");
+
-    return world;
+    // console.log("cache get", id);
+    let item = this.cache.get(id);
+    if (!item) {
+      // console.log("remote cache get", id);
+      item = await this.kv.get<T>(id, "json");
+      if (item) {
+        // console.log("local cache miss, remote cache hit", id);
+        this.cache.set(id, item);
+      }
+    }
+    return item;
   }
 
-  async put(id: string, world: WorldObject): Promise<WorldObject> {
+  async put<T>(id: string, world: T): Promise<T> {
     if (this.disableCache) {
       return world;
     }
 
+    this.cache.set(id, world);
+
     await this.kv.put(id, JSON.stringify(world), {
       expirationTtl: 60 * 3,
     });

src/handlers.ts

@@ -4,8 +4,8 @@ import { fisuFetchWorld } from "./sources/fisu";
 import { honuFetchWorld } from "./sources/honu";
 import { voidwellFetchWorld } from "./sources/voidwell";
 import { noData } from "./errors";
-import { DebugPayload, Flags, OnePayload } from "./types";
+import { DebugPayload, Flags, OnePayload, ServiceResponse } from "./types";
-import { WorldCache } from "./cache";
+import { Cache } from "./cache";
 
 const avgOf = (arr: number[]) =>
   Math.floor(arr.reduce((a, b) => a + b, 0) / arr.length);
@@ -13,35 +13,43 @@ const avgOf = (arr: number[]) =>
 const flatMapBy = (arr: any[], key: string) =>
   arr.reduce((a, b) => [...a, b[key]], []);
 
-const defaultServiceResponse = {
+const defaultServiceResponse: ServiceResponse<number, null> = {
   population: {
     total: -1,
-    nc: null,
+    nc: -1,
-    tr: null,
+    tr: -1,
-    vs: null,
+    vs: -1,
   },
   raw: null,
-  cachedAt: undefined,
+  cachedAt: new Date(),
 };
 
-export const getWorld = async (id: string, cache: WorldCache, flags: Flags) => {
+type World = {
-  const cached = await cache.get(id);
+  world: OnePayload | null;
+  debug: DebugPayload;
+};
+
+export const getWorld = async (id: string, cache: Cache, flags: Flags) => {
+  const cached = await cache.get<World>(id);
   if (cached) {
     return cached;
   }
 
   const [saerro, fisu, honu, voidwell] = await Promise.all([
     !flags.disableSaerro
-      ? saerroFetchWorld(id).catch(() => defaultServiceResponse)
+      ? saerroFetchWorld(id, cache).catch((e) => {
+          console.error("SAERRO ERROR:", e);
+          return defaultServiceResponse;
+        })
       : defaultServiceResponse,
     !flags.disableFisu
-      ? fisuFetchWorld(id).catch(() => defaultServiceResponse)
+      ? fisuFetchWorld(id, cache).catch(() => defaultServiceResponse)
       : defaultServiceResponse,
     !flags.disableHonu
-      ? honuFetchWorld(id).catch(() => defaultServiceResponse)
+      ? honuFetchWorld(id, cache).catch(() => defaultServiceResponse)
       : defaultServiceResponse,
     !flags.disableVoidwell
-      ? voidwellFetchWorld(id).catch(() => defaultServiceResponse)
+      ? voidwellFetchWorld(id, cache).catch(() => defaultServiceResponse)
       : defaultServiceResponse,
   ]);
 
@@ -52,6 +60,12 @@ export const getWorld = async (id: string, cache: WorldCache, flags: Flags) => {
       honu: honu.raw,
       voidwell: voidwell.raw,
     },
+    timings: {
+      saerro: saerro?.timings || null,
+      fisu: fisu?.timings || null,
+      honu: honu?.timings || null,
+      voidwell: voidwell?.timings || null,
+    },
     lastFetchTimes: {
       saerro: saerro.cachedAt,
       fisu: fisu.cachedAt,
@@ -68,7 +82,7 @@ export const getWorld = async (id: string, cache: WorldCache, flags: Flags) => {
   ].filter((x) => x > 0);
 
   if (totalPopulations.length === 0) {
-    return await cache.put(id, {
+    return await cache.put<World>(id, {
       world:
         id !== "19"
           ? null
@@ -121,10 +135,10 @@ export const handleOne = async (
   { params: { id }, query: { debug: debugParam } }: IRequest,
   _1: unknown,
   _2: unknown,
-  worldCache: WorldCache,
+  Cache: Cache,
   flags: Flags
 ) => {
-  const { world, debug } = await getWorld(id, worldCache, flags);
+  const { world, debug } = await getWorld(id, Cache, flags);
 
   if (world === null) {
     return noData();
@@ -144,22 +158,29 @@ export const handleOne = async (
 };
 
 export const handleAll = async (
-  _1: unknown,
+  { query: { debug } }: IRequest,
   _2: unknown,
   _3: unknown,
-  worldCache: WorldCache,
+  Cache: Cache,
   flags: Flags
 ): Promise<Response> => {
   const worlds = ["1", "10", "13", "17", "19", "40", "1000", "2000"];
 
-  const worldData = await Promise.all(
+  const worldData = [];
-    worlds.map((x) =>
+
-      getWorld(x, worldCache, flags).catch(() => {
+  for (const world of worlds) {
-        error: "World data is missing. Is it down?";
+    worldData.push(await getWorld(world, Cache, flags));
-      })
+  }
-    )
+
-  );
+  if (debug === "1") {
-  const worldPayloads = worldData.map((x) => x?.world || x);
+    return new Response(JSON.stringify(worldData), {
+      headers: {
+        "content-type": "application/json",
+      },
+    });
+  }
+
+  const worldPayloads = worldData.map((x: any) => x.world || x);
 
   return new Response(JSON.stringify(worldPayloads), {
     headers: {
@@ -258,9 +279,11 @@ GET /all - Get all worlds
     }
   ]
 
+  -- This also has a debug query parameter, but it's extremely verbose. It's good for debugging extreme async issues with the platform.
+
 ## Caching and usage limits
 
-This service cached on a world basis for 3 minutes.`;
+This service cached on a world basis for 3 minutes. Debug data is cached alongside world data too.`;
 
   return new Response(body, {
     headers: {

src/index.ts

@@ -1,7 +1,7 @@
 import { Route, Router, RouterType } from "itty-router";
 import { handleAll, handleOne, index } from "./handlers";
 import { Env, Flags } from "./types";
-import { WorldCache } from "./cache";
+import { Cache } from "./cache";
 
 interface BasicRouter extends RouterType {
   all: Route;
@@ -11,9 +11,14 @@ interface BasicRouter extends RouterType {
 const router = <BasicRouter>Router();
 
 router
-  .get<BasicRouter>("/", index)
+  .get<BasicRouter>(
-  .get<BasicRouter>("/all", handleAll)
+    "/",
-  .get<BasicRouter>("/:id", handleOne)
+    () =>
+      new Response(null, { status: 303, headers: { location: "/population/" } })
+  )
+  .get<BasicRouter>("/population/", index)
+  .get<BasicRouter>("/population/all", handleAll)
+  .get<BasicRouter>("/population/:id", handleOne)
   .all<BasicRouter>("*", () => {
     return new Response("Not found", {
       headers: { "content-type": "text/plain" },
@@ -22,7 +27,7 @@ router
 
 export default {
   fetch: async (request: Request, env: Env, ctx: ExecutionContext) => {
-    const worldCache = new WorldCache(env.CACHE, env.DISABLE_CACHE === "1");
+    const worldCache = new Cache(env.CACHE, env.DISABLE_CACHE === "1");
 
     const flags: Flags = {
       disableFisu: env.DISABLE_FISU === "1",
@@ -31,6 +36,8 @@ export default {
       disableVoidwell: env.DISABLE_VOIDWELL === "1",
     };
 
+    const start = Date.now();
+
     return router
       .handle(request as any, env, ctx, worldCache, flags)
       .then((response) => {
@@ -39,6 +46,7 @@ export default {
           "access-control-allow-method",
           "GET, HEAD, OPTIONS"
         );
+        response.headers.set("x-timing", `${Date.now() - start}ms`);
         return response;
       });
   },

src/sources/fisu.ts

@@ -1,50 +1,80 @@
-import { Population, ServiceResponse } from "../types";
+import { Cache } from "../cache";
-
+import { ServiceResponse } from "../types";
-const subdomain = (worldID: string) => {
-  switch (worldID) {
-    case "1000":
-      return "ps4us.ps2";
-    case "2000":
-      return "ps4eu.ps2";
-    default:
-      return "ps2";
-  }
-};
 
 interface FisuResponse {
-  config: {
+  result: Record<
-    world: string[];
+    string,
-  };
+    {
-  result: {
       worldId: number;
       vs: number;
       nc: number;
       tr: number;
       ns: number;
-  }[];
+    }[]
-  timing: {
+  >;
-    "start-ms": number;
-    "query-ms": number;
-    "total-ms": number;
-    "process-ms": number;
-  };
 }
 
+const fisuFetchAllWorlds = async (cache: Cache): Promise<FisuResponse> => {
+  const cached = await cache.get<FisuResponse>("fisu");
+  if (cached) {
+    // console.log("FISU data cached", cached);
+    return cached;
+  }
+
+  const [pc, ps4us, ps4eu] = await Promise.all([
+    fetch(`https://ps2.fisu.pw/api/population/?world=1,10,13,17,19,40`)
+      .then((res) => res.json<FisuResponse>())
+      .catch((e) => {
+        console.error("FISU PC ERROR", e);
+        return { result: {} } as FisuResponse;
+      }),
+    fetch(`https://ps4us.ps2.fisu.pw/api/population/?world=1000`)
+      .then((res) => res.json<FisuResponse>())
+      .catch((e) => {
+        console.error("FISU PS4US ERROR", e);
+        return { result: {} } as FisuResponse;
+      }),
+    fetch(`https://ps4eu.ps2.fisu.pw/api/population/?world=2000`)
+      .then((res) => res.json<FisuResponse>())
+      .catch((e) => {
+        console.error("FISU PS4EU ERROR", e);
+        return { result: {} } as FisuResponse;
+      }),
+  ]).catch((e) => {
+    console.error("FISU ERROR", e);
+    return [{ result: {} }, { result: {} }, { result: {} }] as FisuResponse[];
+  });
+
+  // console.log("FISU data fetched", JSON.stringify({ pc, ps4us, ps4eu }));
+  const response: FisuResponse = {
+    result: {
+      ...pc.result,
+      "1000": [ps4us.result[0] as any],
+      "2000": [ps4eu.result[0] as any],
+    },
+  };
+
+  return await cache.put("fisu", response);
+};
+
 export const fisuFetchWorld = async (
-  worldID: string
+  worldID: string,
-): Promise<ServiceResponse<number | undefined, FisuResponse | null>> => {
+  cache: Cache
-  const url = `https://${subdomain(
+): Promise<ServiceResponse<number, any>> => {
-    worldID
+  const start = Date.now();
-  )}.fisu.pw/api/population/?world=${worldID}`;
+  const data: FisuResponse = await fisuFetchAllWorlds(cache);
+  const end = Date.now();
 
-  const res = await fetch(url);
+  const world = data.result[worldID];
+  if (!world) {
+    console.error(`fisu: World ${worldID} not found`);
+    throw new Error(`fisu: World ${worldID} not found`);
+  }
 
-  const data: FisuResponse = await res.json();
+  const { nc, tr, vs, ns } = world[0];
-
-  const { vs, nc, tr, ns } = data.result[0];
 
   return {
-    raw: data,
+    raw: world[0],
     population: {
       total: vs + nc + tr + ns,
       nc,
@@ -52,5 +82,10 @@ export const fisuFetchWorld = async (
       vs,
     },
     cachedAt: new Date(),
+    timings: {
+      enter: start,
+      exit: end,
+      upstream: end - start,
+    },
   };
 };

src/sources/honu.ts

@@ -1,6 +1,7 @@
+import { Cache } from "../cache";
 import { ServiceResponse } from "../types";
 
-interface HonuResponse {
+type HonuResponse = {
   worldID: number;
   timestamp: string;
   cachedUntil: string;
@@ -12,13 +13,34 @@ interface HonuResponse {
   ns_tr: number;
   ns_nc: number;
   nsOther: number;
-}
+}[];
+
+const honuFetchAllWorlds = async (cache: Cache): Promise<HonuResponse> => {
+  const cached = await cache.get<HonuResponse>("honu");
+  if (cached) {
+    return cached;
+  }
+
+  const req = await fetch(
+    `https://wt.honu.pw/api/population/multiple?worldID=1&worldID=10&worldID=13&worldID=17&worldID=19&worldID=40&worldID=1000&worldID=2000`
+  );
+
+  return await cache.put("honu", await req.json<HonuResponse>());
+};
 
 export const honuFetchWorld = async (
-  worldID: string
+  worldID: string,
+  cache: Cache
 ): Promise<ServiceResponse<number, any>> => {
-  const res = await fetch(`https://wt.honu.pw/api/population/${worldID}`);
+  const start = Date.now();
-  const data: HonuResponse = await res.json();
+  const resp = await honuFetchAllWorlds(cache);
+  const end = Date.now();
+
+  const data = resp.find((w) => w.worldID === Number(worldID));
+
+  if (!data) {
+    throw new Error(`honu: World ${worldID} not found`);
+  }
 
   return {
     population: {
@@ -29,5 +51,10 @@ export const honuFetchWorld = async (
     },
     raw: data,
     cachedAt: new Date(),
+    timings: {
+      enter: start,
+      exit: end,
+      upstream: end - start,
+    },
   };
 };

src/sources/saerro.ts

@@ -1,17 +1,21 @@
+import { Cache } from "../cache";
 import { Population, ServiceResponse } from "../types";
 
-interface OneResponse {
+interface SaerroResponse {
   data: {
-    world: {
+    allWorlds: {
-      id: string;
+      id: number;
       population: Population<number>;
-    };
+    }[];
   };
 }
 
-export const saerroFetchWorld = async (
+const saerroFetchAllWorlds = async (cache: Cache): Promise<SaerroResponse> => {
-  id: string
+  const cached = await cache.get<SaerroResponse>("saerro");
-): Promise<ServiceResponse<number, OneResponse>> => {
+  if (cached) {
+    return cached;
+  }
+
   const req = await fetch(`https://saerro.ps2.live/graphql`, {
     method: "POST",
     headers: {
@@ -19,7 +23,7 @@ export const saerroFetchWorld = async (
     },
     body: JSON.stringify({
       query: `{
-                world(by: {id: ${id}}) {
+                allWorlds {
                   id 
                   population {
                     total
@@ -32,11 +36,32 @@ export const saerroFetchWorld = async (
     }),
   });
 
-  const json: OneResponse = await req.json();
+  return await cache.put("saerro", await req.json<SaerroResponse>());
+};
+
+export const saerroFetchWorld = async (
+  id: string,
+  cache: Cache
+): Promise<ServiceResponse<number, SaerroResponse["data"]["allWorlds"][1]>> => {
+  const start = Date.now();
+
+  const json: SaerroResponse = await saerroFetchAllWorlds(cache);
+  const end = Date.now();
+
+  const world = json.data.allWorlds.find((w) => w.id === Number(id));
+
+  if (!world) {
+    throw new Error(`World ${id} not found`);
+  }
 
   return {
-    population: json.data.world.population,
+    population: world.population,
-    raw: json,
+    raw: world,
     cachedAt: new Date(),
+    timings: {
+      enter: start,
+      exit: end,
+      upstream: end - start,
+    },
   };
 };

src/sources/voidwell.ts

@@ -1,6 +1,7 @@
+import { Cache } from "../cache";
 import { ServiceResponse } from "../types";
 
-interface VoidwellResponse {
+type VoidwellResponse = Array<{
   id: number;
   name: string;
   isOnline: boolean;
@@ -22,39 +23,76 @@ interface VoidwellResponse {
       ns: number;
     }[];
   };
-}
+}>;
 
-const platform = (worldID: string) => {
+const voidwellFetchAllWorlds = async (
-  switch (worldID) {
+  cache: Cache
-    case "1000":
+): Promise<VoidwellResponse> => {
-      return "ps4us";
+  const cached = await cache.get<VoidwellResponse>("voidwell");
-    case "2000":
+  if (cached) {
-      return "ps4eu";
+    return cached;
-    default:
-      return "pc";
   }
+
+  const [pc, ps4us, ps4eu] = await Promise.all([
+    fetch(`https://api.voidwell.com/ps2/worldstate/?platform=pc`)
+      .then((res) => res.json<VoidwellResponse>())
+      .catch((e) => {
+        console.error("voidwell PC ERROR", e);
+        return [] as VoidwellResponse;
+      }),
+    fetch(`https://api.voidwell.com/ps2/worldstate/?platform=ps4us`)
+      .then((res) => res.json<VoidwellResponse>())
+      .catch((e) => {
+        console.error("voidwell PS4US ERROR", e);
+        return [] as VoidwellResponse;
+      }),
+    fetch(`https://api.voidwell.com/ps2/worldstate/?platform=ps4eu`)
+      .then((res) => res.json<VoidwellResponse>())
+      .catch((e) => {
+        console.error("voidwell PS4EU ERROR", e);
+        return [] as VoidwellResponse;
+      }),
+  ]);
+
+  // console.log("voidwell data fetched", JSON.stringify({ pc, ps4us, ps4eu }));
+  const response: VoidwellResponse = [
+    ...pc,
+    ...ps4us,
+    ...ps4eu,
+  ] as VoidwellResponse;
+
+  return await cache.put("voidwell", response);
 };
 
 // Voidwell is missing Oshur, and since zoneStates are the only way we can get a faction-specific population count,
 // we're stuck with not counting faction populations.
 export const voidwellFetchWorld = async (
-  worldID: string
+  worldID: string,
-): Promise<ServiceResponse<undefined, VoidwellResponse>> => {
+  cache: Cache
-  const res = await fetch(
+): Promise<ServiceResponse<undefined, VoidwellResponse[0]>> => {
-    `https://api.voidwell.com/ps2/worldstate/${worldID}?platform=${platform(
+  const start = Date.now();
-      worldID
+  const data = await voidwellFetchAllWorlds(cache);
-    )}`
+  const end = Date.now();
-  );
+
-  const data: VoidwellResponse = await res.json();
+  const world = data.find((w) => w.id === Number(worldID));
+
+  if (!world) {
+    throw new Error(`voidwell: World ${worldID} not found`);
+  }
 
   return {
-    raw: data,
+    raw: world,
     population: {
-      total: data.onlineCharacters,
+      total: world.onlineCharacters,
       nc: undefined,
       tr: undefined,
       vs: undefined,
     },
     cachedAt: new Date(),
+    timings: {
+      enter: start,
+      exit: end,
+      upstream: end - start,
+    },
   };
 };

src/types.ts

@@ -9,6 +9,11 @@ export interface ServiceResponse<PT extends number | undefined, Raw> {
   population: Population<PT>;
   raw: Raw;
   cachedAt: Date;
+  timings?: {
+    enter: number;
+    upstream: number;
+    exit: number;
+  };
 }
 
 export interface Env {
@@ -43,6 +48,12 @@ export type DebugPayload = {
     honu: any;
     voidwell: any;
   };
+  timings: {
+    saerro: any;
+    fisu: any;
+    honu: any;
+    voidwell: any;
+  };
   lastFetchTimes: {
     saerro?: Date;
     fisu?: Date;

wrangler.toml

@@ -2,6 +2,10 @@ name = "agg-population"
 main = "src/index.ts"
 compatibility_date = "2022-12-19"
 
+[route]
+pattern = "agg.ps2.live/population*"
+zone_name = "ps2.live"
+
 [vars]
 DISABLE_HONU = "0"
 DISABLE_FISU = "0"