refactor, add ~/health

src/fetcher.ts

@@ -0,0 +1,134 @@
+import { Cache } from "./cache";
+import { fisuFetchWorld } from "./sources/fisu";
+import { honuFetchWorld } from "./sources/honu";
+import { saerroFetchWorld } from "./sources/saerro";
+import { voidwellFetchWorld } from "./sources/voidwell";
+import { DebugPayload, Flags, OnePayload, ServiceResponse } from "./types";
+
+const avgOf = (arr: number[]) =>
+  Math.floor(arr.reduce((a, b) => a + b, 0) / arr.length);
+
+const flatMapBy = (arr: any[], key: string) =>
+  arr.reduce((a, b) => [...a, b[key]], []);
+
+const defaultServiceResponse: ServiceResponse<number, null> = {
+  population: {
+    total: -1,
+    nc: -1,
+    tr: -1,
+    vs: -1,
+  },
+  raw: null,
+  cachedAt: new Date(),
+};
+
+type World = {
+  world: OnePayload | null;
+  debug: DebugPayload;
+};
+
+export const getWorld = async (id: string, cache: Cache, flags: Flags) => {
+  const cached = await cache.get<World>(id);
+  if (cached) {
+    return cached;
+  }
+
+  const [saerro, fisu, honu, voidwell] = await Promise.all([
+    !flags.disableSaerro
+      ? saerroFetchWorld(id, cache).catch((e) => {
+          console.error("SAERRO ERROR:", e);
+          return defaultServiceResponse;
+        })
+      : defaultServiceResponse,
+    !flags.disableFisu
+      ? fisuFetchWorld(id, cache, flags.fisuUsePS4EU).catch(
+          () => defaultServiceResponse
+        )
+      : defaultServiceResponse,
+    !flags.disableHonu
+      ? honuFetchWorld(id, cache).catch(() => defaultServiceResponse)
+      : defaultServiceResponse,
+    !flags.disableVoidwell
+      ? voidwellFetchWorld(id, cache, flags.voidwellUsePS4).catch(
+          () => defaultServiceResponse
+        )
+      : defaultServiceResponse,
+  ]);
+
+  const debug: DebugPayload = {
+    raw: {
+      saerro: saerro.raw,
+      fisu: fisu.raw,
+      honu: honu.raw,
+      voidwell: voidwell.raw,
+    },
+    timings: {
+      saerro: saerro?.timings || null,
+      fisu: fisu?.timings || null,
+      honu: honu?.timings || null,
+      voidwell: voidwell?.timings || null,
+    },
+    lastFetchTimes: {
+      saerro: saerro.cachedAt,
+      fisu: fisu.cachedAt,
+      honu: honu.cachedAt,
+      voidwell: voidwell.cachedAt,
+    },
+  };
+
+  const totalPopulations = [
+    saerro.population.total,
+    fisu.population.total,
+    honu.population.total,
+    voidwell.population.total,
+  ].filter((x) => x > 0);
+
+  if (totalPopulations.length === 0) {
+    return await cache.put<World>(id, {
+      world:
+        id !== "19"
+          ? null
+          : {
+              // Jaeger gets a special case, we assume it's always up, but empty.
+              id: 19,
+              average: 0,
+              factions: {
+                nc: 0,
+                tr: 0,
+                vs: 0,
+              },
+              services: {
+                saerro: 0,
+                fisu: 0,
+                honu: 0,
+                voidwell: 0,
+              },
+            },
+      debug,
+    });
+  }
+
+  const factionPopulations = [
+    saerro.population,
+    fisu.population,
+    honu.population,
+  ].filter((x) => x.total > 0);
+
+  const payload: OnePayload = {
+    id: Number(id),
+    average: avgOf(totalPopulations),
+    factions: {
+      nc: avgOf(flatMapBy(factionPopulations, "nc")),
+      tr: avgOf(flatMapBy(factionPopulations, "tr")),
+      vs: avgOf(flatMapBy(factionPopulations, "vs")),
+    },
+    services: {
+      saerro: saerro.population.total,
+      fisu: fisu.population.total,
+      honu: honu.population.total,
+      voidwell: voidwell.population.total,
+    },
+  };
+
+  return await cache.put(id, { world: payload, debug });
+};

src/handlers.ts

@@ -1,139 +1,8 @@
 import { IRequest } from "itty-router";
-import { saerroFetchWorld } from "./sources/saerro";
-import { fisuFetchWorld } from "./sources/fisu";
-import { honuFetchWorld } from "./sources/honu";
-import { voidwellFetchWorld } from "./sources/voidwell";
 import { noData } from "./errors";
 import { DebugPayload, Flags, OnePayload, ServiceResponse } from "./types";
 import { Cache } from "./cache";
-
+import { getWorld } from "./fetcher";
-const avgOf = (arr: number[]) =>
-  Math.floor(arr.reduce((a, b) => a + b, 0) / arr.length);
-
-const flatMapBy = (arr: any[], key: string) =>
-  arr.reduce((a, b) => [...a, b[key]], []);
-
-const defaultServiceResponse: ServiceResponse<number, null> = {
-  population: {
-    total: -1,
-    nc: -1,
-    tr: -1,
-    vs: -1,
-  },
-  raw: null,
-  cachedAt: new Date(),
-};
-
-type World = {
-  world: OnePayload | null;
-  debug: DebugPayload;
-};
-
-export const getWorld = async (id: string, cache: Cache, flags: Flags) => {
-  const cached = await cache.get<World>(id);
-  if (cached) {
-    return cached;
-  }
-
-  const [saerro, fisu, honu, voidwell] = await Promise.all([
-    !flags.disableSaerro
-      ? saerroFetchWorld(id, cache).catch((e) => {
-          console.error("SAERRO ERROR:", e);
-          return defaultServiceResponse;
-        })
-      : defaultServiceResponse,
-    !flags.disableFisu
-      ? fisuFetchWorld(id, cache, flags.fisuUsePS4EU).catch(
-          () => defaultServiceResponse
-        )
-      : defaultServiceResponse,
-    !flags.disableHonu
-      ? honuFetchWorld(id, cache).catch(() => defaultServiceResponse)
-      : defaultServiceResponse,
-    !flags.disableVoidwell
-      ? voidwellFetchWorld(id, cache, flags.voidwellUsePS4).catch(
-          () => defaultServiceResponse
-        )
-      : defaultServiceResponse,
-  ]);
-
-  const debug: DebugPayload = {
-    raw: {
-      saerro: saerro.raw,
-      fisu: fisu.raw,
-      honu: honu.raw,
-      voidwell: voidwell.raw,
-    },
-    timings: {
-      saerro: saerro?.timings || null,
-      fisu: fisu?.timings || null,
-      honu: honu?.timings || null,
-      voidwell: voidwell?.timings || null,
-    },
-    lastFetchTimes: {
-      saerro: saerro.cachedAt,
-      fisu: fisu.cachedAt,
-      honu: honu.cachedAt,
-      voidwell: voidwell.cachedAt,
-    },
-  };
-
-  const totalPopulations = [
-    saerro.population.total,
-    fisu.population.total,
-    honu.population.total,
-    voidwell.population.total,
-  ].filter((x) => x > 0);
-
-  if (totalPopulations.length === 0) {
-    return await cache.put<World>(id, {
-      world:
-        id !== "19"
-          ? null
-          : {
-              // Jaeger gets a special case, we assume it's always up, but empty.
-              id: 19,
-              average: 0,
-              factions: {
-                nc: 0,
-                tr: 0,
-                vs: 0,
-              },
-              services: {
-                saerro: 0,
-                fisu: 0,
-                honu: 0,
-                voidwell: 0,
-              },
-            },
-      debug,
-    });
-  }
-
-  const factionPopulations = [
-    saerro.population,
-    fisu.population,
-    honu.population,
-  ].filter((x) => x.total > 0);
-
-  const payload: OnePayload = {
-    id: Number(id),
-    average: avgOf(totalPopulations),
-    factions: {
-      nc: avgOf(flatMapBy(factionPopulations, "nc")),
-      tr: avgOf(flatMapBy(factionPopulations, "tr")),
-      vs: avgOf(flatMapBy(factionPopulations, "vs")),
-    },
-    services: {
-      saerro: saerro.population.total,
-      fisu: fisu.population.total,
-      honu: honu.population.total,
-      voidwell: voidwell.population.total,
-    },
-  };
-
-  return await cache.put(id, { world: payload, debug });
-};
 
 export const handleOne = async (
   { params: { id }, query: { debug: debugParam } }: IRequest,
@@ -165,17 +34,29 @@ export const handleAll = async (
   { query: { debug } }: IRequest,
   _2: unknown,
   _3: unknown,
-  Cache: Cache,
+  cache: Cache,
   flags: Flags
 ): Promise<Response> => {
+  const cached = await cache.get("all");
+  if (cached) {
+    return new Response(JSON.stringify(cached), {
+      headers: {
+        "content-type": "application/json",
+      },
+    });
+  }
+
   const worlds = ["1", "10", "13", "17", "19", "40", "1000", "2000"];
 
-  const worldData = [];
+  const worldTasks = [];
 
   for (const world of worlds) {
-    worldData.push(await getWorld(world, Cache, flags));
+    worldTasks.push(getWorld(world, cache, flags));
   }
 
+  await worldTasks[0]; // Force the first one to cache for the rest
+  const worldData = await Promise.all(worldTasks.slice(1));
+
   if (debug === "1") {
     return new Response(JSON.stringify(worldData), {
       headers: {
@@ -186,112 +67,11 @@ export const handleAll = async (
 
   const worldPayloads = worldData.map((x: any) => x.world || x);
 
+  await cache.put("all", worldPayloads);
+
   return new Response(JSON.stringify(worldPayloads), {
     headers: {
       "content-type": "application/json",
     },
   });
 };
-
-export const index = (): Response => {
-  const body = `Aggregate Planetside 2 World Population
-
-GitHub: https://github.com/genudine/agg-population
-Production: https://agg.ps2.live/population
-
-Need help with this data? 
-
-## Methodology
-
-This service aggregates the population data from the following sources:
-- https://saerro.ps2.live/
-- https://ps2.fisu.pw/
-- https://wt.honu.pw/
-- https://voidwell.com/ (caveat: no factions, non-standard counting method)
-
-## Routes
-
-GET /:id - Get one world by ID
-
-  {
-    "id": 17,
-    "average": 285,
-    "factions": {
-      "nc": 91,
-      "tr": 92,
-      "vs": 91
-    },
-    "services": {
-      "saerro": 282,
-      "fisu": 271,
-      "honu": 292,
-      "voidwell": 298
-    }
-  }
-
-  Query Parameters:
-
-    ?debug=1 - Adds these fields to the response:
-      { 
-        /// ... other fields
-        "raw": {
-          "saerro": { ... },
-          "fisu": { ... },
-          "honu": { ... },
-          "voidwell": { ... }
-        },
-        "lastFetchTimes": {
-          "saerro": "2020-10-10T00:00:00.000Z",
-          "fisu": "2020-10-10T00:00:00.000Z",
-          "honu": "2020-10-10T00:00:00.000Z",
-          "voidwell": "2020-10-10T00:00:00.000Z"
-        }
-      }
-
-GET /all - Get all worlds
-
-  [
-    {
-      "id": 17,
-      "average": 285,
-      "factions": {
-        "nc": 91,
-        "tr": 92,
-        "vs": 91
-      },
-      "services": {
-        "saerro": 282,
-        "fisu": 271,
-        "honu": 292,
-        "voidwell": 298
-      }
-    },
-    {
-      "id": 1,
-      "average": 83,
-      "factions": {
-        "nc": 30,
-        "tr": 15,
-        "vs": 29
-      },
-      "services": {
-        "saerro": 95,
-        "fisu": 48,
-        "honu": 91,
-        "voidwell": 99
-      }
-    }
-  ]
-
-  -- This also has a debug query parameter, but it's extremely verbose. It's good for debugging extreme async issues with the platform.
-
-## Caching and usage limits
-
-This service cached on a world basis for 3 minutes. Debug data is cached alongside world data too.`;
-
-  return new Response(body, {
-    headers: {
-      "content-type": "text/plain",
-    },
-  });
-};

src/index.ts

@@ -1,7 +1,8 @@
 import { Route, Router, RouterType } from "itty-router";
-import { handleAll, handleOne, index } from "./handlers";
+import { handleAll, handleOne } from "./handlers";
 import { Env, Flags } from "./types";
 import { Cache } from "./cache";
+import { index } from "./landing";
 
 interface BasicRouter extends RouterType {
   all: Route;
@@ -33,6 +34,27 @@ router
       );
     }
   )
+  .get<BasicRouter>("/population~/health", async () => {
+    const [saerro, voidwell, honu, fisu] = await Promise.all([
+      fetch("https://saerro.ps2.live/health").then((r) => r.status === 200),
+      fetch("https://voidwell.com/").then((r) => r.status === 200),
+      fetch("https://wt.honu.pw/api/health").then((r) => r.status === 200),
+      fetch("https://ps2.fisu.pw").then((r) => r.status === 200),
+    ]);
+
+    return new Response(
+      JSON.stringify({
+        saerro,
+        voidwell,
+        honu,
+        fisu,
+      }),
+      {
+        headers: { "content-type": "application/json" },
+        status: saerro && voidwell && honu && fisu ? 200 : 502,
+      }
+    );
+  })
   .all<BasicRouter>("*", () => {
     return new Response("Not found", {
       headers: { "content-type": "text/plain" },

src/landing.ts

@@ -0,0 +1,106 @@
+export const index = (): Response => {
+  const body = `Aggregate Planetside 2 World Population
+
+GitHub: https://github.com/genudine/agg-population
+Production: https://agg.ps2.live/population
+
+Need help with this data? 
+
+## Methodology
+
+This service aggregates the population data from the following sources:
+- https://saerro.ps2.live/
+- https://ps2.fisu.pw/
+- https://wt.honu.pw/
+- https://voidwell.com/ (caveat: no factions, non-standard counting method)
+
+## Routes
+
+GET /:id - Get one world by ID
+
+  {
+    "id": 17,
+    "average": 285,
+    "factions": {
+      "nc": 91,
+      "tr": 92,
+      "vs": 91
+    },
+    "services": {
+      "saerro": 282,
+      "fisu": 271,
+      "honu": 292,
+      "voidwell": 298
+    }
+  }
+
+  Query Parameters:
+
+    ?debug=1 - Adds these fields to the response:
+      { 
+        /// ... other fields
+        "raw": {
+          "saerro": { ... },
+          "fisu": { ... },
+          "honu": { ... },
+          "voidwell": { ... }
+        },
+        "lastFetchTimes": {
+          "saerro": "2020-10-10T00:00:00.000Z",
+          "fisu": "2020-10-10T00:00:00.000Z",
+          "honu": "2020-10-10T00:00:00.000Z",
+          "voidwell": "2020-10-10T00:00:00.000Z"
+        }
+      }
+
+GET /all - Get all worlds
+
+  [
+    {
+      "id": 17,
+      "average": 285,
+      "factions": {
+        "nc": 91,
+        "tr": 92,
+        "vs": 91
+      },
+      "services": {
+        "saerro": 282,
+        "fisu": 271,
+        "honu": 292,
+        "voidwell": 298
+      }
+    },
+    {
+      "id": 1,
+      "average": 83,
+      "factions": {
+        "nc": 30,
+        "tr": 15,
+        "vs": 29
+      },
+      "services": {
+        "saerro": 95,
+        "fisu": 48,
+        "honu": 91,
+        "voidwell": 99
+      }
+    }
+  ]
+
+  -- This also has a debug query parameter, but it's extremely verbose. It's good for debugging extreme async issues with the platform.
+
+GET ~/flags - Get the current feature flags. These wiggle knobs that affect request timings, caching, and other things.
+
+GET ~/health - Gets health of this and upstream services.
+
+## Caching and usage limits
+
+This service cached on a world basis for 3 minutes. Debug data is cached alongside world data too.`;
+
+  return new Response(body, {
+    headers: {
+      "content-type": "text/plain",
+    },
+  });
+};