ps2.live/agg-population
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rebuild caching layer

Browse source
This commit is contained in:
41666 2022-12-20 18:56:19 -05:00
parent bd89eee6e8
commit 715e367137
9 changed files with 295 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/cache.ts
View file

@ -1,26 +1,32 @@
import { DebugPayload, OnePayload } from "./types";
type WorldObject = {
world: OnePayload | null;
debug: DebugPayload;
};
export class WorldCache {
export class Cache {
private cache: Map<string, any> = new Map();
constructor(public kv: KVNamespace, public disableCache: boolean = false) {}
async get(id: string): Promise<WorldObject | null> {
async get<T>(id: string): Promise<T | null> {
if (this.disableCache) {
return null;
}
const world = await this.kv.get<WorldObject>(id, "json");
return world;
// console.log("cache get", id);
let item = this.cache.get(id);
if (!item) {
// console.log("remote cache get", id);
item = await this.kv.get<T>(id, "json");
if (item) {
// console.log("local cache miss, remote cache hit", id);
this.cache.set(id, item);
}
}
return item;
}
async put(id: string, world: WorldObject): Promise<WorldObject> {
async put<T>(id: string, world: T): Promise<T> {
if (this.disableCache) {
return world;
}
this.cache.set(id, world);
await this.kv.put(id, JSON.stringify(world), {
expirationTtl: 60 * 3,
});

77
src/handlers.ts
View file

@ -4,8 +4,8 @@ import { fisuFetchWorld } from "./sources/fisu";
import { honuFetchWorld } from "./sources/honu";
import { voidwellFetchWorld } from "./sources/voidwell";
import { noData } from "./errors";
import { DebugPayload, Flags, OnePayload } from "./types";
import { WorldCache } from "./cache";
import { DebugPayload, Flags, OnePayload, ServiceResponse } from "./types";
import { Cache } from "./cache";
const avgOf = (arr: number[]) =>
Math.floor(arr.reduce((a, b) => a + b, 0) / arr.length);
@ -13,35 +13,43 @@ const avgOf = (arr: number[]) =>
const flatMapBy = (arr: any[], key: string) =>
arr.reduce((a, b) => [...a, b[key]], []);
const defaultServiceResponse = {
const defaultServiceResponse: ServiceResponse<number, null> = {
population: {
total: -1,
nc: null,
tr: null,
vs: null,
nc: -1,
tr: -1,
vs: -1,
},
raw: null,
cachedAt: undefined,
cachedAt: new Date(),
};
export const getWorld = async (id: string, cache: WorldCache, flags: Flags) => {
const cached = await cache.get(id);
type World = {
world: OnePayload | null;
debug: DebugPayload;
};
export const getWorld = async (id: string, cache: Cache, flags: Flags) => {
const cached = await cache.get<World>(id);
if (cached) {
return cached;
}
const [saerro, fisu, honu, voidwell] = await Promise.all([
!flags.disableSaerro
? saerroFetchWorld(id).catch(() => defaultServiceResponse)
? saerroFetchWorld(id, cache).catch((e) => {
console.error("SAERRO ERROR:", e);
return defaultServiceResponse;
})
: defaultServiceResponse,
!flags.disableFisu
? fisuFetchWorld(id).catch(() => defaultServiceResponse)
? fisuFetchWorld(id, cache).catch(() => defaultServiceResponse)
: defaultServiceResponse,
!flags.disableHonu
? honuFetchWorld(id).catch(() => defaultServiceResponse)
? honuFetchWorld(id, cache).catch(() => defaultServiceResponse)
: defaultServiceResponse,
!flags.disableVoidwell
? voidwellFetchWorld(id).catch(() => defaultServiceResponse)
? voidwellFetchWorld(id, cache).catch(() => defaultServiceResponse)
: defaultServiceResponse,
]);
@ -52,6 +60,12 @@ export const getWorld = async (id: string, cache: WorldCache, flags: Flags) => {
honu: honu.raw,
voidwell: voidwell.raw,
},
timings: {
saerro: saerro?.timings || null,
fisu: fisu?.timings || null,
honu: honu?.timings || null,
voidwell: voidwell?.timings || null,
},
lastFetchTimes: {
saerro: saerro.cachedAt,
fisu: fisu.cachedAt,
@ -68,7 +82,7 @@ export const getWorld = async (id: string, cache: WorldCache, flags: Flags) => {
].filter((x) => x > 0);
if (totalPopulations.length === 0) {
return await cache.put(id, {
return await cache.put<World>(id, {
world:
id !== "19"
? null
@ -121,10 +135,10 @@ export const handleOne = async (
{ params: { id }, query: { debug: debugParam } }: IRequest,
_1: unknown,
_2: unknown,
worldCache: WorldCache,
Cache: Cache,
flags: Flags
) => {
const { world, debug } = await getWorld(id, worldCache, flags);
const { world, debug } = await getWorld(id, Cache, flags);
if (world === null) {
return noData();
@ -144,22 +158,29 @@ export const handleOne = async (
};
export const handleAll = async (
_1: unknown,
{ query: { debug } }: IRequest,
_2: unknown,
_3: unknown,
worldCache: WorldCache,
Cache: Cache,
flags: Flags
): Promise<Response> => {
const worlds = ["1", "10", "13", "17", "19", "40", "1000", "2000"];
const worldData = await Promise.all(
worlds.map((x) =>
getWorld(x, worldCache, flags).catch(() => {
error: "World data is missing. Is it down?";
})
)
);
const worldPayloads = worldData.map((x) => x?.world || x);
const worldData = [];
for (const world of worlds) {
worldData.push(await getWorld(world, Cache, flags));
}
if (debug === "1") {
return new Response(JSON.stringify(worldData), {
headers: {
"content-type": "application/json",
},
});
}
const worldPayloads = worldData.map((x: any) => x.world || x);
return new Response(JSON.stringify(worldPayloads), {
headers: {
@ -258,9 +279,11 @@ GET /all - Get all worlds
}
]
-- This also has a debug query parameter, but it's extremely verbose. It's good for debugging extreme async issues with the platform.
## Caching and usage limits
This service cached on a world basis for 3 minutes.`;
This service cached on a world basis for 3 minutes. Debug data is cached alongside world data too.`;
return new Response(body, {
headers: {

18
src/index.ts
View file

@ -1,7 +1,7 @@
import { Route, Router, RouterType } from "itty-router";
import { handleAll, handleOne, index } from "./handlers";
import { Env, Flags } from "./types";
import { WorldCache } from "./cache";
import { Cache } from "./cache";
interface BasicRouter extends RouterType {
all: Route;
@ -11,9 +11,14 @@ interface BasicRouter extends RouterType {
const router = <BasicRouter>Router();
router
.get<BasicRouter>("/", index)
.get<BasicRouter>("/all", handleAll)
.get<BasicRouter>("/:id", handleOne)
.get<BasicRouter>(
"/",
() =>
new Response(null, { status: 303, headers: { location: "/population/" } })
)
.get<BasicRouter>("/population/", index)
.get<BasicRouter>("/population/all", handleAll)
.get<BasicRouter>("/population/:id", handleOne)
.all<BasicRouter>("*", () => {
return new Response("Not found", {
headers: { "content-type": "text/plain" },
@ -22,7 +27,7 @@ router
export default {
fetch: async (request: Request, env: Env, ctx: ExecutionContext) => {
const worldCache = new WorldCache(env.CACHE, env.DISABLE_CACHE === "1");
const worldCache = new Cache(env.CACHE, env.DISABLE_CACHE === "1");
const flags: Flags = {
disableFisu: env.DISABLE_FISU === "1",
@ -31,6 +36,8 @@ export default {
disableVoidwell: env.DISABLE_VOIDWELL === "1",
};
const start = Date.now();
return router
.handle(request as any, env, ctx, worldCache, flags)
.then((response) => {
@ -39,6 +46,7 @@ export default {
"access-control-allow-method",
"GET, HEAD, OPTIONS"
);
response.headers.set("x-timing", `${Date.now() - start}ms`);
return response;
});
},

111
src/sources/fisu.ts
View file

@ -1,50 +1,80 @@
import { Population, ServiceResponse } from "../types";
const subdomain = (worldID: string) => {
switch (worldID) {
case "1000":
return "ps4us.ps2";
case "2000":
return "ps4eu.ps2";
default:
return "ps2";
}
};
import { Cache } from "../cache";
import { ServiceResponse } from "../types";
interface FisuResponse {
config: {
world: string[];
};
result: {
worldId: number;
vs: number;
nc: number;
tr: number;
ns: number;
}[];
timing: {
"start-ms": number;
"query-ms": number;
"total-ms": number;
"process-ms": number;
};
result: Record<
string,
{
worldId: number;
vs: number;
nc: number;
tr: number;
ns: number;
}[]
>;
}
const fisuFetchAllWorlds = async (cache: Cache): Promise<FisuResponse> => {
const cached = await cache.get<FisuResponse>("fisu");
if (cached) {
// console.log("FISU data cached", cached);
return cached;
}
const [pc, ps4us, ps4eu] = await Promise.all([
fetch(`https://ps2.fisu.pw/api/population/?world=1,10,13,17,19,40`)
.then((res) => res.json<FisuResponse>())
.catch((e) => {
console.error("FISU PC ERROR", e);
return { result: {} } as FisuResponse;
}),
fetch(`https://ps4us.ps2.fisu.pw/api/population/?world=1000`)
.then((res) => res.json<FisuResponse>())
.catch((e) => {
console.error("FISU PS4US ERROR", e);
return { result: {} } as FisuResponse;
}),
fetch(`https://ps4eu.ps2.fisu.pw/api/population/?world=2000`)
.then((res) => res.json<FisuResponse>())
.catch((e) => {
console.error("FISU PS4EU ERROR", e);
return { result: {} } as FisuResponse;
}),
]).catch((e) => {
console.error("FISU ERROR", e);
return [{ result: {} }, { result: {} }, { result: {} }] as FisuResponse[];
});
// console.log("FISU data fetched", JSON.stringify({ pc, ps4us, ps4eu }));
const response: FisuResponse = {
result: {
...pc.result,
"1000": [ps4us.result[0] as any],
"2000": [ps4eu.result[0] as any],
},
};
return await cache.put("fisu", response);
};
export const fisuFetchWorld = async (
worldID: string
): Promise<ServiceResponse<number | undefined, FisuResponse | null>> => {
const url = `https://${subdomain(
worldID
)}.fisu.pw/api/population/?world=${worldID}`;
worldID: string,
cache: Cache
): Promise<ServiceResponse<number, any>> => {
const start = Date.now();
const data: FisuResponse = await fisuFetchAllWorlds(cache);
const end = Date.now();
const res = await fetch(url);
const world = data.result[worldID];
if (!world) {
console.error(`fisu: World ${worldID} not found`);
throw new Error(`fisu: World ${worldID} not found`);
}
const data: FisuResponse = await res.json();
const { vs, nc, tr, ns } = data.result[0];
const { nc, tr, vs, ns } = world[0];
return {
raw: data,
raw: world[0],
population: {
total: vs + nc + tr + ns,
nc,
@ -52,5 +82,10 @@ export const fisuFetchWorld = async (
vs,
},
cachedAt: new Date(),
timings: {
enter: start,
exit: end,
upstream: end - start,
},
};
};

37
src/sources/honu.ts
View file

@ -1,6 +1,7 @@
import { Cache } from "../cache";
import { ServiceResponse } from "../types";
interface HonuResponse {
type HonuResponse = {
worldID: number;
timestamp: string;
cachedUntil: string;
@ -12,13 +13,34 @@ interface HonuResponse {
ns_tr: number;
ns_nc: number;
nsOther: number;
}
}[];
const honuFetchAllWorlds = async (cache: Cache): Promise<HonuResponse> => {
const cached = await cache.get<HonuResponse>("honu");
if (cached) {
return cached;
}
const req = await fetch(
`https://wt.honu.pw/api/population/multiple?worldID=1&worldID=10&worldID=13&worldID=17&worldID=19&worldID=40&worldID=1000&worldID=2000`
);
return await cache.put("honu", await req.json<HonuResponse>());
};
export const honuFetchWorld = async (
worldID: string
worldID: string,
cache: Cache
): Promise<ServiceResponse<number, any>> => {
const res = await fetch(`https://wt.honu.pw/api/population/${worldID}`);
const data: HonuResponse = await res.json();
const start = Date.now();
const resp = await honuFetchAllWorlds(cache);
const end = Date.now();
const data = resp.find((w) => w.worldID === Number(worldID));
if (!data) {
throw new Error(`honu: World ${worldID} not found`);
}
return {
population: {
@ -29,5 +51,10 @@ export const honuFetchWorld = async (
},
raw: data,
cachedAt: new Date(),
timings: {
enter: start,
exit: end,
upstream: end - start,
},
};
};

47
src/sources/saerro.ts
View file

@ -1,17 +1,21 @@
import { Cache } from "../cache";
import { Population, ServiceResponse } from "../types";
interface OneResponse {
interface SaerroResponse {
data: {
world: {
id: string;
allWorlds: {
id: number;
population: Population<number>;
};
}[];
};
}
export const saerroFetchWorld = async (
id: string
): Promise<ServiceResponse<number, OneResponse>> => {
const saerroFetchAllWorlds = async (cache: Cache): Promise<SaerroResponse> => {
const cached = await cache.get<SaerroResponse>("saerro");
if (cached) {
return cached;
}
const req = await fetch(`https://saerro.ps2.live/graphql`, {
method: "POST",
headers: {
@ -19,7 +23,7 @@ export const saerroFetchWorld = async (
},
body: JSON.stringify({
query: `{
world(by: {id: ${id}}) {
allWorlds {
id
population {
total
@ -32,11 +36,32 @@ export const saerroFetchWorld = async (
}),
});
const json: OneResponse = await req.json();
return await cache.put("saerro", await req.json<SaerroResponse>());
};
export const saerroFetchWorld = async (
id: string,
cache: Cache
): Promise<ServiceResponse<number, SaerroResponse["data"]["allWorlds"][1]>> => {
const start = Date.now();
const json: SaerroResponse = await saerroFetchAllWorlds(cache);
const end = Date.now();
const world = json.data.allWorlds.find((w) => w.id === Number(id));
if (!world) {
throw new Error(`World ${id} not found`);
}
return {
population: json.data.world.population,
raw: json,
population: world.population,
raw: world,
cachedAt: new Date(),
timings: {
enter: start,
exit: end,
upstream: end - start,
},
};
};

78
src/sources/voidwell.ts
View file

@ -1,6 +1,7 @@
import { Cache } from "../cache";
import { ServiceResponse } from "../types";
interface VoidwellResponse {
type VoidwellResponse = Array<{
id: number;
name: string;
isOnline: boolean;
@ -22,39 +23,76 @@ interface VoidwellResponse {
ns: number;
}[];
};
}
}>;
const platform = (worldID: string) => {
switch (worldID) {
case "1000":
return "ps4us";
case "2000":
return "ps4eu";
default:
return "pc";
const voidwellFetchAllWorlds = async (
cache: Cache
): Promise<VoidwellResponse> => {
const cached = await cache.get<VoidwellResponse>("voidwell");
if (cached) {
return cached;
}
const [pc, ps4us, ps4eu] = await Promise.all([
fetch(`https://api.voidwell.com/ps2/worldstate/?platform=pc`)
.then((res) => res.json<VoidwellResponse>())
.catch((e) => {
console.error("voidwell PC ERROR", e);
return [] as VoidwellResponse;
}),
fetch(`https://api.voidwell.com/ps2/worldstate/?platform=ps4us`)
.then((res) => res.json<VoidwellResponse>())
.catch((e) => {
console.error("voidwell PS4US ERROR", e);
return [] as VoidwellResponse;
}),
fetch(`https://api.voidwell.com/ps2/worldstate/?platform=ps4eu`)
.then((res) => res.json<VoidwellResponse>())
.catch((e) => {
console.error("voidwell PS4EU ERROR", e);
return [] as VoidwellResponse;
}),
]);
// console.log("voidwell data fetched", JSON.stringify({ pc, ps4us, ps4eu }));
const response: VoidwellResponse = [
...pc,
...ps4us,
...ps4eu,
] as VoidwellResponse;
return await cache.put("voidwell", response);
};
// Voidwell is missing Oshur, and since zoneStates are the only way we can get a faction-specific population count,
// we're stuck with not counting faction populations.
export const voidwellFetchWorld = async (
worldID: string
): Promise<ServiceResponse<undefined, VoidwellResponse>> => {
const res = await fetch(
`https://api.voidwell.com/ps2/worldstate/${worldID}?platform=${platform(
worldID
)}`
);
const data: VoidwellResponse = await res.json();
worldID: string,
cache: Cache
): Promise<ServiceResponse<undefined, VoidwellResponse[0]>> => {
const start = Date.now();
const data = await voidwellFetchAllWorlds(cache);
const end = Date.now();
const world = data.find((w) => w.id === Number(worldID));
if (!world) {
throw new Error(`voidwell: World ${worldID} not found`);
}
return {
raw: data,
raw: world,
population: {
total: data.onlineCharacters,
total: world.onlineCharacters,
nc: undefined,
tr: undefined,
vs: undefined,
},
cachedAt: new Date(),
timings: {
enter: start,
exit: end,
upstream: end - start,
},
};
};

11
src/types.ts
View file

@ -9,6 +9,11 @@ export interface ServiceResponse<PT extends number | undefined, Raw> {
population: Population<PT>;
raw: Raw;
cachedAt: Date;
timings?: {
enter: number;
upstream: number;
exit: number;
};
}
export interface Env {
@ -43,6 +48,12 @@ export type DebugPayload = {
honu: any;
voidwell: any;
};
timings: {
saerro: any;
fisu: any;
honu: any;
voidwell: any;
};
lastFetchTimes: {
saerro?: Date;
fisu?: Date;

4
wrangler.toml
View file

@ -2,6 +2,10 @@ name = "agg-population"
main = "src/index.ts"
compatibility_date = "2022-12-19"
[route]
pattern = "agg.ps2.live/population*"
zone_name = "ps2.live"
[vars]
DISABLE_HONU = "0"
DISABLE_FISU = "0"