chore: add backend-y bits of heartbeats

packages/api/bindings.d.ts

@@ -10,4 +10,5 @@ declare global {
     const KV_SESSIONS: KVNamespace;
     const KV_GUILDS: KVNamespace;
     const KV_GUILD_DATA: KVNamespace;
+    const KV_INFRASTRUCTURE: KVNamespace;
 }

packages/api/handlers/bot-heartbeat.ts

@@ -0,0 +1,23 @@
+import { BotHeartbeatData } from 'packages/types';
+import { getSharedKey, respond } from '../utils/api-tools';
+import { botHeartbeatToken } from '../utils/config';
+import { Infrastructure } from '../utils/kv';
+
+const denied = () => respond({ error: 'robots only, human.' }, { status: 403 });
+
+export const BotHeartbeat = async (request: Request): Promise<Response> => {
+    const { id } = getSharedKey(request) || {};
+    if (!id || id !== botHeartbeatToken) {
+        return denied();
+    }
+
+    const heartbeat: BotHeartbeatData = await request.json();
+
+    await Infrastructure.put<BotHeartbeatData>(`shard_${heartbeat.shard}`, {
+        lastHeartbeat: Date.now(),
+        state: heartbeat.state,
+        shard: heartbeat.shard,
+    });
+
+    return respond({ status: 'ok' });
+};

packages/api/handlers/get-infrastructure-status.ts

@@ -0,0 +1,85 @@
+import { BotHeartbeatData } from 'packages/types';
+import { respond } from '../utils/api-tools';
+import { Infrastructure } from '../utils/kv';
+
+export const GetInfrastructureStatus = async (request: Request): Promise<Response> => {
+    const cachedStatus = await Infrastructure.get('cached_status');
+    if (cachedStatus) {
+        return respond({ status: cachedStatus });
+    }
+
+    const [shardStatus, discord, cloudflare] = await Promise.all([
+        getAllShardStatus(),
+        getDiscordStatus(),
+        getCloudflareStatus(),
+    ]);
+
+    const status = {
+        shards: shardStatus,
+        discord,
+        cloudflare,
+    };
+
+    await Infrastructure.put('cached_status', status, 60 * 5);
+
+    return respond({
+        status,
+    });
+};
+
+const getAllShardStatus = async (): Promise<BotHeartbeatData[]> => {
+    const shards: BotHeartbeatData[] = [];
+    let currentShard = 0;
+
+    do {
+        const thisShard = await getShardStatus(currentShard);
+        if (!thisShard) {
+            break;
+        }
+
+        shards.push(thisShard);
+        currentShard++;
+    } while (currentShard < 64); // If we hit 64, there's bigger issues.
+
+    return shards;
+};
+
+const getShardStatus = async (shardNumber: number): Promise<BotHeartbeatData | null> => {
+    const shardHeartbeat = await Infrastructure.get<BotHeartbeatData>(
+        `shard_${shardNumber}`
+    );
+    if (!shardHeartbeat) {
+        return null;
+    }
+
+    return shardHeartbeat;
+};
+
+const getDiscordStatus = async () => {
+    const req = await fetch('https://srhpyqt94yxb.statuspage.io/api/v2/summary.json');
+    const status = await req.json();
+
+    const api =
+        status.components.find(
+            (component: { id: string }) => component.id === 'rhznvxg4v7yh'
+        )?.status !== 'operational';
+
+    return { api };
+};
+
+const getCloudflareStatus = async () => {
+    const req = await fetch('https://yh6f0r4529hb.statuspage.io/api/v2/summary.json');
+    const status = await req.json();
+
+    const workers =
+        status.components.find(
+            (component: { id: string }) => component.id === '57srcl8zcn7c'
+        )?.status !== 'operational';
+
+    const storage =
+        status.components.find(
+            (component: { id: string }) => component.id === 'tmh50tx2nprs'
+        )?.status !== 'operational';
+
+    return { workers, storage };
+};

packages/api/index.ts

@@ -1,5 +1,7 @@
+import { BotHeartbeat } from './handlers/bot-heartbeat';
 import { BotJoin } from './handlers/bot-join';
 import { CreateRoleypolyData } from './handlers/create-roleypoly-data';
+import { GetInfrastructureStatus } from './handlers/get-infrastructure-status';
 import { GetPickerData } from './handlers/get-picker-data';
 import { GetSession } from './handlers/get-session';
 import { GetSlug } from './handlers/get-slug';
@@ -13,6 +15,9 @@ import { uiPublicURI } from './utils/config';
 
 const router = new Router();
 
+// Public
+router.add('GET', 'get-infrastructure-status', GetInfrastructureStatus);
+
 // OAuth
 router.add('GET', 'bot-join', BotJoin);
 router.add('GET', 'login-bounce', LoginBounce);
@@ -30,6 +35,9 @@ router.add('PATCH', 'update-roles', UpdateRoles);
 // Root users only
 router.add('GET', 'x-create-roleypoly-data', CreateRoleypolyData);
 
+// Internal routes
+router.add('PUT', 'bot-heartbeat', BotHeartbeat);
+
 // Tester Routes
 router.add('GET', 'x-headers', (request) => {
     const headers: { [x: string]: string } = {};

packages/api/utils/api-tools.ts

@@ -69,6 +69,20 @@ export const getSessionID = (request: Request): { type: string; id: string } | n
     return { type, id };
 };
 
+export const getSharedKey = (request: Request): { type: string; id: string } | null => {
+    const sharedKey = request.headers.get('authorization');
+    if (!sharedKey) {
+        return null;
+    }
+
+    const [type, id] = sharedKey.split(' ');
+    if (type !== 'Shared') {
+        return null;
+    }
+
+    return { type, id };
+};
+
 export const userAgent =
     'DiscordBot (https://github.com/roleypoly/roleypoly, git-main) (+https://roleypoly.com)';
 

packages/api/utils/config.ts

@@ -8,6 +8,7 @@ const list = (x: string) => x.split(',');
 export const botClientID = env('BOT_CLIENT_ID');
 export const botClientSecret = env('BOT_CLIENT_SECRET');
 export const botToken = env('BOT_TOKEN');
+export const botHeartbeatToken = env('BOT_HEARTBEAT_TOKEN');
 export const uiPublicURI = safeURI(env('UI_PUBLIC_URI'));
 export const apiPublicURI = safeURI(env('API_PUBLIC_URI'));
 export const rootUsers = list(env('ROOT_USERS'));

packages/api/utils/kv.ts

@@ -88,3 +88,6 @@ const self = (global as any) as Record<string, any>;
 export const Sessions = new WrappedKVNamespace(kvOrLocal(self.KV_SESSIONS ?? null));
 export const GuildData = new WrappedKVNamespace(kvOrLocal(self.KV_GUILD_DATA ?? null));
 export const Guilds = new WrappedKVNamespace(kvOrLocal(self.KV_GUILDS ?? null));
+export const Infrastructure = new WrappedKVNamespace(
+    kvOrLocal(self.KV_INFRASTRUCTURE ?? null)
+);

packages/api/worker.config.js

@@ -7,9 +7,10 @@ module.exports = {
         'BOT_CLIENT_ID',
         'BOT_CLIENT_SECRET',
         'BOT_TOKEN',
+        'BOT_HEARTBEAT_TOKEN',
         'UI_PUBLIC_URI',
         'API_PUBLIC_URI',
         'ROOT_USERS',
     ]),
-    kv: ['KV_SESSIONS', 'KV_GUILDS', 'KV_GUILD_DATA'],
+    kv: ['KV_SESSIONS', 'KV_GUILDS', 'KV_GUILD_DATA', 'KV_INFRASTRUCTURE'],
 };

packages/types/Internal.ts

@@ -0,0 +1,5 @@
+export type BotHeartbeatData = {
+    shard: number;
+    state: 'ok' | 'degraded' | 'failed';
+    lastHeartbeat?: number; // Date.now()
+};

packages/types/index.ts

@@ -1,5 +1,6 @@
 export * from './Category';
 export * from './Guild';
+export * from './Internal';
 export * from './Role';
 export * from './Session';
 export * from './User';

terraform/bot.tf

@@ -3,6 +3,14 @@ locals {
   botRegion = var.gcp_region
 }
 
+resource "random_password" "bot_heartbeat_token" {
+  length = 64
+  keepers = {
+    vmName = locals.vmName // Always regenerate this on a new deploy.
+    envtag = var.environment_tag
+  }
+}
+
 data "google_compute_zones" "gcp_zones" {
   region = local.botRegion
   status = "UP"
@@ -40,9 +48,17 @@ locals {
         name  = "BOT_CLIENT_ID",
         value = var.bot_client_id
       },
+      {
+        name  = "BOT_HEARTBEAT_TOKEN",
+        value = resource.random_password.bot_heartbeat_token.result
+      },
       {
         name  = "UI_PUBLIC_URI",
         value = var.ui_public_uri
+      },
+      {
+        name  = "API_PUBLIC_URI",
+        value = var.api_public_uri
       }
     ]
   }
@@ -88,4 +104,4 @@ resource "google_compute_instance" "bot" {
   labels = {
     container-vm = module.gce_container.vm_container_label
   }
-}
+}

terraform/workers.tf

@@ -10,6 +10,10 @@ resource "cloudflare_workers_kv_namespace" "guild_data" {
   title = "roleypoly-guild_data-${var.environment_tag}"
 }
 
+resource "cloudflare_workers_kv_namespace" "infrastructure" {
+  title = "roleypoly-infrastructure-${var.environment_tag}"
+}
+
 resource "cloudflare_worker_script" "backend" {
   name    = "roleypoly-backend-${var.environment_tag}"
   content = file("${path.module}/${var.api_path_to_worker}")
@@ -29,6 +33,11 @@ resource "cloudflare_worker_script" "backend" {
     namespace_id = cloudflare_workers_kv_namespace.guild_data.id
   }
 
+  kv_namespace_binding {
+    name         = "KV_INFRASTRUCTURE"
+    namespace_id = cloudflare_workers_kv_namespace.infrastructure.id
+  }
+
   plain_text_binding {
     name = "BOT_CLIENT_ID"
     text = var.bot_client_id
@@ -44,6 +53,11 @@ resource "cloudflare_worker_script" "backend" {
     text = var.bot_token
   }
 
+  secret_text_binding {
+    name = "BOT_HEARTBEAT_TOKEN"
+    text = resource.random_password.bot_heartbeat_token.result
+  }
+
   plain_text_binding {
     name = "UI_PUBLIC_URI"
     text = var.ui_public_uri