From a34aebabe33a85719819f30972c23d51695fdeeb Mon Sep 17 00:00:00 2001 From: Katalina Okano Date: Fri, 12 Mar 2021 20:45:35 -0500 Subject: [PATCH] chore: add backend-y bits of heartbeats --- packages/api/bindings.d.ts | 1 + packages/api/handlers/bot-heartbeat.ts | 23 +++++ .../api/handlers/get-infrastructure-status.ts | 85 +++++++++++++++++++ packages/api/index.ts | 8 ++ packages/api/utils/api-tools.ts | 14 +++ packages/api/utils/config.ts | 1 + packages/api/utils/kv.ts | 3 + packages/api/worker.config.js | 3 +- packages/types/Internal.ts | 5 ++ packages/types/index.ts | 1 + terraform/bot.tf | 18 +++- terraform/workers.tf | 14 +++ 12 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 packages/api/handlers/bot-heartbeat.ts create mode 100644 packages/api/handlers/get-infrastructure-status.ts create mode 100644 packages/types/Internal.ts diff --git a/packages/api/bindings.d.ts b/packages/api/bindings.d.ts index 342afdd..30ceb23 100644 --- a/packages/api/bindings.d.ts +++ b/packages/api/bindings.d.ts @@ -10,4 +10,5 @@ declare global { const KV_SESSIONS: KVNamespace; const KV_GUILDS: KVNamespace; const KV_GUILD_DATA: KVNamespace; + const KV_INFRASTRUCTURE: KVNamespace; } diff --git a/packages/api/handlers/bot-heartbeat.ts b/packages/api/handlers/bot-heartbeat.ts new file mode 100644 index 0000000..51d05a8 --- /dev/null +++ b/packages/api/handlers/bot-heartbeat.ts @@ -0,0 +1,23 @@ +import { BotHeartbeatData } from 'packages/types'; +import { getSharedKey, respond } from '../utils/api-tools'; +import { botHeartbeatToken } from '../utils/config'; +import { Infrastructure } from '../utils/kv'; + +const denied = () => respond({ error: 'robots only, human.' }, { status: 403 }); + +export const BotHeartbeat = async (request: Request): Promise => { + const { id } = getSharedKey(request) || {}; + if (!id || id !== botHeartbeatToken) { + return denied(); + } + + const heartbeat: BotHeartbeatData = await request.json(); + + await Infrastructure.put(`shard_${heartbeat.shard}`, { + lastHeartbeat: Date.now(), + state: heartbeat.state, + shard: heartbeat.shard, + }); + + return respond({ status: 'ok' }); +}; diff --git a/packages/api/handlers/get-infrastructure-status.ts b/packages/api/handlers/get-infrastructure-status.ts new file mode 100644 index 0000000..4cc07bc --- /dev/null +++ b/packages/api/handlers/get-infrastructure-status.ts @@ -0,0 +1,85 @@ +import { BotHeartbeatData } from 'packages/types'; +import { respond } from '../utils/api-tools'; +import { Infrastructure } from '../utils/kv'; + +export const GetInfrastructureStatus = async (request: Request): Promise => { + const cachedStatus = await Infrastructure.get('cached_status'); + if (cachedStatus) { + return respond({ status: cachedStatus }); + } + + const [shardStatus, discord, cloudflare] = await Promise.all([ + getAllShardStatus(), + getDiscordStatus(), + getCloudflareStatus(), + ]); + + const status = { + shards: shardStatus, + discord, + cloudflare, + }; + + await Infrastructure.put('cached_status', status, 60 * 5); + + return respond({ + status, + }); +}; + +const getAllShardStatus = async (): Promise => { + const shards: BotHeartbeatData[] = []; + let currentShard = 0; + + do { + const thisShard = await getShardStatus(currentShard); + if (!thisShard) { + break; + } + + shards.push(thisShard); + currentShard++; + } while (currentShard < 64); // If we hit 64, there's bigger issues. + + return shards; +}; + +const getShardStatus = async (shardNumber: number): Promise => { + const shardHeartbeat = await Infrastructure.get( + `shard_${shardNumber}` + ); + if (!shardHeartbeat) { + return null; + } + + return shardHeartbeat; +}; + +const getDiscordStatus = async () => { + const req = await fetch('https://srhpyqt94yxb.statuspage.io/api/v2/summary.json'); + const status = await req.json(); + + const api = + status.components.find( + (component: { id: string }) => component.id === 'rhznvxg4v7yh' + )?.status !== 'operational'; + + return { api }; +}; + +const getCloudflareStatus = async () => { + const req = await fetch('https://yh6f0r4529hb.statuspage.io/api/v2/summary.json'); + const status = await req.json(); + + const workers = + status.components.find( + (component: { id: string }) => component.id === '57srcl8zcn7c' + )?.status !== 'operational'; + + const storage = + status.components.find( + (component: { id: string }) => component.id === 'tmh50tx2nprs' + )?.status !== 'operational'; + + return { workers, storage }; +}; diff --git a/packages/api/index.ts b/packages/api/index.ts index 6db467b..43f5132 100644 --- a/packages/api/index.ts +++ b/packages/api/index.ts @@ -1,5 +1,7 @@ +import { BotHeartbeat } from './handlers/bot-heartbeat'; import { BotJoin } from './handlers/bot-join'; import { CreateRoleypolyData } from './handlers/create-roleypoly-data'; +import { GetInfrastructureStatus } from './handlers/get-infrastructure-status'; import { GetPickerData } from './handlers/get-picker-data'; import { GetSession } from './handlers/get-session'; import { GetSlug } from './handlers/get-slug'; @@ -13,6 +15,9 @@ import { uiPublicURI } from './utils/config'; const router = new Router(); +// Public +router.add('GET', 'get-infrastructure-status', GetInfrastructureStatus); + // OAuth router.add('GET', 'bot-join', BotJoin); router.add('GET', 'login-bounce', LoginBounce); @@ -30,6 +35,9 @@ router.add('PATCH', 'update-roles', UpdateRoles); // Root users only router.add('GET', 'x-create-roleypoly-data', CreateRoleypolyData); +// Internal routes +router.add('PUT', 'bot-heartbeat', BotHeartbeat); + // Tester Routes router.add('GET', 'x-headers', (request) => { const headers: { [x: string]: string } = {}; diff --git a/packages/api/utils/api-tools.ts b/packages/api/utils/api-tools.ts index f251788..f09d5e7 100644 --- a/packages/api/utils/api-tools.ts +++ b/packages/api/utils/api-tools.ts @@ -69,6 +69,20 @@ export const getSessionID = (request: Request): { type: string; id: string } | n return { type, id }; }; +export const getSharedKey = (request: Request): { type: string; id: string } | null => { + const sharedKey = request.headers.get('authorization'); + if (!sharedKey) { + return null; + } + + const [type, id] = sharedKey.split(' '); + if (type !== 'Shared') { + return null; + } + + return { type, id }; +}; + export const userAgent = 'DiscordBot (https://github.com/roleypoly/roleypoly, git-main) (+https://roleypoly.com)'; diff --git a/packages/api/utils/config.ts b/packages/api/utils/config.ts index c089a1e..33a0654 100644 --- a/packages/api/utils/config.ts +++ b/packages/api/utils/config.ts @@ -8,6 +8,7 @@ const list = (x: string) => x.split(','); export const botClientID = env('BOT_CLIENT_ID'); export const botClientSecret = env('BOT_CLIENT_SECRET'); export const botToken = env('BOT_TOKEN'); +export const botHeartbeatToken = env('BOT_HEARTBEAT_TOKEN'); export const uiPublicURI = safeURI(env('UI_PUBLIC_URI')); export const apiPublicURI = safeURI(env('API_PUBLIC_URI')); export const rootUsers = list(env('ROOT_USERS')); diff --git a/packages/api/utils/kv.ts b/packages/api/utils/kv.ts index b1b8e2e..eb850b9 100644 --- a/packages/api/utils/kv.ts +++ b/packages/api/utils/kv.ts @@ -88,3 +88,6 @@ const self = (global as any) as Record; export const Sessions = new WrappedKVNamespace(kvOrLocal(self.KV_SESSIONS ?? null)); export const GuildData = new WrappedKVNamespace(kvOrLocal(self.KV_GUILD_DATA ?? null)); export const Guilds = new WrappedKVNamespace(kvOrLocal(self.KV_GUILDS ?? null)); +export const Infrastructure = new WrappedKVNamespace( + kvOrLocal(self.KV_INFRASTRUCTURE ?? null) +); diff --git a/packages/api/worker.config.js b/packages/api/worker.config.js index fe82c13..4173007 100644 --- a/packages/api/worker.config.js +++ b/packages/api/worker.config.js @@ -7,9 +7,10 @@ module.exports = { 'BOT_CLIENT_ID', 'BOT_CLIENT_SECRET', 'BOT_TOKEN', + 'BOT_HEARTBEAT_TOKEN', 'UI_PUBLIC_URI', 'API_PUBLIC_URI', 'ROOT_USERS', ]), - kv: ['KV_SESSIONS', 'KV_GUILDS', 'KV_GUILD_DATA'], + kv: ['KV_SESSIONS', 'KV_GUILDS', 'KV_GUILD_DATA', 'KV_INFRASTRUCTURE'], }; diff --git a/packages/types/Internal.ts b/packages/types/Internal.ts new file mode 100644 index 0000000..3f9cd30 --- /dev/null +++ b/packages/types/Internal.ts @@ -0,0 +1,5 @@ +export type BotHeartbeatData = { + shard: number; + state: 'ok' | 'degraded' | 'failed'; + lastHeartbeat?: number; // Date.now() +}; diff --git a/packages/types/index.ts b/packages/types/index.ts index 6d88788..4880ebf 100644 --- a/packages/types/index.ts +++ b/packages/types/index.ts @@ -1,5 +1,6 @@ export * from './Category'; export * from './Guild'; +export * from './Internal'; export * from './Role'; export * from './Session'; export * from './User'; diff --git a/terraform/bot.tf b/terraform/bot.tf index a6bf2ba..3c250f2 100644 --- a/terraform/bot.tf +++ b/terraform/bot.tf @@ -3,6 +3,14 @@ locals { botRegion = var.gcp_region } +resource "random_password" "bot_heartbeat_token" { + length = 64 + keepers = { + vmName = locals.vmName // Always regenerate this on a new deploy. + envtag = var.environment_tag + } +} + data "google_compute_zones" "gcp_zones" { region = local.botRegion status = "UP" @@ -40,9 +48,17 @@ locals { name = "BOT_CLIENT_ID", value = var.bot_client_id }, + { + name = "BOT_HEARTBEAT_TOKEN", + value = resource.random_password.bot_heartbeat_token.result + }, { name = "UI_PUBLIC_URI", value = var.ui_public_uri + }, + { + name = "API_PUBLIC_URI", + value = var.api_public_uri } ] } @@ -88,4 +104,4 @@ resource "google_compute_instance" "bot" { labels = { container-vm = module.gce_container.vm_container_label } -} \ No newline at end of file +} diff --git a/terraform/workers.tf b/terraform/workers.tf index 9c9546a..3da64a9 100644 --- a/terraform/workers.tf +++ b/terraform/workers.tf @@ -10,6 +10,10 @@ resource "cloudflare_workers_kv_namespace" "guild_data" { title = "roleypoly-guild_data-${var.environment_tag}" } +resource "cloudflare_workers_kv_namespace" "infrastructure" { + title = "roleypoly-infrastructure-${var.environment_tag}" +} + resource "cloudflare_worker_script" "backend" { name = "roleypoly-backend-${var.environment_tag}" content = file("${path.module}/${var.api_path_to_worker}") @@ -29,6 +33,11 @@ resource "cloudflare_worker_script" "backend" { namespace_id = cloudflare_workers_kv_namespace.guild_data.id } + kv_namespace_binding { + name = "KV_INFRASTRUCTURE" + namespace_id = cloudflare_workers_kv_namespace.infrastructure.id + } + plain_text_binding { name = "BOT_CLIENT_ID" text = var.bot_client_id @@ -44,6 +53,11 @@ resource "cloudflare_worker_script" "backend" { text = var.bot_token } + secret_text_binding { + name = "BOT_HEARTBEAT_TOKEN" + text = resource.random_password.bot_heartbeat_token.result + } + plain_text_binding { name = "UI_PUBLIC_URI" text = var.ui_public_uri