big overhaul (#474)

* miniflare init

* feat(api): add tests

* chore: more tests, almost 100%

* add sessions/state spec

* add majority of routes and datapaths, start on interactions

* nevermind, no interactions

* nevermind x2, tweetnacl is bad but SubtleCrypto has what we need apparently

* simplify interactions verify

* add brute force interactions tests

* every primary path API route is refactored!

* automatically import from legacy, or die trying.

* check that we only fetch legacy once, ever

* remove old-src, same some historic pieces

* remove interactions & worker-utils package, update misc/types

* update some packages we don't need specific pinning for anymore

* update web references to API routes since they all changed

* fix all linting issues, upgrade most packages

* fix tests, divorce enzyme where-ever possible

* update web, fix integration issues

* pre-build api

* fix tests

* move api pretest to api package.json instead of CI

* remove interactions from terraform, fix deploy side configs

* update to tf 1.1.4

* prevent double writes to worker in GCS, port to newer GCP auth workflow

* fix api.tf var refs, upgrade node action

* change to curl-based script upload for worker script due to terraform provider limitations

* oh no, cloudflare freaked out :(

packages/api/src/utils/config.ts

@@ -0,0 +1,78 @@
+import { WrappedKVNamespace } from './kv';
+
+export type Environment = {
+  BOT_CLIENT_ID: string;
+  BOT_CLIENT_SECRET: string;
+  BOT_TOKEN: string;
+  UI_PUBLIC_URI: string;
+  API_PUBLIC_URI: string;
+  ROOT_USERS: string;
+  ALLOWED_CALLBACK_HOSTS: string;
+  BOT_IMPORT_TOKEN: string;
+  INTERACTIONS_SHARED_KEY: string;
+  RP_SERVER_ID: string;
+  RP_HELPER_ROLE_IDS: string;
+  DISCORD_PUBLIC_KEY: string;
+  KV_SESSIONS: KVNamespace;
+  KV_GUILDS: KVNamespace;
+  KV_GUILD_DATA: KVNamespace;
+};
+
+export type Config = {
+  botClientID: string;
+  botClientSecret: string;
+  botToken: string;
+  publicKey: string;
+  uiPublicURI: string;
+  apiPublicURI: string;
+  rootUsers: string[];
+  allowedCallbackHosts: string[];
+  importSharedKey: string;
+  interactionsSharedKey: string;
+  roleypolyServerID: string;
+  helperRoleIDs: string[];
+  kv: {
+    sessions: WrappedKVNamespace;
+    guilds: WrappedKVNamespace;
+    guildData: WrappedKVNamespace;
+  };
+  retention: {
+    session: number;
+    sessionState: number;
+    guild: number;
+    member: number;
+  };
+  _raw: Environment;
+};
+
+const toList = (x: string): string[] => String(x).split(',');
+const safeURI = (x: string) => String(x).replace(/\/$/, '');
+
+export const parseEnvironment = (env: Environment): Config => {
+  return {
+    _raw: env,
+    botClientID: env.BOT_CLIENT_ID,
+    botClientSecret: env.BOT_CLIENT_SECRET,
+    botToken: env.BOT_TOKEN,
+    publicKey: env.DISCORD_PUBLIC_KEY,
+    uiPublicURI: safeURI(env.UI_PUBLIC_URI),
+    apiPublicURI: safeURI(env.API_PUBLIC_URI),
+    rootUsers: toList(env.ROOT_USERS),
+    allowedCallbackHosts: toList(env.ALLOWED_CALLBACK_HOSTS),
+    importSharedKey: env.BOT_IMPORT_TOKEN,
+    interactionsSharedKey: env.INTERACTIONS_SHARED_KEY,
+    roleypolyServerID: env.RP_SERVER_ID,
+    helperRoleIDs: toList(env.RP_HELPER_ROLE_IDS),
+    kv: {
+      sessions: new WrappedKVNamespace(env.KV_SESSIONS),
+      guilds: new WrappedKVNamespace(env.KV_GUILDS),
+      guildData: new WrappedKVNamespace(env.KV_GUILD_DATA),
+    },
+    retention: {
+      session: 60 * 60 * 6, // 6 hours
+      sessionState: 60 * 5, // 5 minutes
+      guild: 60 * 60 * 2, // 2 hours
+      member: 60 * 5, // 5 minutes
+    },
+  };
+};

packages/api/src/utils/context.ts

@@ -0,0 +1,40 @@
+import { Config } from '@roleypoly/api/src/utils/config';
+import { SessionData } from '@roleypoly/types';
+
+export type AuthMode =
+  | {
+      type: 'anonymous';
+    }
+  | {
+      type: 'bearer';
+      sessionId: string;
+    }
+  | {
+      type: 'bot';
+      identity: string;
+    };
+
+export type Context = {
+  config: Config;
+  fetchContext: {
+    waitUntil: FetchEvent['waitUntil'];
+  };
+  authMode: AuthMode;
+  params: {
+    guildId?: string;
+    memberId?: string;
+  };
+
+  // Must include withSession middleware for population
+  session?: SessionData;
+};
+
+export type RoleypolyHandler = (
+  request: Request,
+  context: Context
+) => Promise<Response> | Response;
+
+export type RoleypolyMiddleware = (
+  request: Request,
+  context: Context
+) => Promise<Response | void> | Response | void;

packages/api/src/utils/discord.spec.ts

@@ -0,0 +1,34 @@
+import { getHighestRole } from './discord';
+
+describe('getHighestRole', () => {
+  it('returns the highest role', () => {
+    const roles = [
+      {
+        id: 'role-1',
+        name: 'Role 1',
+        color: 0,
+        position: 17,
+        permissions: '',
+        managed: false,
+      },
+      {
+        id: 'role-2',
+        name: 'Role 2',
+        color: 0,
+        position: 2,
+        permissions: '',
+        managed: false,
+      },
+      {
+        id: 'role-3',
+        name: 'Role 3',
+        color: 0,
+        position: 19,
+        permissions: '',
+        managed: false,
+      },
+    ];
+
+    expect(getHighestRole(roles)).toEqual(roles[2]);
+  });
+});

packages/api/src/utils/discord.ts

@@ -0,0 +1,153 @@
+import {
+  evaluatePermission,
+  permissions as Permissions,
+} from '@roleypoly/misc-utils/hasPermission';
+import {
+  AuthTokenResponse,
+  DiscordUser,
+  GuildSlug,
+  Role,
+  UserGuildPermissions,
+} from '@roleypoly/types';
+
+export const userAgent =
+  'DiscordBot (https://github.com/roleypoly/roleypoly, git-main) (+https://roleypoly.com)';
+
+export const discordAPIBase = 'https://discord.com/api/v9';
+
+export enum AuthType {
+  Bearer = 'Bearer',
+  Bot = 'Bot',
+  None = 'None',
+}
+
+export const discordFetch = async <T>(
+  url: string,
+  auth: string,
+  authType: AuthType = AuthType.Bearer,
+  init?: RequestInit
+): Promise<T | null> => {
+  const response = await fetch(discordAPIBase + url, {
+    ...(init || {}),
+    headers: {
+      ...(init?.headers || {}),
+      ...(authType !== AuthType.None
+        ? {
+            authorization: `${AuthType[authType]} ${auth}`,
+          }
+        : {}),
+      'user-agent': userAgent,
+    },
+  });
+
+  if (response.status >= 400) {
+    console.error('discordFetch failed', {
+      url,
+      authType,
+      payload: await response.text(),
+    });
+  }
+
+  if (response.ok) {
+    return (await response.json()) as T;
+  } else {
+    return null;
+  }
+};
+
+export const getTokenUser = async (
+  accessToken: AuthTokenResponse['access_token']
+): Promise<DiscordUser | null> => {
+  const user = await discordFetch<DiscordUser>(
+    '/users/@me',
+    accessToken,
+    AuthType.Bearer
+  );
+
+  if (!user) {
+    return null;
+  }
+
+  const { id, username, discriminator, bot, avatar } = user;
+
+  return { id, username, discriminator, bot, avatar };
+};
+
+type UserGuildsPayload = {
+  id: string;
+  name: string;
+  icon: string;
+  owner: boolean;
+  permissions: number;
+  features: string[];
+}[];
+
+export const getTokenGuilds = async (accessToken: string) => {
+  const guilds = await discordFetch<UserGuildsPayload>(
+    '/users/@me/guilds',
+    accessToken,
+    AuthType.Bearer
+  );
+
+  if (!guilds) {
+    return [];
+  }
+
+  const guildSlugs = guilds.map<GuildSlug>((guild) => ({
+    id: guild.id,
+    name: guild.name,
+    icon: guild.icon,
+    permissionLevel: parsePermissions(BigInt(guild.permissions), guild.owner),
+  }));
+
+  return guildSlugs;
+};
+
+export type APIGuild = {
+  // Only relevant stuff
+  id: string;
+  name: string;
+  icon: string;
+  roles: APIRole[];
+};
+
+export type APIRole = {
+  id: string;
+  name: string;
+  color: number;
+  position: number;
+  permissions: string;
+  managed: boolean;
+};
+
+export type APIMember = {
+  // Only relevant stuff, again.
+  roles: string[];
+  pending: boolean;
+  nick: string;
+  user: {
+    id: string;
+  };
+};
+
+export const parsePermissions = (
+  permissions: bigint,
+  owner: boolean = false
+): UserGuildPermissions => {
+  if (owner || evaluatePermission(permissions, Permissions.ADMINISTRATOR)) {
+    return UserGuildPermissions.Admin;
+  }
+
+  if (evaluatePermission(permissions, Permissions.MANAGE_ROLES)) {
+    return UserGuildPermissions.Manager;
+  }
+
+  return UserGuildPermissions.User;
+};
+
+export const getHighestRole = (roles: (Role | APIRole)[]): Role | APIRole => {
+  return roles.reduce(
+    (highestRole, role) => (highestRole.position > role.position ? highestRole : role),
+    roles[0]
+  );
+};

packages/api/src/utils/id.spec.ts

@@ -0,0 +1,9 @@
+import { dateFromID, getID } from './id';
+
+it('returns an id', () => {
+  expect(getID()).toBeTruthy();
+});
+
+it('outputs a valid millisecond decoded from id', () => {
+  expect(dateFromID(getID())).toBeCloseTo(Date.now(), Date.now().toString.length - 4);
+});

packages/api/src/utils/id.ts

@@ -0,0 +1,6 @@
+import { decodeTime, monotonicFactory } from 'ulid-workers';
+
+const ulid = monotonicFactory();
+
+export const getID = () => ulid();
+export const dateFromID = (id: string) => decodeTime(id);

packages/api/src/utils/kv.spec.ts

@@ -0,0 +1,91 @@
+import { configContext } from './testHelpers';
+
+it('serializes data via get and put', async () => {
+  const [config] = configContext();
+
+  const data = {
+    foo: 'bar',
+    baz: 'qux',
+  };
+
+  await config.kv.guilds.put('test-guild-id', data, config.retention.guild);
+
+  const result = await config.kv.guilds.get('test-guild-id');
+
+  expect(result).toEqual(data);
+});
+
+describe('cacheThrough', () => {
+  it('passes through for data on misses', async () => {
+    const [config] = configContext();
+
+    const data = {
+      foo: 'bar',
+      baz: 'qux',
+    };
+    const testFn = jest.fn();
+    const result = await config.kv.guilds.cacheThrough(
+      'test-guild-id',
+      async () => {
+        testFn();
+        return data;
+      },
+      config.retention.guild
+    );
+
+    expect(testFn).toHaveBeenCalledTimes(1);
+    expect(result).toEqual(data);
+  });
+
+  it('uses cache data on hits', async () => {
+    const [config] = configContext();
+
+    const data = {
+      foo: 'bar',
+      baz: 'qux',
+    };
+    const testFn = jest.fn();
+    await config.kv.guilds.put('test-guild-id', data, config.retention.guild);
+
+    const result = await config.kv.guilds.cacheThrough(
+      'test-guild-id',
+      async () => {
+        testFn();
+        return data;
+      },
+      config.retention.guild
+    );
+
+    expect(testFn).not.toHaveBeenCalled();
+    expect(result).toEqual(data);
+  });
+
+  it('skips cache when instructed to miss', async () => {
+    const [config] = configContext();
+
+    const data = {
+      foo: 'bar',
+      baz: 'qux',
+    };
+    const testFn = jest.fn();
+    await config.kv.guilds.put('test-guild-id', data, config.retention.guild);
+
+    const run = (skip: boolean) => {
+      return config.kv.guilds.cacheThrough(
+        'test-guild-id',
+        async () => {
+          testFn();
+          return data;
+        },
+        config.retention.guild,
+        skip
+      );
+    };
+
+    await run(true);
+    await run(true);
+    await run(false); // use cache this time
+
+    expect(testFn).toHaveBeenCalledTimes(2);
+  });
+});

packages/api/src/utils/kv.ts

@@ -0,0 +1,63 @@
+export class WrappedKVNamespace {
+  constructor(private kvNamespace: KVNamespace) {
+    this.getRaw = kvNamespace.get.bind(kvNamespace);
+    this.putRaw = kvNamespace.put.bind(kvNamespace);
+    this.delete = kvNamespace.delete.bind(kvNamespace);
+    this.list = kvNamespace.list.bind(kvNamespace);
+    this.getWithMetadata = kvNamespace.getWithMetadata.bind(kvNamespace);
+  }
+
+  async get<T>(key: string): Promise<T | null> {
+    const data = await this.kvNamespace.get(key, 'text');
+    if (!data) {
+      return null;
+    }
+
+    return JSON.parse(data) as T;
+  }
+
+  async put<T>(key: string, value: T, ttlSeconds?: number) {
+    await this.kvNamespace.put(key, JSON.stringify(value), {
+      expirationTtl: ttlSeconds,
+    });
+  }
+
+  async cacheThrough<Data>(
+    cacheKey: string,
+    missHandler: () => Promise<Data | null>,
+    retention?: number,
+    forceMiss?: boolean
+  ): Promise<Data | null> {
+    if (!forceMiss) {
+      const value = await this.get<Data>(cacheKey);
+      if (value) {
+        return value;
+      }
+    }
+
+    const fallbackValue = await missHandler();
+    if (!fallbackValue) {
+      return null;
+    }
+
+    await this.put(cacheKey, fallbackValue, retention);
+
+    return fallbackValue;
+  }
+
+  public getRaw: (
+    ...args: Parameters<KVNamespace['get']>
+  ) => ReturnType<KVNamespace['get']>;
+  public putRaw: (
+    ...args: Parameters<KVNamespace['put']>
+  ) => ReturnType<KVNamespace['put']>;
+  public list: (
+    ...args: Parameters<KVNamespace['list']>
+  ) => ReturnType<KVNamespace['list']>;
+  public getWithMetadata: (
+    ...args: Parameters<KVNamespace['getWithMetadata']>
+  ) => ReturnType<KVNamespace['getWithMetadata']>;
+  public delete: (
+    ...args: Parameters<KVNamespace['delete']>
+  ) => ReturnType<KVNamespace['delete']>;
+}

packages/api/src/utils/legacy.ts

@@ -0,0 +1,69 @@
+import { Config } from '@roleypoly/api/src/utils/config';
+import { getID } from '@roleypoly/api/src/utils/id';
+import { sortBy } from '@roleypoly/misc-utils/sortBy';
+import { CategoryType, Features, GuildData } from '@roleypoly/types';
+
+export type LegacyCategory = {
+  id: string;
+  name: string;
+  roles: string[];
+  hidden: boolean;
+  type: 'single' | 'multi';
+  position: number;
+};
+
+export type LegacyGuildData = {
+  id: string;
+  categories: LegacyCategory[];
+  message: string;
+};
+
+export const fetchLegacyServer = async (
+  config: Config,
+  id: string
+): Promise<LegacyGuildData | null> => {
+  if (!config.importSharedKey) {
+    return null;
+  }
+
+  const guildDataResponse = await fetch(
+    `https://beta.roleypoly.com/x/import-to-next/${id}`,
+    {
+      headers: {
+        authorization: `Shared ${config.importSharedKey}`,
+      },
+    }
+  );
+
+  if (guildDataResponse.status === 404) {
+    return null;
+  }
+
+  if (guildDataResponse.status !== 200) {
+    throw new Error('Guild data fetch failed');
+  }
+
+  return await guildDataResponse.json();
+};
+
+export const transformLegacyGuild = (guild: LegacyGuildData): GuildData => {
+  return {
+    id: guild.id,
+    message: guild.message,
+    features: Features.LegacyGuild,
+    auditLogWebhook: null,
+    accessControl: {
+      allowList: [],
+      blockList: [],
+      blockPending: true,
+    },
+    categories: sortBy(Object.values(guild.categories), 'position').map(
+      (category, idx) => ({
+        ...category,
+        id: getID(),
+        position: idx, // Reset positions by index. May have side-effects but oh well.
+        type: category.type === 'multi' ? CategoryType.Multi : CategoryType.Single,
+      })
+    ),
+  };
+};

packages/api/src/utils/request.spec.ts

@@ -0,0 +1,45 @@
+import { formData, formDataRequest, getQuery } from './request';
+
+describe('getQuery', () => {
+  it('splits query string into object', () => {
+    const query = getQuery(new Request('http://local.test/?a=1&b=2'));
+
+    expect(query).toEqual({
+      a: '1',
+      b: '2',
+    });
+  });
+});
+
+describe('formData & formDataRequest', () => {
+  it('formats object into form data', () => {
+    const body = formData({
+      a: 1,
+      b: 2,
+    });
+
+    expect(body).toEqual('a=1&b=2');
+  });
+
+  it('formats object into form data with custom headers', () => {
+    const body = formDataRequest(
+      {
+        a: 1,
+        b: 2,
+      },
+      {
+        headers: {
+          'content-type': 'application/x-www-form-urlencoded',
+        },
+      }
+    );
+
+    expect(body).toEqual({
+      method: 'POST',
+      headers: {
+        'content-type': 'application/x-www-form-urlencoded',
+      },
+      body: 'a=1&b=2',
+    });
+  });
+});

packages/api/src/utils/request.ts

@@ -0,0 +1,42 @@
+import { Context, RoleypolyMiddleware } from '@roleypoly/api/src/utils/context';
+
+export const getQuery = (request: Request): { [x: string]: string } => {
+  const output: { [x: string]: string } = {};
+
+  new URL(request.url).searchParams.forEach((value, key) => {
+    output[key] = value;
+  });
+
+  return output;
+};
+
+export const formData = (obj: Record<string, any>): string => {
+  return Object.keys(obj)
+    .map((key) => `${encodeURIComponent(key)}=${encodeURIComponent(obj[key])}`)
+    .join('&');
+};
+
+export const formDataRequest = (
+  obj: Record<string, any>,
+  init?: RequestInit
+): RequestInit => {
+  return {
+    method: 'POST', // First, so it can be overridden.
+    ...init,
+    headers: {
+      ...(init?.headers || {}),
+      'content-type': 'application/x-www-form-urlencoded',
+    },
+    body: formData(obj),
+  };
+};
+
+export const injectParams: RoleypolyMiddleware = (
+  request: Request & { params?: Record<string, string> },
+  context: Context
+) => {
+  context.params = {
+    guildId: request.params?.guildId,
+    memberId: request.params?.memberId,
+  };
+};

packages/api/src/utils/response.ts

@@ -0,0 +1,46 @@
+export const json = <T>(obj: T, init?: ResponseInit): Response => {
+  const body = JSON.stringify(obj);
+  return new Response(body, {
+    ...init,
+    headers: {
+      ...init?.headers,
+      'content-type': 'application/json; charset=utf-8',
+      ...corsHeaders,
+    },
+  });
+};
+
+export const corsHeaders = {
+  'Access-Control-Allow-Origin': '*',
+  'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS, PATCH',
+  'Access-Control-Allow-Headers': 'Content-Type, Authorization',
+  'Access-Control-Max-Age': '86400',
+};
+
+export const noContent = () => new Response(null, { status: 204 });
+export const seeOther = (url: string) =>
+  new Response(
+    `<!doctype html>If you are not redirected soon, <a href="${url}">click here.</a>`,
+    {
+      status: 303,
+      headers: {
+        location: url,
+        'content-type': 'text/html; charset=utf-8',
+      },
+    }
+  );
+
+export const invalid = () => json({ error: 'invalid request' }, { status: 400 });
+export const unauthorized = () => json({ error: 'unauthorized' }, { status: 401 });
+export const forbidden = () => json({ error: 'forbidden' }, { status: 403 });
+export const notFound = () => json({ error: 'not found' }, { status: 404 });
+export const serverError = (error: Error) => {
+  console.error(error);
+  return json({ error: 'internal server error' }, { status: 500 });
+};
+export const notImplemented = () => json({ error: 'not implemented' }, { status: 501 });
+
+// Only used to bully you in particular.
+// Maybe make better choices.
+export const engineeringProblem = (extra?: string) =>
+  json({ error: 'engineering problem', extra }, { status: 418 });

packages/api/src/utils/testHelpers.ts

@@ -0,0 +1,102 @@
+import { Config, Environment, parseEnvironment } from '@roleypoly/api/src/utils/config';
+import { Context } from '@roleypoly/api/src/utils/context';
+import { getID } from '@roleypoly/api/src/utils/id';
+import { SessionData, UserGuildPermissions } from '@roleypoly/types';
+import index from '../index';
+
+export const makeRequest = (
+  method: string,
+  path: string,
+  init?: RequestInit,
+  env?: Partial<Environment>
+): Promise<Response> => {
+  const request = new Request(`https://localhost:22000${path}`, {
+    method,
+    ...init,
+  });
+
+  return index.fetch(
+    request,
+    {
+      ...getMiniflareBindings(),
+      ...env,
+    },
+    {
+      waitUntil: async (promise: Promise<{}>) => await promise,
+    }
+  );
+};
+
+export const getBindings = (): Environment => getMiniflareBindings();
+
+export const makeSession = async (
+  config: Config,
+  data?: Partial<SessionData>
+): Promise<SessionData> => {
+  const sessionID = getID();
+
+  const session: SessionData = {
+    sessionID,
+    tokens: {
+      access_token: 'test-access-token',
+      refresh_token: 'test-refresh-token',
+      expires_in: 3600,
+      scope: 'identify guilds',
+      token_type: 'Bearer',
+    },
+    user: {
+      id: 'test-user-id',
+      username: 'test-username',
+      discriminator: 'test-discriminator',
+      avatar: 'test-avatar',
+      bot: false,
+    },
+    guilds: [
+      {
+        id: 'test-guild-id',
+        name: 'test-guild-name',
+        icon: 'test-guild-icon',
+        permissionLevel: UserGuildPermissions.User,
+      },
+      {
+        id: 'test-guild-id-editor',
+        name: 'test-guild-name',
+        icon: 'test-guild-icon',
+        permissionLevel: UserGuildPermissions.Manager,
+      },
+      {
+        id: 'test-guild-id-admin',
+        name: 'test-guild-name',
+        icon: 'test-guild-icon',
+        permissionLevel: UserGuildPermissions.Manager | UserGuildPermissions.Admin,
+      },
+    ],
+    ...data,
+  };
+
+  await config.kv.sessions.put(sessionID, session, config.retention.session);
+
+  return session;
+};
+
+export const configContext = (): [Config, Context] => {
+  const config = parseEnvironment({
+    ...getBindings(),
+    BOT_CLIENT_SECRET: 'test-client-secret',
+    BOT_CLIENT_ID: 'test-client-id',
+    BOT_TOKEN: 'test-bot-token',
+    INTERACTIONS_SHARED_KEY: '', // IMPORTANT: setting this properly can have unexpected results.
+  });
+  const context: Context = {
+    config,
+    fetchContext: {
+      waitUntil: () => {},
+    },
+    authMode: {
+      type: 'anonymous',
+    },
+    params: {},
+  };
+
+  return [config, context];
+};