[rpc]: implement both HTTP and NATS transports

docker-compose.yml

@@ -11,3 +11,9 @@ services:
             POSTGRES_DB: roleypoly
             POSTGRES_USER: roleypoly
             POSTGRES_INITDB_ARGS: -A trust
+
+    nats:
+        image: nats:1.4.1-linux
+        container_name: roleypoly-nats
+        ports:
+            - 4222:4222

packages/roleypoly-rpc/package.json

@@ -2,6 +2,8 @@
   "name": "@roleypoly/rpc",
   "version": "2.0.0",
   "devDependencies": {
+    "@types/superagent": "^4.1.1",
+    "@types/cookie": "^0.3.3",
     "lint-staged": "^8.1.7",
     "tslint": "^5.17.0",
     "typescript": "^3.5.1"
@@ -13,6 +15,10 @@
   },
   "private": true,
   "dependencies": {
-    "@kayteh/bento": "^0.1.1"
+    "@kayteh/bento": "^0.2.2",
+    "cookie": "^0.4.0",
+    "cross-fetch": "^3.0.3",
+    "nats": "^1.2.10",
+    "superagent": "^5.0.5"
   }
 }

packages/roleypoly-rpc/src/index.ts

@@ -0,0 +1,2 @@
+export { default as NATSTransport } from './utils/NATSTransport'
+export { default as HTTPTransport } from './utils/HTTPTransport'

packages/roleypoly-rpc/src/proto-lib/types.bento.ts

@@ -1,11 +0,0 @@
-/**
- * GENERATED FILE. This file was generated by @kayteh/bento. Editing it is a bad idea.
- * @generated
- */
-import Bento, { IBentoTransport } from '@kayteh/bento'
-export type ServerSlug = {
-  id?: string
-  name?: string
-  ownerID?: string
-  icon?: string
-}

packages/roleypoly-rpc/src/proto-lib/types.proto

@@ -1,5 +1,7 @@
 syntax = "proto3";
 
+// @bento-exclude
+
 message ServerSlug {
   string id = 1;
   string name = 2;

packages/roleypoly-rpc/src/utils/HTTPTransport.ts

@@ -0,0 +1,155 @@
+import Bento, { Transport, IBentoSerializer } from '@kayteh/bento'
+import http from 'http'
+import superagent from 'superagent'
+import cookie from 'cookie'
+
+const txtEnc = new TextEncoder()
+const txtDec = new TextDecoder()
+
+const castString = (val: string | string[] | undefined): string => {
+  if (typeof val === 'string') {
+    return val
+  }
+
+  if (Array.isArray(val)) {
+    return val[0]
+  }
+
+  return val || ''
+}
+
+export type HTTPContext = {
+  cookies: { [x: string]: string }
+  headers: http.IncomingHttpHeaders
+  requestor: {
+    userAgent: string
+    clientIp: string
+  }
+}
+
+export default class HTTPTransport extends Transport {
+  constructor (
+    bento: Bento,
+    serializer: IBentoSerializer,
+    private endpoint: string,
+    private injectHeaders: { [x: string]: string } = {}
+  ) {
+    super(bento, serializer)
+  }
+  handler = () => (req: http.IncomingMessage, res: http.ServerResponse) => {
+    // we're using bare http, so this can get a little dicey
+    // we do not assume we are routing in any special way here.
+    // a standardized approach would be POST /api/_rpc
+    if (req.method !== 'POST') {
+      res.statusCode = 405
+      res.end('Method not acceptable.')
+      return
+    }
+
+    return this.run(req, res)
+  }
+
+  run = (req: http.IncomingMessage, res: http.ServerResponse) => {
+    let buf = ''
+    req.on('data', (chunk: string) => {
+      buf += chunk
+    })
+
+    req.on('end', async () => {
+      const o = await this.receiver({ buffer: txtEnc.encode(buf), ctx: this.getContext(req, res) })
+      res.statusCode = 200
+      res.end(o)
+    })
+  }
+
+  /**
+   * get real client IP from headers or fallback to a default.
+   * since proxies add headers to tell a backend what is relevant,
+   * we use this failover pattern:
+   *  - True-Client-IP (from Cloudflare)
+   *  - X-Forwarded-For (0 position is true client)
+   *  - X-Client-IP (from Cloudfront, or even HAProxy by hand)
+   *  - default
+   * @param h http headers
+   * @param def fallback (usually socket remoteAddr)
+   */
+  getClientIP (h: http.IncomingHttpHeaders, def: string): string {
+    // we cast all of these to string because there will literally never be another.
+    if (h['true-client-ip'] !== undefined) {
+      return castString(h['true-client-ip'])
+    }
+
+    if (h['x-client-ip'] !== undefined) {
+      return castString(h['x-client-ip'])
+    }
+
+    if (h['x-forwarded-for'] !== undefined) {
+      return castString(h['x-forwarded-for']).split(', ')[0]
+    }
+
+    return def
+  }
+
+  // overridable
+  getContext = (req: http.IncomingMessage, res: http.ServerResponse): HTTPContext => {
+    return {
+      headers: req.headers,
+      cookies: cookie.parse(req.headers.cookie || ''),
+      requestor: {
+        clientIp: this.getClientIP(req.headers, req.socket.remoteAddress || ''),
+        userAgent: req.headers['user-agent'] || ''
+      }
+    }
+  }
+
+  /**
+   * creates a fake header that we extract JSON from to properly pass cookies in a server->server environment.
+   * @param o cookie object
+   */
+  withCookies (o: { [x: string]: string }) {
+    const out: string[] = []
+
+    for (const [key, val] of Object.entries(o)) {
+      out.push(cookie.serialize(key, val))
+    }
+
+    this.injectHeaders['@@-Set-Cookie'] = JSON.stringify(out)
+  }
+
+  withAuthorization (token: string) {
+    this.injectHeaders['Authorization'] = token
+  }
+
+  /**
+   * parses and removes the synthetic cookie header
+   * @param o headers
+   */
+  cookiesFromSyntheticHeaders (o: { '@@-Set-Cookie'?: string }): string[] {
+    if (o['@@-Set-Cookie'] !== undefined) {
+      const out = JSON.parse(o['@@-Set-Cookie'])
+      delete o['@@-Set-Cookie']
+      return out
+    }
+
+    return []
+  }
+
+  async sender (data: ArrayBuffer, _: { service: string, fn: string }): Promise<ArrayBuffer> {
+    const c = this.cookiesFromSyntheticHeaders(this.injectHeaders)
+
+    const r = superagent.post(this.endpoint)
+    .type('')
+    .send(txtDec.decode(data))
+    .set('User-Agent', 'roleypoly/2.0 bento http client (+https://roleypoly.com)')
+    .withCredentials()
+    .set(this.injectHeaders)
+
+    if (c.length > 0) {
+      r.set('Cookie', c)
+    }
+
+    const res = await r
+
+    return Buffer.from(res.body)
+  }
+}

packages/roleypoly-rpc/src/utils/NATSTransport.ts

@@ -0,0 +1,47 @@
+import NATS, { connect, NatsError } from 'nats'
+import Bento, { Transport, IBentoSerializer } from '@kayteh/bento'
+
+export default class NATSTransport extends Transport {
+  NATS: NATS.Client
+
+  constructor (
+    bento: Bento,
+    serializer: IBentoSerializer,
+    addr: string = 'nats://localhsot:4222/',
+    private prefix: string = ''
+  ) {
+    super(bento, serializer)
+
+    this.NATS = connect({
+      url: addr,
+      preserveBuffers: true
+    })
+  }
+
+  public hookHandlers = () => {
+    for (const svc in this.bento.serviceRegistry.keys) {
+      this.NATS.subscribe(`${this.prefix}-rpc:${svc}`, this.rpcHandler)
+    }
+  }
+
+  rpcHandler = async (request: ArrayBuffer, replyTo: string) => {
+    this.NATS.publish(replyTo, await this.receiver({
+      ctx: {},
+      buffer: request
+    }))
+  }
+
+  sender (data: ArrayBuffer, { service }: { service: string }): Promise<ArrayBuffer> {
+    return new Promise((resolve, reject) => {
+      this.NATS.requestOne(`${this.prefix}-rpc:${service}`, data, 5000, (incoming: NatsError | Buffer) => {
+        if (incoming instanceof NatsError) {
+          reject(incoming)
+          return
+        }
+
+        resolve(incoming)
+        return
+      })
+    })
+  }
+}

packages/roleypoly-rpc/src/utils/__test__/HTTPTransport.test.ts

@@ -0,0 +1,34 @@
+import HTTPTransport from '../HTTPTransport'
+import Bento, { JSONSerializer } from '@kayteh/bento'
+import { MockBackendClient } from './mock.bento'
+import MockBackendServer from './mock-server'
+import * as http from 'http'
+import * as sinon from 'sinon'
+
+describe('HTTPTransport', () => {
+  const NOW = Date.now()
+  const PORT = 20000 + (+(('' + NOW).slice(-4)))
+  const bento = new Bento()
+  const tt = new HTTPTransport(
+      bento,
+      new JSONSerializer(),
+      `https://localhost:${PORT}/api/_rpc`,
+      {}
+    )
+  const h = tt.handler()
+  const spy = sinon.spy(h)
+  const s = http.createServer(spy)
+  s.listen(PORT)
+
+  bento.transport = tt
+
+  bento.service(MockBackendClient.__SERVICE__, MockBackendServer)
+  const cc = bento.client(MockBackendClient)
+
+  it('handles full flow properly', async () => {
+    const out = await cc.helloBackend({ hello: 'yes!' })
+    expect(out.message).toBe(`hello, yes!! i'm bot!!`)
+  })
+
+  s.close()
+})

packages/roleypoly-rpc/src/utils/__test__/NATSTransport.test.ts

@@ -0,0 +1,26 @@
+import NATSTransport from '../NATSTransport'
+import Bento, { JSONSerializer } from '@kayteh/bento'
+import { MockBackendClient } from './mock.bento'
+import MockBackendServer from './mock-server'
+
+describe('NATSTransport', () => {
+  const NOW = Date.now()
+  const bento = new Bento()
+
+  const tt = new NATSTransport(
+    bento,
+    new JSONSerializer(),
+    'nats://localhost:4222/',
+    '' + NOW
+  )
+
+  bento.transport = tt
+
+  bento.service(MockBackendClient.__SERVICE__, MockBackendServer)
+  const cc = bento.client(MockBackendClient)
+
+  it('handles full flow properly', async () => {
+    const out = await cc.helloBackend({ hello: 'yes!' })
+    expect(out.message).toBe(`hello, yes!! i'm bot!!`)
+  })
+})

packages/roleypoly-rpc/src/utils/__test__/mock-server.ts

@@ -0,0 +1,9 @@
+import { IMockBackendService, HelloMsg, HelloReply } from './mock.bento'
+
+export default class MockBackendService implements IMockBackendService {
+  helloBackend (ctx: any, msg: HelloMsg): HelloReply {
+    return {
+      message: `hello, ${msg.hello}! i'm bot!!`
+    }
+  }
+}

packages/roleypoly-rpc/src/utils/__test__/mock.bento.ts

@@ -0,0 +1,34 @@
+/**
+ * GENERATED FILE. This file was generated by @kayteh/bento. Editing it is a bad idea.
+ * @generated
+ */
+import Bento, { IBentoTransport } from '@kayteh/bento'
+export type HelloMsg = {
+  hello?: string
+}
+
+export type HelloReply = {
+  message?: string
+}
+
+export interface IMockBotService {
+  helloBot (ctx: any, request: HelloMsg): Promise<HelloReply> | HelloReply
+}
+export class MockBotClient {
+  static __SERVICE__: string = 'MockBot'
+  constructor (private bento: Bento, private transport?: IBentoTransport) {}
+  async helloBot (request: HelloMsg): Promise<HelloReply> {
+    return this.bento.makeRequest(this.transport || undefined, 'MockBot', 'HelloBot', request)
+  }
+}
+
+export interface IMockBackendService {
+  helloBackend (ctx: any, request: HelloMsg): Promise<HelloReply> | HelloReply
+}
+export class MockBackendClient {
+  static __SERVICE__: string = 'MockBackend'
+  constructor (private bento: Bento, private transport?: IBentoTransport) {}
+  async helloBackend (request: HelloMsg): Promise<HelloReply> {
+    return this.bento.makeRequest(this.transport || undefined, 'MockBackend', 'HelloBackend', request)
+  }
+}

packages/roleypoly-rpc/src/utils/__test__/mock.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+
+service MockBot {
+  rpc HelloBot (HelloMsg) returns (HelloReply) {};
+}
+
+service MockBackend {
+  rpc HelloBackend (HelloMsg) returns (HelloReply) {};
+}
+
+message HelloMsg {
+  string hello = 1;
+}
+
+message HelloReply {
+  string message = 1;
+}

yarn.lock

@@ -1371,10 +1371,10 @@
     "@types/istanbul-reports" "^1.1.1"
     "@types/yargs" "^12.0.9"
 
-"@kayteh/bento@^0.1.1":
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/@kayteh/bento/-/bento-0.1.1.tgz#76e76f5261f41121b450a1dd68e49aa6c80f5525"
-  integrity sha512-BCGSHXn8eaPWzrvitvzu1lnajIXXIJTxE8D04w7ZMMPqOpTfD0jFaM+IRUKKygchdvL9mZVYcdCn+xroqL0mOA==
+"@kayteh/bento@^0.2.2":
+  version "0.2.2"
+  resolved "https://registry.yarnpkg.com/@kayteh/bento/-/bento-0.2.2.tgz#939b416ec7391e9f2df99550b96b0dc501e629e6"
+  integrity sha512-eyAwPcDKEu9eXjodxbWhMmDBJ+jHokMzBIotkV71cBNzmFfWvFIfL/1hqXs0nGqBYEBizQlJacjUbTraPezPEw==
   dependencies:
     camel-case "^3.0.0"
     fs-extra "^8.0.1"
@@ -2780,6 +2780,16 @@
   dependencies:
     "@types/node" "*"
 
+"@types/cookie@^0.3.3":
+  version "0.3.3"
+  resolved "https://registry.yarnpkg.com/@types/cookie/-/cookie-0.3.3.tgz#85bc74ba782fb7aa3a514d11767832b0e3bc6803"
+  integrity sha512-LKVP3cgXBT9RYj+t+9FDKwS5tdI+rPBXaNSkma7hvqy35lc7mAokC2zsqWJH0LaqIt3B962nuYI77hsJoT1gow==
+
+"@types/cookiejar@*":
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/@types/cookiejar/-/cookiejar-2.1.1.tgz#90b68446364baf9efd8e8349bb36bd3852b75b80"
+  integrity sha512-aRnpPa7ysx3aNW60hTiCtLHlQaIFsXFCgQlpakNgDNVFzbtusSY8PwjAQgRWfSk0ekNoBjO51eQRB6upA9uuyw==
+
 "@types/cookies@*":
   version "0.7.2"
   resolved "https://registry.yarnpkg.com/@types/cookies/-/cookies-0.7.2.tgz#5e0560d46ed9998082dce799af1058dd6a49780a"
@@ -3087,6 +3097,14 @@
     "@types/react" "*"
     csstype "^2.2.0"
 
+"@types/superagent@^4.1.1":
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/@types/superagent/-/superagent-4.1.1.tgz#61f0b43d7db93a3c286c124512b7c228183c32fa"
+  integrity sha512-NetXrraTWPcdGG6IwYJhJ5esUGx8AYNiozbc1ENWEsF6BsD4JmNODJczI6Rm1xFPVp6HZESds9YCfqz4zIsM6A==
+  dependencies:
+    "@types/cookiejar" "*"
+    "@types/node" "*"
+
 "@types/tapable@*":
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.4.tgz#b4ffc7dc97b498c969b360a41eee247f82616370"
@@ -5597,7 +5615,7 @@ cookie-signature@1.0.6:
   resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c"
   integrity sha1-4wOogrNCzD7oylE6eZmXNNqzriw=
 
-cookie@0.4.0:
+cookie@0.4.0, cookie@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.0.tgz#beb437e7022b3b6d49019d088665303ebe9c14ba"
   integrity sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==
@@ -5760,6 +5778,14 @@ create-react-context@^0.2.1:
     fbjs "^0.8.0"
     gud "^1.0.0"
 
+cross-fetch@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.0.3.tgz#b988beab275939cb77ad4e362c873f7fa8a221a5"
+  integrity sha512-mplYkc4CopHu+AdHK8wxjJcPskUxp5CvPTdtDij3MUgVNBa0xOb9CQqbbn1zO23qISM4WLxIBociyEVQL7WQAg==
+  dependencies:
+    node-fetch "2.6.0"
+    whatwg-fetch "3.0.0"
+
 cross-spawn@6.0.5, cross-spawn@^6.0.0, cross-spawn@^6.0.5:
   version "6.0.5"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
@@ -10962,6 +10988,14 @@ nanomatch@^1.2.9:
     snapdragon "^0.8.1"
     to-regex "^3.0.1"
 
+nats@^1.2.10:
+  version "1.2.10"
+  resolved "https://registry.yarnpkg.com/nats/-/nats-1.2.10.tgz#3f673e80d3a513f7802b0a5c1f227127a3d1d780"
+  integrity sha512-0FQMINZbyRkFMRbrpc6+IkKMQ+Zi2Ibr4YPhoEBlbP0Gw3ta23e/GB+LvXNqnV3htOPJNJ54+ToMI43BCYATGQ==
+  dependencies:
+    nuid "^1.0.0"
+    ts-nkeys "^1.0.8"
+
 natural-compare@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
@@ -11130,6 +11164,11 @@ node-fetch@2.3.0:
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.3.0.tgz#1a1d940bbfb916a1d3e0219f037e89e71f8c5fa5"
   integrity sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA==
 
+node-fetch@2.6.0, node-fetch@^2.2.0, node-fetch@^2.3.0:
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
+  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
+
 node-fetch@^1.0.1:
   version "1.7.3"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef"
@@ -11138,11 +11177,6 @@ node-fetch@^1.0.1:
     encoding "^0.1.11"
     is-stream "^1.0.1"
 
-node-fetch@^2.2.0, node-fetch@^2.3.0:
-  version "2.6.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
-  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
-
 node-gyp@^3.6.2:
   version "3.8.0"
   resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.8.0.tgz#540304261c330e80d0d5edce253a68cb3964218c"
@@ -11433,6 +11467,11 @@ nth-check@^1.0.2, nth-check@~1.0.1:
   dependencies:
     boolbase "~1.0.0"
 
+nuid@^1.0.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/nuid/-/nuid-1.1.0.tgz#e0c5746350a25562f58283197f34c50861f44675"
+  integrity sha512-C/JdZ6PtCqKsCEs4ni76nhBsdmuQgLAT/CTLNprkcLViDAnkk7qx5sSA8PVC2vmSsdBlSsFuGb52v6pwn1oaeg==
+
 num2fraction@^1.2.2:
   version "1.2.2"
   resolved "https://registry.yarnpkg.com/num2fraction/-/num2fraction-1.2.2.tgz#6f682b6a027a4e9ddfa4564cd2589d1d4e669ede"
@@ -15400,6 +15439,13 @@ ts-jest@^24.0.2:
     semver "^5.5"
     yargs-parser "10.x"
 
+ts-nkeys@^1.0.8:
+  version "1.0.12"
+  resolved "https://registry.yarnpkg.com/ts-nkeys/-/ts-nkeys-1.0.12.tgz#cda47f4842fe2c4f88b1303817050673e16acb89"
+  integrity sha512-5TgA+wbfxTy/9pdSuAhvneuL65KKoI7phonzNQH2UhnorAQAWehUwHNLEuli596wu/Fxh0SAhMeKZVLNx4s7Ow==
+  dependencies:
+    tweetnacl "^1.0.1"
+
 tslib@1.9.0:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.9.0.tgz#e37a86fda8cbbaf23a057f473c9f4dc64e5fc2e8"
@@ -15488,7 +15534,7 @@ tweetnacl@^0.14.3, tweetnacl@~0.14.0:
   resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
   integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=
 
-tweetnacl@^1.0.0:
+tweetnacl@^1.0.0, tweetnacl@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-1.0.1.tgz#2594d42da73cd036bd0d2a54683dd35a6b55ca17"
   integrity sha512-kcoMoKTPYnoeS50tzoqjPY3Uv9axeuuFAZY9M/9zFnhoVvRfxz9K29IMPD7jGmt2c8SW7i3gT9WqDl2+nV7p4A==
@@ -16161,7 +16207,7 @@ whatwg-encoding@^1.0.1, whatwg-encoding@^1.0.3:
   dependencies:
     iconv-lite "0.4.24"
 
-whatwg-fetch@>=0.10.0:
+whatwg-fetch@3.0.0, whatwg-fetch@>=0.10.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz#fc804e458cc460009b1a2b966bc8817d2578aefb"
   integrity sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q==