more ci/cd work, build worker and publish

.github/workflows/build.yml

@@ -50,12 +50,52 @@ jobs:
 
             - run: yarn test
 
+    worker_build:
+        runs-on: ubuntu-latest
+        name: Worker Build & Publish
+        needs:
+            - node_test
+        steps:
+            - uses: actions/setup-node@v2-beta
+              with:
+                  node-version: '14'
+
+            - name: Get yarn cache directory path
+              id: yarn-cache-dir-path
+              run: echo "::set-output name=dir::$(yarn cache dir)"
+
+            - uses: actions/cache@v2
+              id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+              with:
+                  path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+                  key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+                  restore-keys: |
+                      ${{ runner.os }}-yarn-
+
+            - run: yarn install --frozen-lockfile
+
+            - run: npx wrangler build
+
+            - name: Set up Cloud SDK
+              uses: google-github-actions/setup-gcloud@master
+              with:
+                  project_id: ${{ secrets.GCP_PROJECT_ID }}
+                  service_account_key: ${{ secrets.GCP_TF_KEY }}
+                  export_default_credentials: true
+
+            - name: Publish to GCS
+              run: |
+                  gsutil cp worker/script.js gs://roleypoly-artifacts/backend-worker/${{ github.sha }}
+
     docker_build:
         name: Docker Build & Publish
         runs-on: ubuntu-latest
         needs:
             - go_test
             - node_test
+        outputs:
+            ui_digest: ${{steps.docker_push_ui.outputs.digest}}
+            bot_digest: ${{steps.docker_push_bot.outputs.digest}}
         strategy:
             matrix:
                 dockerfile:
@@ -77,7 +117,7 @@ jobs:
               with:
                   images: |
                       ghcr.io/roleypoly/${{matrix.dockerfile}}
-                      us-docker.pkg.dev/roleypoly/roleypoly/${{matrix.dockerfile}}
+                  tag-sha: true
 
             - name: Set up Docker Buildx
               id: buildx
@@ -92,15 +132,9 @@ jobs:
                   username: roleypoly
                   password: ${{ secrets.GHCR_PAT }}
 
-            - name: Login to GAR
-              uses: docker/login-action@v1
-              with:
-                  registry: us-docker.pkg.dev
-                  username: _json_key
-                  password: ${{ secrets.GAR_JSON_KEY }}
-
             - name: Build and push
               uses: docker/build-push-action@v2
+              id: docker_push_${{ matrix.dockerfile }}
               with:
                   context: .
                   file: ./hack/dockerfiles/${{matrix.dockerfile}}.Dockerfile

.github/workflows/deploy.yml

@@ -0,0 +1,132 @@
+name: Deploy
+
+on:
+    workflow_dispatch:
+        inputs:
+            environment:
+                description: 'One of: stage, prod'
+                required: true
+                default: stage
+            ui_tag:
+                description: 'tag/digest reference to a UI container build'
+                required: false
+                default: ':main'
+            bot_tag:
+                description: 'tag/digest reference to a UI container build'
+                required: false
+                default: ':main'
+            worker_tag:
+                description: 'bucket key to fetch worker from'
+                required: false
+                default: '' # Empty will try using current main branch hash
+
+jobs:
+    docker_sync:
+        runs-on: ubuntu-latest
+        outputs:
+            ui_tag: steps.tags.ui_tag
+            bot_tag: steps.tags.bot_tag
+        steps:
+            - uses: actions/checkout@master
+
+            - uses: docker/setup-buildx-action@v1
+              id: buildx
+              with:
+                  install: true
+
+            - name: Login to GHCR
+              uses: docker/login-action@v1
+              with:
+                  registry: ghcr.io
+                  username: roleypoly
+                  password: ${{ secrets.GHCR_PAT }}
+
+            - name: Login to GAR US
+              uses: docker/login-action@v1
+              with:
+                  registry: us-docker.pkg.io
+                  username: _json_key
+                  password: ${{ secrets.GAR_JSON_KEY }}
+
+            - name: Login to GAR Europe
+              uses: docker/login-action@v1
+              with:
+                  registry: europe-docker.pkg.io
+                  username: _json_key
+                  password: ${{ secrets.GAR_JSON_KEY }}
+
+            - name: Login to GAR Asia
+              uses: docker/login-action@v1
+              with:
+                  registry: asia-docker.pkg.io
+                  username: _json_key
+                  password: ${{ secrets.GAR_JSON_KEY }}
+
+            - name: Retag
+              id: tags
+              run: |
+                  retag_push() {
+                    docker tag $1 $2
+                    docker push $2
+                  }
+
+                  UI_IMAGE_SRC=ghcr.io/roleypoly/ui${{event.inputs.ui_tag}}
+                  UI_IMAGE_DEST_BASE=docker.pkg.dev/roleypoly/roleypoly/ui:${{event.inputs.environment}}
+
+                  docker pull $UI_IMAGE_SRC
+                  retag_push $UI_IMAGE_SRC us-$UI_IMAGE_DEST_BASE
+                  retag_push $UI_IMAGE_SRC europe-$UI_IMAGE_DEST_BASE
+                  retag_push $UI_IMAGE_SRC asia-$UI_IMAGE_DEST_BASE
+                  echo ::set-output name=ui_tag::@$(docker inspect $UI_IMAGE_SRC -f '{{.Id}}')
+
+                  BOT_IMAGE_SRC=ghcr.io/roleypoly/bot${{event.inputs.bot_tag}}
+                  BOT_IMAGE_DEST_BASE=docker.pkg.dev/roleypoly/roleypoly/bot:${{event.inputs.environment}}
+
+                  docker pull $BOT_IMAGE_SRC
+                  retag_push $BOT_IMAGE_SRC us-$BOT_IMAGE_DEST_BASE
+                  echo ::set-output name=bot_tag::@$(docker inspect $BOT_IMAGE_SRC -f '{{.Id}}'
+
+    deploy_terraform:
+        runs-on: ubuntu-latest
+        needs:
+            - docker_sync
+        steps:
+            - uses: actions/checkout@master
+
+            - uses: hashicorp/setup-terraform@v1
+              with:
+                  terraform_version: ^0.14.0
+
+            - run: cd terraform
+
+            - name: Set up Cloud SDK
+              uses: google-github-actions/setup-gcloud@master
+              with:
+                  project_id: ${{ secrets.GCP_PROJECT_ID }}
+                  service_account_key: ${{ secrets.GCP_TF_KEY }}
+                  export_default_credentials: true
+
+            - name: Pull necessary artifacts
+              run: |
+                  currentHash=${{ github.sha }}
+                  selectedTargetArtifact=${${{ event.inputs.worker_tag }}:-$currentHash}
+
+                  mkdir worker-dist
+                  gsutil cp gs://roleypoly-artifacts/backend-worker/$selectedTargetArtifact worker-dist/backend-worker.js
+
+            - run: terraform init --backend-config "prefix=${{event.inputs.environment}}"
+
+            - name: Write tags to tags.tfvars.auto.json
+              run: |
+                  echo '{"ui_tag": "${{needs.docker_sync.outputs.ui_tag}}", "bot_tag": "${{needs.docker_sync.outputs.bot_tag}}", "worker_path": "./worker-dist/backend-worker.js"}' | jq | tee tags.tfvars.auto.json
+
+            - name: Write TF_DATA_${{event.inputs.environment}} to actions.tfvars.auto.json
+              run: |
+                  sh -c 'echo $TF_DATA | jq > actions.tfvars.auto.json' >/dev/null 2>&1
+              env:
+                  TF_DATA: ${{ secrets['TF_DATA_'+event.inputs.environment] }}
+
+            - run: |
+                  terraform plan \
+                    --var-file variables/global.tfvars \
+                    --var-file variables/${{event.inputs.environment}}.tfvars

terraform/providers.tf

@@ -26,6 +26,10 @@ terraform {
       source  = "hashicorp/null"
     }
   }
+
+  backend "gcs" {
+    bucket  = "roleypoly-tfstate"
+  }
 }
 
 variable "cloudflare_email" {

terraform/variables.tf

@@ -40,6 +40,12 @@ variable "api_public_uri" {
   description = "API Public Base Path"
 }
 
+variable "api_path_to_worker" {
+  type = string
+  description = "Path to worker JS, relative to this file/terraform folder."
+  default = "../worker/script.js"
+}
+
 variable "root_users" {
   type        = list(string)
   description = "Root users to use for role elevation calculations"

terraform/workers.tf

@@ -12,7 +12,7 @@ resource "cloudflare_workers_kv_namespace" "guild_data" {
 
 resource "cloudflare_worker_script" "backend" {
   name    = "roleypoly-backend-${var.environment_tag}"
-  content = file("${path.module}/../worker/script.js")
+  content = file("${path.module}/${var.api_path_to_worker}")
 
   kv_namespace_binding {
     name         = "KV_SESSIONS"