From 95a573245716612466f15d61f598429988c9dd30 Mon Sep 17 00:00:00 2001 From: Katalina Okano Date: Sun, 6 Dec 2020 17:59:24 -0500 Subject: [PATCH] more ci/cd work, build worker and publish --- .github/workflows/build.yml | 50 ++++++++++--- .github/workflows/deploy.yml | 132 +++++++++++++++++++++++++++++++++++ terraform/providers.tf | 4 ++ terraform/variables.tf | 6 ++ terraform/workers.tf | 2 +- 5 files changed, 185 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e8cd057..a1e3466 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,12 +50,52 @@ jobs: - run: yarn test + worker_build: + runs-on: ubuntu-latest + name: Worker Build & Publish + needs: + - node_test + steps: + - uses: actions/setup-node@v2-beta + with: + node-version: '14' + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + + - uses: actions/cache@v2 + id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - run: yarn install --frozen-lockfile + + - run: npx wrangler build + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@master + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_TF_KEY }} + export_default_credentials: true + + - name: Publish to GCS + run: | + gsutil cp worker/script.js gs://roleypoly-artifacts/backend-worker/${{ github.sha }} + docker_build: name: Docker Build & Publish runs-on: ubuntu-latest needs: - go_test - node_test + outputs: + ui_digest: ${{steps.docker_push_ui.outputs.digest}} + bot_digest: ${{steps.docker_push_bot.outputs.digest}} strategy: matrix: dockerfile: @@ -77,7 +117,7 @@ jobs: with: images: | ghcr.io/roleypoly/${{matrix.dockerfile}} - us-docker.pkg.dev/roleypoly/roleypoly/${{matrix.dockerfile}} + tag-sha: true - name: Set up Docker Buildx id: buildx @@ -92,15 +132,9 @@ jobs: username: roleypoly password: ${{ secrets.GHCR_PAT }} - - name: Login to GAR - uses: docker/login-action@v1 - with: - registry: us-docker.pkg.dev - username: _json_key - password: ${{ secrets.GAR_JSON_KEY }} - - name: Build and push uses: docker/build-push-action@v2 + id: docker_push_${{ matrix.dockerfile }} with: context: . file: ./hack/dockerfiles/${{matrix.dockerfile}}.Dockerfile diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..904b8b0 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,132 @@ +name: Deploy + +on: + workflow_dispatch: + inputs: + environment: + description: 'One of: stage, prod' + required: true + default: stage + ui_tag: + description: 'tag/digest reference to a UI container build' + required: false + default: ':main' + bot_tag: + description: 'tag/digest reference to a UI container build' + required: false + default: ':main' + worker_tag: + description: 'bucket key to fetch worker from' + required: false + default: '' # Empty will try using current main branch hash + +jobs: + docker_sync: + runs-on: ubuntu-latest + outputs: + ui_tag: steps.tags.ui_tag + bot_tag: steps.tags.bot_tag + steps: + - uses: actions/checkout@master + + - uses: docker/setup-buildx-action@v1 + id: buildx + with: + install: true + + - name: Login to GHCR + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: roleypoly + password: ${{ secrets.GHCR_PAT }} + + - name: Login to GAR US + uses: docker/login-action@v1 + with: + registry: us-docker.pkg.io + username: _json_key + password: ${{ secrets.GAR_JSON_KEY }} + + - name: Login to GAR Europe + uses: docker/login-action@v1 + with: + registry: europe-docker.pkg.io + username: _json_key + password: ${{ secrets.GAR_JSON_KEY }} + + - name: Login to GAR Asia + uses: docker/login-action@v1 + with: + registry: asia-docker.pkg.io + username: _json_key + password: ${{ secrets.GAR_JSON_KEY }} + + - name: Retag + id: tags + run: | + retag_push() { + docker tag $1 $2 + docker push $2 + } + + UI_IMAGE_SRC=ghcr.io/roleypoly/ui${{event.inputs.ui_tag}} + UI_IMAGE_DEST_BASE=docker.pkg.dev/roleypoly/roleypoly/ui:${{event.inputs.environment}} + + docker pull $UI_IMAGE_SRC + retag_push $UI_IMAGE_SRC us-$UI_IMAGE_DEST_BASE + retag_push $UI_IMAGE_SRC europe-$UI_IMAGE_DEST_BASE + retag_push $UI_IMAGE_SRC asia-$UI_IMAGE_DEST_BASE + echo ::set-output name=ui_tag::@$(docker inspect $UI_IMAGE_SRC -f '{{.Id}}') + + BOT_IMAGE_SRC=ghcr.io/roleypoly/bot${{event.inputs.bot_tag}} + BOT_IMAGE_DEST_BASE=docker.pkg.dev/roleypoly/roleypoly/bot:${{event.inputs.environment}} + + docker pull $BOT_IMAGE_SRC + retag_push $BOT_IMAGE_SRC us-$BOT_IMAGE_DEST_BASE + echo ::set-output name=bot_tag::@$(docker inspect $BOT_IMAGE_SRC -f '{{.Id}}' + + deploy_terraform: + runs-on: ubuntu-latest + needs: + - docker_sync + steps: + - uses: actions/checkout@master + + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: ^0.14.0 + + - run: cd terraform + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@master + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_TF_KEY }} + export_default_credentials: true + + - name: Pull necessary artifacts + run: | + currentHash=${{ github.sha }} + selectedTargetArtifact=${${{ event.inputs.worker_tag }}:-$currentHash} + + mkdir worker-dist + gsutil cp gs://roleypoly-artifacts/backend-worker/$selectedTargetArtifact worker-dist/backend-worker.js + + - run: terraform init --backend-config "prefix=${{event.inputs.environment}}" + + - name: Write tags to tags.tfvars.auto.json + run: | + echo '{"ui_tag": "${{needs.docker_sync.outputs.ui_tag}}", "bot_tag": "${{needs.docker_sync.outputs.bot_tag}}", "worker_path": "./worker-dist/backend-worker.js"}' | jq | tee tags.tfvars.auto.json + + - name: Write TF_DATA_${{event.inputs.environment}} to actions.tfvars.auto.json + run: | + sh -c 'echo $TF_DATA | jq > actions.tfvars.auto.json' >/dev/null 2>&1 + env: + TF_DATA: ${{ secrets['TF_DATA_'+event.inputs.environment] }} + + - run: | + terraform plan \ + --var-file variables/global.tfvars \ + --var-file variables/${{event.inputs.environment}}.tfvars diff --git a/terraform/providers.tf b/terraform/providers.tf index 6ac2d55..db3c654 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -26,6 +26,10 @@ terraform { source = "hashicorp/null" } } + + backend "gcs" { + bucket = "roleypoly-tfstate" + } } variable "cloudflare_email" { diff --git a/terraform/variables.tf b/terraform/variables.tf index 5d4b596..5d4529a 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -40,6 +40,12 @@ variable "api_public_uri" { description = "API Public Base Path" } +variable "api_path_to_worker" { + type = string + description = "Path to worker JS, relative to this file/terraform folder." + default = "../worker/script.js" +} + variable "root_users" { type = list(string) description = "Root users to use for role elevation calculations" diff --git a/terraform/workers.tf b/terraform/workers.tf index bfb826e..06c97c1 100644 --- a/terraform/workers.tf +++ b/terraform/workers.tf @@ -12,7 +12,7 @@ resource "cloudflare_workers_kv_namespace" "guild_data" { resource "cloudflare_worker_script" "backend" { name = "roleypoly-backend-${var.environment_tag}" - content = file("${path.module}/../worker/script.js") + content = file("${path.module}/${var.api_path_to_worker}") kv_namespace_binding { name = "KV_SESSIONS"