Merge branch 'tf' into main

.github/workflows/build.yml

@@ -1,6 +1,7 @@
 name: Bazel Build
 
-on: push
+on: 
+  push: [branch]
 
 jobs:
   bazel_build:

.github/workflows/dev-container.yml

@@ -19,7 +19,7 @@ jobs:
         uses: actions/cache@v2.1.1
         with:
           path: "/home/runner/.cache/bazel"
-          key: bazel
+          key: bazel_dev_container
 
       - name: Install bazelisk
         run: |

.github/workflows/release.yml

@@ -20,3 +20,5 @@ jobs:
           git tag $TAG
           git push origin $TAG
           echo "::set-output release_tag=${TAG}"
+
+      - name: Retag images

.github/workflows/terraform.yml

@@ -0,0 +1,2 @@
+name: Terraform
+

terraform/.gitignore

@@ -0,0 +1,33 @@
+# Local .terraform directories
+**/.terraform
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+
+# Ignore any .tfvars files that are generated automatically for each Terraform run. Most
+# .tfvars files are managed as part of configuration and so should be included in
+# version control.
+#
+# example.tfvars
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+#
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc

terraform/app/discord.tf

@@ -0,0 +1,56 @@
+locals {
+  discord_labels = {
+    "app.kubernetes.io/name"    = "discord"
+    "app.kubernetes.io/part-of" = "roleypoly"
+    "roleypoly/environment"     = var.environment_tag
+  }
+}
+
+resource "kubernetes_deployment" "discord" {
+  metadata {
+    name      = "discord"
+    namespace = local.ns
+    labels    = local.discord_labels
+  }
+
+  spec {
+    replicas = 1
+
+    selector {
+      match_labels = local.discord_labels
+    }
+
+    template {
+      metadata {
+        labels = local.discord_labels
+      }
+
+      spec {
+        container {
+          image = "roleypoly/discord:${local.tags.discord}"
+          name  = "discord"
+
+          liveness_probe {
+            http_get {
+              path = "/"
+              port = 16777
+            }
+
+            initial_delay_seconds = 3
+            period_seconds        = 3
+          }
+
+          readiness_probe {
+            http_get {
+              path = "/"
+              port = 16777
+            }
+
+            initial_delay_seconds = 3
+            period_seconds        = 3
+          }
+        }
+      }
+    }
+  }
+}

terraform/app/provision.tf

@@ -0,0 +1,26 @@
+terraform {
+  required_version = ">=0.12.6"
+
+  backend "remote" {
+    organization = "Roleypoly"
+
+    workspaces {
+      prefix = "roleypoly-app-"
+    }
+  }
+}
+
+variable "k8s_endpoint" { type = string }
+variable "k8s_token" { type = string }
+variable "k8s_cert" { type = string }
+variable "k8s_namespace" { type = string }
+provider "kubernetes" {
+  load_config_file       = false
+  token                  = var.k8s_token
+  host                   = var.k8s_endpoint
+  cluster_ca_certificate = var.k8s_cert
+}
+
+locals {
+  ns = var.k8s_namespace
+}

terraform/app/tags.auto.tfvars.json

@@ -0,0 +1,6 @@
+{
+  "deployment_env": {
+    "production": {},
+    "staging": {}
+  }
+}

terraform/app/variables.tf

@@ -0,0 +1,12 @@
+variable "deployment_env" {
+  type = map(map(string))
+}
+
+variable "environment_tag" {
+  type        = string
+  description = "One of: production, staging, test"
+}
+
+locals {
+  tags = var.deployment_env[var.environment_tag]
+}

terraform/modules/cloudflare-cluster-dns/main.tf

@@ -0,0 +1,66 @@
+# Primary cluster hostname
+resource "cloudflare_record" "cluster" {
+  zone_id = var.cloudflare-zone-id
+  name    = var.record-name
+  value   = var.ingress-endpoint
+  type    = "A"
+  proxied = true
+}
+
+# PRD & STG records for direct FQDN usage
+# Long term, these will also be CNAME'd to 
+# - prd == roleypoly.com
+# - stg == beta.roleypoly.com
+resource "cloudflare_record" "prd" {
+  zone_id = var.cloudflare-zone-id
+  name    = "prd.${var.record-name}"
+  value   = cloudflare_record.cluster.hostname
+  type    = "CNAME"
+  proxied = true
+}
+
+resource "cloudflare_record" "stg" {
+  zone_id = var.cloudflare-zone-id
+  name    = "stg.${var.record-name}"
+  value   = cloudflare_record.cluster.hostname
+  type    = "CNAME"
+  proxied = true
+}
+
+# Origin CA Cert
+resource "tls_private_key" "origin-ca-key" {
+  algorithm = "ECDSA"
+}
+
+resource "tls_cert_request" "origin-ca-csr" {
+  key_algorithm   = tls_private_key.origin-ca-key.algorithm
+  private_key_pem = tls_private_key.origin-ca-key.private_key_pem
+
+  subject {
+    common_name  = "roleypoly.com"
+    organization = "Roleypoly"
+  }
+}
+
+resource "cloudflare_origin_ca_certificate" "origin-ca-cert" {
+  csr = tls_cert_request.origin-ca-csr.cert_request_pem
+  hostnames = [
+    cloudflare_record.cluster.hostname,
+    cloudflare_record.prd.hostname,
+    cloudflare_record.stg.hostname
+  ]
+  request_type       = "origin-ecc"
+  requested_validity = 1095 # 3 years
+}
+
+resource "kubernetes_secret" "cloudflare-origin" {
+  type = "kubernetes.io/tls"
+  metadata {
+    name      = "cloudflare-origin"
+    namespace = "default"
+  }
+  data = {
+    "tls.crt" = base64encode(cloudflare_origin_ca_certificate.origin-ca-cert.certificate),
+    "tls.key" = base64encode(tls_private_key.origin-ca-key.private_key_pem)
+  }
+}

terraform/modules/cloudflare-cluster-dns/variables.tf

@@ -0,0 +1,19 @@
+variable "ingress-name" {
+  type = string
+}
+
+variable "ingress-namespace" {
+  type = string
+}
+
+variable "ingress-endpoint" {
+  type = string
+}
+
+variable "cloudflare-zone-id" {
+  type = string
+}
+
+variable "record-name" {
+  type = string
+}

terraform/modules/cloudflare-cluster-dns/version.tf

@@ -0,0 +1,4 @@
+terraform {
+  required_version = ">=0.12"
+}
+

terraform/modules/cluster-environment/main.tf

@@ -0,0 +1,56 @@
+locals {
+  ns = "${var.app_name}-${var.environment_tag}"
+  labels = {
+    "app.kubernetes.io/name"    = var.app_name
+    "app.kubernetes.io/part-of" = var.app_name
+    "roleypoly/environment"     = var.environment_tag
+  }
+}
+
+resource "kubernetes_namespace" "ns" {
+  metadata {
+    name   = local.ns
+    labels = local.labels
+  }
+}
+
+resource "kubernetes_service_account" "sa" {
+  metadata {
+    name      = "${local.ns}-sa-tf"
+    namespace = local.ns
+    labels    = local.labels
+  }
+}
+
+resource "kubernetes_secret" "sa-key" {
+  metadata {
+    name      = "${local.ns}-sa-tf-key"
+    namespace = local.ns
+    labels    = local.labels
+    annotations = {
+      "kubernetes.io/service-account.name" = kubernetes_service_account.sa.metadata.0.name
+    }
+  }
+
+  type = "kubernetes.io/service-account-token"
+}
+
+resource "kubernetes_role_binding" "sa-admin-rb" {
+  metadata {
+    name      = "${local.ns}-sa-admin-binding"
+    namespace = local.ns
+    labels    = local.labels
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.sa.metadata.0.name
+    namespace = local.ns
+  }
+
+  role_ref {
+    kind      = "ClusterRole"
+    name      = "admin"
+    api_group = "rbac.authorization.k8s.io"
+  }
+}

terraform/modules/cluster-environment/output.tf

@@ -0,0 +1,7 @@
+output "service_account_token" {
+  value = lookup(kubernetes_secret.sa-key, "data.token", "")
+}
+
+output "namespace" {
+  value = local.ns
+}

terraform/modules/cluster-environment/variables.tf

@@ -0,0 +1,9 @@
+variable "environment_tag" {
+  type    = string
+  default = "main"
+}
+
+variable "app_name" {
+  type = string
+}
+

terraform/modules/nginx-ingress-controller/main.tf

@@ -0,0 +1,331 @@
+locals {
+  ns = kubernetes_namespace.ns.metadata.0.name
+  labels = {
+    "app.kubernetes.io/name"    = "ingress-nginx"
+    "app.kubernetes.io/part-of" = "ingress-nginx"
+  }
+}
+
+resource "kubernetes_namespace" "ns" {
+  metadata {
+    name   = "ingress-nginx"
+    labels = local.labels
+  }
+}
+
+resource "kubernetes_config_map" "cm-nginx" {
+  metadata {
+    name      = "nginx-configuration"
+    namespace = local.ns
+    labels    = local.labels
+  }
+}
+
+resource "kubernetes_config_map" "cm-tcp" {
+  metadata {
+    name      = "tcp-services"
+    namespace = local.ns
+    labels    = local.labels
+  }
+}
+
+resource "kubernetes_config_map" "cm-udp" {
+  metadata {
+    name      = "udp-services"
+    namespace = local.ns
+    labels    = local.labels
+  }
+}
+
+resource "kubernetes_service_account" "sa" {
+  metadata {
+    name      = "nginx-ingress-serviceaccount"
+    namespace = local.ns
+    labels    = local.labels
+  }
+}
+
+resource "kubernetes_cluster_role" "cr" {
+  metadata {
+    name   = "nginx-ingress-clusterrole"
+    labels = local.labels
+  }
+  rule {
+    api_groups = [""]
+    resources  = ["configmaps", "endpoints", "nodes", "pods", "secrets"]
+    verbs      = ["list", "watch"]
+  }
+  rule {
+    api_groups = [""]
+    resources  = ["nodes"]
+    verbs      = ["get"]
+  }
+  rule {
+    api_groups = [""]
+    resources  = ["services"]
+    verbs      = ["get", "list", "watch"]
+  }
+  rule {
+    api_groups = [""]
+    resources  = ["events"]
+    verbs      = ["create", "patch"]
+  }
+  rule {
+    api_groups = ["extensions", "networking.k8s.io"]
+    resources  = ["ingresses"]
+    verbs      = ["get", "list", "watch"]
+  }
+  rule {
+    api_groups = ["extensions", "networking.k8s.io"]
+    resources  = ["ingresses/status"]
+    verbs      = ["update"]
+  }
+}
+
+resource "kubernetes_role" "role" {
+  metadata {
+    name      = "nginx-ingress-role"
+    namespace = local.ns
+    labels    = local.labels
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["configmaps", "pods", "secrets", "namespaces"]
+    verbs      = ["get"]
+  }
+
+  rule {
+    api_groups     = [""]
+    resources      = ["configmaps"]
+    resource_names = ["ingress-controller-leader-nginx"]
+    verbs          = ["get", "update"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["configmaps"]
+    verbs      = ["create"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["endpoints"]
+    verbs      = ["get"]
+  }
+}
+
+resource "kubernetes_role_binding" "rb" {
+  metadata {
+    name      = "nginx-ingress-role-nisa-binding"
+    namespace = local.ns
+    labels    = local.labels
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "Role"
+    name      = kubernetes_role.role.metadata.0.name
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.sa.metadata.0.name
+    namespace = local.ns
+  }
+}
+
+resource "kubernetes_cluster_role_binding" "crb" {
+  metadata {
+    name   = "nginx-ingress-clusterrole-nisa-binding"
+    labels = local.labels
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = kubernetes_cluster_role.cr.metadata.0.name
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.sa.metadata.0.name
+    namespace = local.ns
+  }
+}
+
+resource "kubernetes_deployment" "deployment" {
+  metadata {
+    name      = "nginx-ingress-controller"
+    namespace = local.ns
+    labels    = local.labels
+  }
+
+  spec {
+    replicas = 3
+
+    selector {
+      match_labels = local.labels
+    }
+
+    template {
+      metadata {
+        labels = local.labels
+        annotations = {
+          "prometheus.io/port"   = "10254"
+          "prometheus.io/scrape" = "true"
+        }
+      }
+
+      spec {
+        automount_service_account_token  = true
+        termination_grace_period_seconds = 300
+        service_account_name             = kubernetes_service_account.sa.metadata.0.name
+        node_selector = {
+          "kubernetes.io/os" = "linux"
+          node_type          = "static"
+        }
+
+        container {
+          name  = "nginx-ingress-controller"
+          image = "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:${var.nginx-ingress-version}"
+          args = [
+            "/nginx-ingress-controller",
+            "--configmap=${local.ns}/${kubernetes_config_map.cm-nginx.metadata.0.name}",
+            "--tcp-services-configmap=${local.ns}/${kubernetes_config_map.cm-tcp.metadata.0.name}",
+            "--udp-services-configmap=${local.ns}/${kubernetes_config_map.cm-udp.metadata.0.name}",
+            "--publish-service=${local.ns}/ingress-nginx",
+            "--annotations-prefix=nginx.ingress.kubernetes.io",
+          ]
+          security_context {
+            allow_privilege_escalation = true
+            capabilities {
+              drop = ["ALL"]
+              add  = ["NET_BIND_SERVICE"]
+            }
+            run_as_user = 101
+          }
+
+          env {
+            name = "POD_NAME"
+            value_from {
+              field_ref {
+                field_path = "metadata.name"
+              }
+            }
+          }
+
+          env {
+            name = "POD_NAMESPACE"
+            value_from {
+              field_ref {
+                field_path = "metadata.namespace"
+              }
+            }
+          }
+
+          port {
+            name           = "http"
+            container_port = 80
+            protocol       = "TCP"
+          }
+
+          port {
+            name           = "https"
+            container_port = 443
+            protocol       = "TCP"
+          }
+
+          liveness_probe {
+            http_get {
+              path   = "/healthz"
+              port   = 10254
+              scheme = "HTTP"
+            }
+            failure_threshold     = 3
+            initial_delay_seconds = 10
+            period_seconds        = 10
+            success_threshold     = 1
+            timeout_seconds       = 10
+          }
+
+          readiness_probe {
+            http_get {
+              path   = "/healthz"
+              port   = 10254
+              scheme = "HTTP"
+            }
+            failure_threshold     = 3
+            initial_delay_seconds = 10
+            period_seconds        = 10
+            success_threshold     = 1
+            timeout_seconds       = 10
+          }
+
+          lifecycle {
+            pre_stop {
+              exec {
+                command = ["/wait-shutdown"]
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_limit_range" "lr" {
+  metadata {
+    name      = "ingress-nginx"
+    namespace = local.ns
+    labels    = local.labels
+  }
+
+  spec {
+    limit {
+      min = {
+        memory = "90Mi"
+        cpu    = "100m"
+      }
+
+      type = "Container"
+    }
+  }
+}
+
+# Specific service related to Google Cloud
+resource "kubernetes_service" "svc" {
+  metadata {
+    name      = "ingress-nginx"
+    namespace = local.ns
+    labels    = local.labels
+  }
+
+  spec {
+    external_traffic_policy = "Local"
+    type                    = "LoadBalancer"
+    selector                = local.labels
+
+    port {
+      name        = "http"
+      port        = 80
+      protocol    = "TCP"
+      target_port = "http"
+    }
+
+    port {
+      name        = "https"
+      port        = 443
+      protocol    = "TCP"
+      target_port = "https"
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [
+      // We add no annotations, but DO adds some.
+      metadata[0].annotations,
+    ]
+  }
+}

terraform/modules/nginx-ingress-controller/outputs.tf

@@ -0,0 +1,11 @@
+output "service-name" {
+  value = kubernetes_service.svc.metadata.0.name
+}
+
+output "service-namespace" {
+  value = kubernetes_service.svc.metadata.0.namespace
+}
+
+output "service-endpoint" {
+  value = kubernetes_service.svc.load_balancer_ingress.0.ip
+}

terraform/modules/nginx-ingress-controller/variables.tf

@@ -0,0 +1,4 @@
+variable "nginx-ingress-version" {
+  type    = string
+  default = "0.30.0"
+}

terraform/modules/nginx-ingress-controller/version.tf

@@ -0,0 +1,3 @@
+terraform {
+    required_version = ">=0.12"
+}

terraform/modules/tfc-workspace/main.tf

@@ -0,0 +1,57 @@
+locals {
+  dependentModulesPathed = formatlist("terraform/modules/%s", var.dependent_modules)
+  variableDescription    = "Terraform-owned variable"
+}
+
+resource "tfe_workspace" "ws" {
+  name              = var.workspace-name
+  organization      = var.tfc_org
+  auto_apply        = var.auto_apply
+  trigger_prefixes  = concat([var.directory], local.dependentModulesPathed)
+  working_directory = var.directory
+
+  vcs_repo {
+    identifier     = var.repo
+    branch         = var.branch
+    oauth_token_id = var.tfc_oauth_token_id
+  }
+}
+
+resource "tfe_notification_configuration" "webhook" {
+  name             = "${var.workspace-name}-webhook"
+  enabled          = true
+  destination_type = "slack"
+  triggers         = ["run:created", "run:planning", "run:needs_attention", "run:applying", "run:completed", "run:errored"]
+  url              = var.tfc_webhook_url
+  workspace_id     = tfe_workspace.ws.id
+}
+
+resource "tfe_variable" "vars" {
+  for_each = var.vars
+
+  key          = each.key
+  value        = each.value
+  category     = "terraform"
+  workspace_id = tfe_workspace.ws.id
+  sensitive    = false
+}
+
+resource "tfe_variable" "sensitive" {
+  for_each = var.secret-vars
+
+  key          = each.key
+  value        = each.value
+  category     = "terraform"
+  workspace_id = tfe_workspace.ws.id
+  sensitive    = true
+}
+
+resource "tfe_variable" "env" {
+  for_each = var.env-vars
+
+  key          = each.key
+  value        = each.value
+  category     = "env"
+  workspace_id = tfe_workspace.ws.id
+  sensitive    = true
+}

terraform/modules/tfc-workspace/outputs.tf

@@ -0,0 +1,3 @@
+output "workspace" {
+  value = tfe_workspace.ws[*]
+}

terraform/modules/tfc-workspace/variables.tf

@@ -0,0 +1,54 @@
+variable "workspace-name" {
+  type = string
+}
+
+variable "secret-vars" {
+  type    = map(string)
+  default = {}
+}
+
+variable "vars" {
+  type    = map(string)
+  default = {}
+}
+
+variable "env-vars" {
+  type    = map(string)
+  default = {}
+}
+
+variable "repo" {
+  type = string
+}
+
+variable "directory" {
+  type    = string
+  default = "/"
+}
+
+variable "branch" {
+  type    = string
+  default = "master"
+}
+
+variable "auto_apply" {
+  type    = bool
+  default = false
+}
+
+variable "dependent_modules" {
+  type    = list(string)
+  default = []
+}
+
+variable "tfc_oauth_token_id" {
+  type = string
+}
+
+variable "tfc_org" {
+  type = string
+}
+
+variable "tfc_webhook_url" {
+  type = string
+}

terraform/modules/tfc-workspace/version.tf

@@ -0,0 +1,7 @@
+terraform {
+  required_version = ">=0.12.6"
+}
+
+provider "tfe" {
+  version = ">=0.15.0"
+}

terraform/platform/app/environments.tf

@@ -0,0 +1,13 @@
+module "app-env-prod" {
+  source = "github.com/roleypoly/devops.git//terraform/modules/cluster-environment"
+
+  environment_tag = "production"
+  app_name        = "roleypoly"
+}
+
+module "app-env-stage" {
+  source = "github.com/roleypoly/devops.git//terraform/modules/cluster-environment"
+
+  environment_tag = "staging"
+  app_name        = "roleypoly"
+}

terraform/platform/app/provision.tf

@@ -0,0 +1,47 @@
+terraform {
+  required_version = ">=0.12.6"
+
+  backend "remote" {
+    organization = "Roleypoly"
+
+    workspaces {
+      name = "roleypoly-platform-app"
+    }
+  }
+}
+
+/*
+    Terraform Cloud
+*/
+variable "tfc_email" { type = string }
+variable "tfc_oauth_token_id" { type = string }
+variable "tfc_webhook_url" { type = string }
+provider "tfe" {
+  version = ">=0.15.0"
+}
+
+/*
+    Cloudflare (for tfc vars)
+*/
+variable "cloudflare_token" { type = string }
+variable "cloudflare_email" { type = string }
+variable "cloudflare_zone_id" { type = string }
+provider "cloudflare" {
+  version              = ">=2.0"
+  email                = var.cloudflare_email
+  api_token            = var.cloudflare_token
+  api_user_service_key = var.cloudflare_origin_ca_token
+}
+
+/*
+    Kubernetes
+*/
+variable "k8s_endpoint" { type = string }
+variable "k8s_token" { type = string }
+variable "k8s_cert" { type = string }
+provider "kubernetes" {
+  load_config_file       = false
+  token                  = var.k8s_token
+  host                   = var.k8s_endpoint
+  cluster_ca_certificate = var.k8s_cert
+}

terraform/platform/app/workspaces.tf

@@ -0,0 +1,76 @@
+locals {
+  repo    = "roleypoly/devops"
+  branch  = "master"
+  tfc_org = "Roleypoly"
+
+  common_vars = {}
+  common_secret_vars = {
+    cloudflare_token   = var.cloudflare_token,
+    cloudflare_email   = var.cloudflare_email,
+    cloudflare_zone_id = var.cloudflare_zone_id,
+    k8s_endpoint       = var.k8s_endpoint,
+  }
+}
+
+module "tfcws-production" {
+  source             = "github.com/roleypoly/devops.git//terraform/modules/tfc-workspace"
+  workspace-name     = "roleypoly-app-production"
+  repo               = local.repo
+  branch             = local.branch
+  tfc_webhook_url    = var.tfc_webhook_url
+  directory          = "terraform/app"
+  auto_apply         = false
+  dependent_modules  = []
+  tfc_org            = local.tfc_org
+  tfc_oauth_token_id = var.tfc_oauth_token_id
+
+  vars = merge(local.common_vars, {
+    environment_tag  = "production",
+    ingress_hostname = "prd.roleypoly-nyc.kc"
+    k8s_namespace    = module.app-env-prod.namespace,
+  })
+
+  secret-vars = merge(local.common_secret_vars, {
+    k8s_cert = var.k8s_cert,
+  })
+}
+
+module "tfcws-staging" {
+  source             = "github.com/roleypoly/devops.git//terraform/modules/tfc-workspace"
+  workspace-name     = "roleypoly-app-staging"
+  repo               = local.repo
+  branch             = local.branch
+  tfc_webhook_url    = var.tfc_webhook_url
+  directory          = "terraform/app"
+  auto_apply         = true
+  dependent_modules  = []
+  tfc_org            = local.tfc_org
+  tfc_oauth_token_id = var.tfc_oauth_token_id
+
+  vars = merge(local.common_vars, {
+    environment_tag  = "staging",
+    ingress_hostname = "stg.roleypoly-nyc.kc"
+    k8s_namespace    = module.app-env-stage.namespace,
+  })
+
+  secret-vars = merge(local.common_secret_vars, {
+    k8s_cert = var.k8s_cert,
+  })
+}
+
+// Due to quirk, we must set secret vars manually.
+resource "tfe_variable" "k8s-token-prod" {
+  key          = "k8s_token"
+  value        = module.app-env-prod.service_account_token
+  category     = "terraform"
+  workspace_id = module.tfcws-production.workspace.0.id
+  sensitive    = true
+}
+
+resource "tfe_variable" "k8s-token-stage" {
+  key          = "k8s_token"
+  value        = module.app-env-stage.service_account_token
+  category     = "terraform"
+  workspace_id = module.tfcws-staging.workspace.0.id
+  sensitive    = true
+}

terraform/platform/bootstrap/global.auto.tfvars

@@ -0,0 +1 @@
+gcs_region = "us-east1-d"

terraform/platform/bootstrap/k8s.tf

@@ -0,0 +1,26 @@
+data "digitalocean_kubernetes_versions" "versions" {
+  version_prefix = "1.16."
+}
+
+resource "digitalocean_kubernetes_cluster" "cluster" {
+  name    = "roleypoly-nyc"
+  region  = "nyc1"
+  version = data.digitalocean_kubernetes_versions.versions.latest_version
+
+  node_pool {
+    name       = "default-worker-pool"
+    size       = "s-2vcpu-2gb"
+    node_count = 3
+    labels = {
+      node_type = "static"
+    }
+  }
+}
+
+locals {
+  k8sEndpoint = digitalocean_kubernetes_cluster.cluster.endpoint
+  k8sToken    = digitalocean_kubernetes_cluster.cluster.kube_config[0].token
+  k8sCert = base64decode(
+    digitalocean_kubernetes_cluster.cluster.kube_config[0].cluster_ca_certificate
+  )
+}

terraform/platform/bootstrap/provision.tf

@@ -0,0 +1,58 @@
+terraform {
+  required_version = ">=0.12.6"
+
+  backend "remote" {
+    organization = "Roleypoly"
+
+    workspaces {
+      name = "roleypoly-platform-bootstrap"
+    }
+  }
+}
+
+/*
+    Google Cloud
+*/
+variable "gcs_token" { type = string }
+variable "gcs_region" { type = string }
+variable "gcs_project" { type = string }
+provider "google" {
+  version     = ">=3.18.0"
+  project     = var.gcs_project
+  region      = var.gcs_region
+  credentials = var.gcs_token
+
+  scopes = [
+    "https://www.googleapis.com/auth/devstorage.full_control",
+    "https://www.googleapis.com/auth/cloud-platform",
+  ]
+}
+
+/*
+    DigitalOcean
+*/
+variable "digitalocean_token" { type = string }
+provider "digitalocean" {
+  version = ">=1.16.0"
+  token   = var.digitalocean_token
+}
+
+/*
+    Terraform Cloud
+*/
+variable "tfc_token" { type = string }
+variable "tfc_email" { type = string }
+variable "tfc_oauth_token_id" { type = string }
+variable "tfc_webhook_url" { type = string }
+provider "tfe" {
+  version = ">=0.15.0"
+  token   = var.tfc_token
+}
+
+/*
+    Cloudflare (for tfc vars)
+*/
+variable "cloudflare_token" { type = string }
+variable "cloudflare_email" { type = string }
+variable "cloudflare_zone_id" { type = string }
+variable "cloudflare_origin_ca_token" { type = string }

terraform/platform/bootstrap/tfcloud.tf

@@ -0,0 +1,65 @@
+locals {
+  repo    = "roleypoly/devops"
+  branch  = "master"
+  tfc_org = "Roleypoly"
+}
+
+module "tfcws-services" {
+  source             = "github.com/roleypoly/devops.git//terraform/modules/tfc-workspace"
+  workspace-name     = "roleypoly-platform-services"
+  repo               = local.repo
+  branch             = local.branch
+  tfc_webhook_url    = var.tfc_webhook_url
+  directory          = "terraform/platform/services"
+  auto_apply         = false
+  dependent_modules  = ["nginx-ingress-controller", "cloudflare-dns"]
+  tfc_org            = local.tfc_org
+  tfc_oauth_token_id = var.tfc_oauth_token_id
+
+  secret-vars = {
+    digitalocean_token         = var.digitalocean_token
+    cloudflare_origin_ca_token = var.cloudflare_origin_ca_token
+    cloudflare_zone_id         = var.cloudflare_zone_id
+    cloudflare_token           = var.cloudflare_token
+    cloudflare_email           = var.cloudflare_email
+    vault_gcs_token            = local.vaultGcsSvcacctKey
+    vault_gcs_url              = local.vaultGcsUrl
+    k8s_endpoint               = local.k8sEndpoint
+    k8s_token                  = local.k8sToken
+    k8s_cert                   = local.k8sCert
+  }
+
+  vars = {
+    gcp_region  = var.gcs_region
+    gcp_project = var.gcs_project
+  }
+}
+
+module "tfcws-app" {
+  source             = "github.com/roleypoly/devops.git//terraform/modules/tfc-workspace"
+  workspace-name     = "roleypoly-platform-app"
+  repo               = local.repo
+  branch             = local.branch
+  tfc_webhook_url    = var.tfc_webhook_url
+  directory          = "terraform/platform/app"
+  auto_apply         = false
+  dependent_modules  = ["tfc-workspace", "cluster-environment"]
+  tfc_org            = local.tfc_org
+  tfc_oauth_token_id = var.tfc_oauth_token_id
+
+  secret-vars = {
+    k8s_endpoint       = local.k8sEndpoint
+    k8s_token          = local.k8sToken
+    k8s_cert           = local.k8sCert
+    cloudflare_zone_id = var.cloudflare_zone_id
+    cloudflare_token   = var.cloudflare_token
+    cloudflare_email   = var.cloudflare_email
+    tfc_email          = var.tfc_email
+    tfc_oauth_token_id = var.tfc_oauth_token_id
+    tfc_webhook_url    = var.tfc_webhook_url
+  }
+
+  env-vars = {
+    TFE_TOKEN = var.tfc_token
+  }
+}

terraform/platform/bootstrap/vault-gcs.tf

@@ -0,0 +1,26 @@
+locals {
+  vaultGcsSvcacctKey = google_service_account_key.vault-svcacct-key.private_key
+  vaultGcsUrl        = google_storage_bucket.vault-backend.url
+}
+
+resource "google_service_account" "vault-svcacct" {
+  account_id   = "vault-gcs"
+  display_name = "Vault Svcacct"
+}
+
+resource "google_service_account_key" "vault-svcacct-key" {
+  service_account_id = google_service_account.vault-svcacct.name
+}
+
+resource "google_storage_bucket" "vault-backend" {
+  name = "roleypoly-vault"
+}
+
+resource "google_storage_bucket_acl" "vault-backend-acl" {
+  bucket = google_storage_bucket.vault-backend.name
+
+  role_entity = [
+    "WRITER:user-${google_service_account.vault-svcacct.email}"
+  ]
+}
+

terraform/platform/bootstrap/vault-kms.tf

@@ -0,0 +1,42 @@
+resource "google_kms_key_ring" "vault-kms-ring" {
+  name     = "vault-keyring"
+  location = "global"
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+locals {
+  iam_members = [
+    "serviceAccount:${google_service_account.vault-svcacct.email}"
+  ]
+}
+
+data "google_iam_policy" "vault" {
+  binding {
+    role    = "roles/editor"
+    members = local.iam_members
+  }
+
+  binding {
+    role    = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+    members = local.iam_members
+  }
+}
+
+resource "google_kms_key_ring_iam_policy" "vault-binding" {
+  key_ring_id = google_kms_key_ring.vault-kms-ring.id
+  policy_data = data.google_iam_policy.vault.policy_data
+}
+
+resource "google_kms_crypto_key" "vault-key" {
+  name            = "vault-key"
+  key_ring        = google_kms_key_ring.vault-kms-ring.id
+  rotation_period = "100000s" // just over one day
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+

terraform/platform/services/ingress.tf

@@ -0,0 +1,13 @@
+module "ingress-controller" {
+  source                = "github.com/roleypoly/devops.git//terraform/modules/nginx-ingress-controller"
+  nginx-ingress-version = "0.32.0"
+}
+
+module "cluster-dns" {
+  source             = "github.com/roleypoly/devops.git//terraform/modules/cloudflare-cluster-dns"
+  ingress-name       = module.ingress-controller.service-name
+  ingress-namespace  = module.ingress-controller.service-namespace
+  ingress-endpoint   = module.ingress-controller.service-endpoint
+  cloudflare-zone-id = var.cloudflare_zone_id
+  record-name        = "roleypoly-nyc.kc"
+}

terraform/platform/services/provision.tf

@@ -0,0 +1,56 @@
+terraform {
+  required_version = ">=0.12.6"
+
+  backend "remote" {
+    organization = "Roleypoly"
+
+    workspaces {
+      name = "roleypoly-platform-services"
+    }
+  }
+}
+
+/*
+    DigitalOcean
+*/
+variable "digitalocean_token" { type = string }
+provider "digitalocean" {
+  version = ">=1.16.0"
+  token   = var.digitalocean_token
+}
+
+/*
+    Cloudflare
+*/
+variable "cloudflare_token" { type = string }
+variable "cloudflare_email" { type = string }
+variable "cloudflare_zone_id" { type = string }
+variable "cloudflare_origin_ca_token" { type = string }
+provider "cloudflare" {
+  version              = ">=2.0"
+  email                = var.cloudflare_email
+  api_token            = var.cloudflare_token
+  api_user_service_key = var.cloudflare_origin_ca_token
+}
+
+/*
+    Kubernetes
+*/
+variable "k8s_endpoint" { type = string }
+variable "k8s_token" { type = string }
+variable "k8s_cert" { type = string }
+provider "kubernetes" {
+  load_config_file       = false
+  token                  = var.k8s_token
+  host                   = var.k8s_endpoint
+  cluster_ca_certificate = var.k8s_cert
+}
+
+/*
+    Others
+*/
+variable "vault_gcs_token" { type = string }
+variable "vault_gcs_url" { type = string }
+variable "gcp_project" { type = string }
+variable "gcp_region" { type = string }
+

terraform/platform/services/vault.tf

@@ -0,0 +1,207 @@
+resource "kubernetes_namespace" "vault" {
+  metadata {
+    name = "vault"
+  }
+}
+
+locals {
+  vaultNs = kubernetes_namespace.vault.metadata.0.name
+  vaultLabels = {
+    "app.kubernetes.io/name"    = "vault"
+    "app.kubernetes.io/part-of" = "vault"
+  }
+}
+
+resource "kubernetes_secret" "vault-svcacct" {
+  metadata {
+    generate_name = "vault-svcacct"
+    namespace     = local.vaultNs
+    labels        = local.vaultLabels
+  }
+
+  data = {
+    "vault-service-account.json" = base64decode(var.vault_gcs_token)
+  }
+}
+
+resource "kubernetes_config_map" "vault-cm" {
+  metadata {
+    generate_name = "vault-config"
+    namespace     = local.vaultNs
+    labels        = local.vaultLabels
+  }
+
+  data = {
+    "vault-config.json" = jsonencode({
+      // Enables UI
+      ui = true,
+
+      // Storage with GCS
+      storage = {
+        gcs = {
+          bucket = "roleypoly-vault",
+        }
+      },
+
+      // Auto-seal setup with GCPKMS
+      seal = {
+        gcpckms = {
+          credentials = "/vault/mounted-secrets/vault-service-account.json",
+          project     = var.gcp_project
+          region      = "global"
+          key_ring    = "vault-keyring"
+          crypto_key  = "vault-key"
+        }
+      },
+
+      // TCP
+      listener = {
+        tcp = {
+          address = "0.0.0.0:8200"
+        }
+      },
+
+      // K8s service registration
+      service_registration = {
+        kubernetes = {}
+      }
+    })
+  }
+}
+
+
+
+resource "kubernetes_deployment" "vault" {
+  metadata {
+    name      = "vault"
+    namespace = local.vaultNs
+    labels    = local.vaultLabels
+  }
+
+  spec {
+    replicas = 1
+
+    selector {
+      match_labels = local.vaultLabels
+    }
+
+    template {
+      metadata {
+        labels = local.vaultLabels
+      }
+
+      spec {
+        service_account_name            = kubernetes_service_account.vault-sa.metadata.0.name
+        automount_service_account_token = true
+
+        container {
+          image = "vault:1.5.0"
+          name  = "vault"
+
+          env {
+            name  = "GOOGLE_APPLICATION_CREDENTIALS"
+            value = "/vault/mounted-secrets/vault-service-account.json"
+          }
+
+          env {
+            name = "VAULT_K8S_NAMESPACE"
+            value_from {
+              field_ref {
+                field_path = "metadata.namespace"
+              }
+            }
+          }
+
+          env {
+            name = "VAULT_K8S_POD_NAME"
+            value_from {
+              field_ref {
+                field_path = "metadata.name"
+              }
+            }
+          }
+
+          volume_mount {
+            mount_path = "/vault/mounted-secrets"
+            name       = "vault-secrets"
+            read_only  = true
+          }
+
+          volume_mount {
+            mount_path = "/vault/config/vault-config.json"
+            name       = "vault-config"
+            sub_path   = "vault-config.json"
+          }
+
+          security_context {
+            capabilities {
+              add = ["IPC_LOCK"]
+            }
+          }
+        }
+
+        node_selector = {
+          node_type = "static"
+        }
+
+        restart_policy = "Always"
+
+        volume {
+          name = "vault-secrets"
+          secret {
+            secret_name = kubernetes_secret.vault-svcacct.metadata.0.name
+          }
+        }
+
+        volume {
+          name = "vault-config"
+          config_map {
+            name = kubernetes_config_map.vault-cm.metadata.0.name
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service_account" "vault-sa" {
+  metadata {
+    namespace = local.vaultNs
+    name      = "vault"
+    labels    = local.vaultLabels
+  }
+}
+
+resource "kubernetes_role" "vault-sa-role" {
+  metadata {
+    namespace = local.vaultNs
+    name      = "vault"
+    labels    = local.vaultLabels
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["pods"]
+    verbs      = ["get", "update"]
+  }
+}
+
+resource "kubernetes_role_binding" "vault-sa-rb" {
+  metadata {
+    namespace = local.vaultNs
+    name      = "vault-rb"
+    labels    = local.vaultLabels
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "Role"
+    name      = kubernetes_role.vault-sa-role.metadata.0.name
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.vault-sa.metadata.0.name
+    namespace = local.vaultNs
+  }
+}