temp tf

terraform/platform/app/environments.tf

@@ -0,0 +1,13 @@
+module "app-env-prod" {
+  source = "github.com/roleypoly/devops.git//terraform/modules/cluster-environment"
+
+  environment_tag = "production"
+  app_name        = "roleypoly"
+}
+
+module "app-env-stage" {
+  source = "github.com/roleypoly/devops.git//terraform/modules/cluster-environment"
+
+  environment_tag = "staging"
+  app_name        = "roleypoly"
+}

terraform/platform/app/provision.tf

@@ -0,0 +1,47 @@
+terraform {
+  required_version = ">=0.12.6"
+
+  backend "remote" {
+    organization = "Roleypoly"
+
+    workspaces {
+      name = "roleypoly-platform-app"
+    }
+  }
+}
+
+/*
+    Terraform Cloud
+*/
+variable "tfc_email" { type = string }
+variable "tfc_oauth_token_id" { type = string }
+variable "tfc_webhook_url" { type = string }
+provider "tfe" {
+  version = ">=0.15.0"
+}
+
+/*
+    Cloudflare (for tfc vars)
+*/
+variable "cloudflare_token" { type = string }
+variable "cloudflare_email" { type = string }
+variable "cloudflare_zone_id" { type = string }
+provider "cloudflare" {
+  version              = ">=2.0"
+  email                = var.cloudflare_email
+  api_token            = var.cloudflare_token
+  api_user_service_key = var.cloudflare_origin_ca_token
+}
+
+/*
+    Kubernetes
+*/
+variable "k8s_endpoint" { type = string }
+variable "k8s_token" { type = string }
+variable "k8s_cert" { type = string }
+provider "kubernetes" {
+  load_config_file       = false
+  token                  = var.k8s_token
+  host                   = var.k8s_endpoint
+  cluster_ca_certificate = var.k8s_cert
+}

terraform/platform/app/workspaces.tf

@@ -0,0 +1,76 @@
+locals {
+  repo    = "roleypoly/devops"
+  branch  = "master"
+  tfc_org = "Roleypoly"
+
+  common_vars = {}
+  common_secret_vars = {
+    cloudflare_token   = var.cloudflare_token,
+    cloudflare_email   = var.cloudflare_email,
+    cloudflare_zone_id = var.cloudflare_zone_id,
+    k8s_endpoint       = var.k8s_endpoint,
+  }
+}
+
+module "tfcws-production" {
+  source             = "github.com/roleypoly/devops.git//terraform/modules/tfc-workspace"
+  workspace-name     = "roleypoly-app-production"
+  repo               = local.repo
+  branch             = local.branch
+  tfc_webhook_url    = var.tfc_webhook_url
+  directory          = "terraform/app"
+  auto_apply         = false
+  dependent_modules  = []
+  tfc_org            = local.tfc_org
+  tfc_oauth_token_id = var.tfc_oauth_token_id
+
+  vars = merge(local.common_vars, {
+    environment_tag  = "production",
+    ingress_hostname = "prd.roleypoly-nyc.kc"
+    k8s_namespace    = module.app-env-prod.namespace,
+  })
+
+  secret-vars = merge(local.common_secret_vars, {
+    k8s_cert = var.k8s_cert,
+  })
+}
+
+module "tfcws-staging" {
+  source             = "github.com/roleypoly/devops.git//terraform/modules/tfc-workspace"
+  workspace-name     = "roleypoly-app-staging"
+  repo               = local.repo
+  branch             = local.branch
+  tfc_webhook_url    = var.tfc_webhook_url
+  directory          = "terraform/app"
+  auto_apply         = true
+  dependent_modules  = []
+  tfc_org            = local.tfc_org
+  tfc_oauth_token_id = var.tfc_oauth_token_id
+
+  vars = merge(local.common_vars, {
+    environment_tag  = "staging",
+    ingress_hostname = "stg.roleypoly-nyc.kc"
+    k8s_namespace    = module.app-env-stage.namespace,
+  })
+
+  secret-vars = merge(local.common_secret_vars, {
+    k8s_cert = var.k8s_cert,
+  })
+}
+
+// Due to quirk, we must set secret vars manually.
+resource "tfe_variable" "k8s-token-prod" {
+  key          = "k8s_token"
+  value        = module.app-env-prod.service_account_token
+  category     = "terraform"
+  workspace_id = module.tfcws-production.workspace.0.id
+  sensitive    = true
+}
+
+resource "tfe_variable" "k8s-token-stage" {
+  key          = "k8s_token"
+  value        = module.app-env-stage.service_account_token
+  category     = "terraform"
+  workspace_id = module.tfcws-staging.workspace.0.id
+  sensitive    = true
+}

terraform/platform/bootstrap/global.auto.tfvars

@@ -0,0 +1 @@
+gcs_region = "us-east1-d"

terraform/platform/bootstrap/k8s.tf

@@ -0,0 +1,26 @@
+data "digitalocean_kubernetes_versions" "versions" {
+  version_prefix = "1.16."
+}
+
+resource "digitalocean_kubernetes_cluster" "cluster" {
+  name    = "roleypoly-nyc"
+  region  = "nyc1"
+  version = data.digitalocean_kubernetes_versions.versions.latest_version
+
+  node_pool {
+    name       = "default-worker-pool"
+    size       = "s-2vcpu-2gb"
+    node_count = 3
+    labels = {
+      node_type = "static"
+    }
+  }
+}
+
+locals {
+  k8sEndpoint = digitalocean_kubernetes_cluster.cluster.endpoint
+  k8sToken    = digitalocean_kubernetes_cluster.cluster.kube_config[0].token
+  k8sCert = base64decode(
+    digitalocean_kubernetes_cluster.cluster.kube_config[0].cluster_ca_certificate
+  )
+}

terraform/platform/bootstrap/provision.tf

@@ -0,0 +1,58 @@
+terraform {
+  required_version = ">=0.12.6"
+
+  backend "remote" {
+    organization = "Roleypoly"
+
+    workspaces {
+      name = "roleypoly-platform-bootstrap"
+    }
+  }
+}
+
+/*
+    Google Cloud
+*/
+variable "gcs_token" { type = string }
+variable "gcs_region" { type = string }
+variable "gcs_project" { type = string }
+provider "google" {
+  version     = ">=3.18.0"
+  project     = var.gcs_project
+  region      = var.gcs_region
+  credentials = var.gcs_token
+
+  scopes = [
+    "https://www.googleapis.com/auth/devstorage.full_control",
+    "https://www.googleapis.com/auth/cloud-platform",
+  ]
+}
+
+/*
+    DigitalOcean
+*/
+variable "digitalocean_token" { type = string }
+provider "digitalocean" {
+  version = ">=1.16.0"
+  token   = var.digitalocean_token
+}
+
+/*
+    Terraform Cloud
+*/
+variable "tfc_token" { type = string }
+variable "tfc_email" { type = string }
+variable "tfc_oauth_token_id" { type = string }
+variable "tfc_webhook_url" { type = string }
+provider "tfe" {
+  version = ">=0.15.0"
+  token   = var.tfc_token
+}
+
+/*
+    Cloudflare (for tfc vars)
+*/
+variable "cloudflare_token" { type = string }
+variable "cloudflare_email" { type = string }
+variable "cloudflare_zone_id" { type = string }
+variable "cloudflare_origin_ca_token" { type = string }

terraform/platform/bootstrap/tfcloud.tf

@@ -0,0 +1,65 @@
+locals {
+  repo    = "roleypoly/devops"
+  branch  = "master"
+  tfc_org = "Roleypoly"
+}
+
+module "tfcws-services" {
+  source             = "github.com/roleypoly/devops.git//terraform/modules/tfc-workspace"
+  workspace-name     = "roleypoly-platform-services"
+  repo               = local.repo
+  branch             = local.branch
+  tfc_webhook_url    = var.tfc_webhook_url
+  directory          = "terraform/platform/services"
+  auto_apply         = false
+  dependent_modules  = ["nginx-ingress-controller", "cloudflare-dns"]
+  tfc_org            = local.tfc_org
+  tfc_oauth_token_id = var.tfc_oauth_token_id
+
+  secret-vars = {
+    digitalocean_token         = var.digitalocean_token
+    cloudflare_origin_ca_token = var.cloudflare_origin_ca_token
+    cloudflare_zone_id         = var.cloudflare_zone_id
+    cloudflare_token           = var.cloudflare_token
+    cloudflare_email           = var.cloudflare_email
+    vault_gcs_token            = local.vaultGcsSvcacctKey
+    vault_gcs_url              = local.vaultGcsUrl
+    k8s_endpoint               = local.k8sEndpoint
+    k8s_token                  = local.k8sToken
+    k8s_cert                   = local.k8sCert
+  }
+
+  vars = {
+    gcp_region  = var.gcs_region
+    gcp_project = var.gcs_project
+  }
+}
+
+module "tfcws-app" {
+  source             = "github.com/roleypoly/devops.git//terraform/modules/tfc-workspace"
+  workspace-name     = "roleypoly-platform-app"
+  repo               = local.repo
+  branch             = local.branch
+  tfc_webhook_url    = var.tfc_webhook_url
+  directory          = "terraform/platform/app"
+  auto_apply         = false
+  dependent_modules  = ["tfc-workspace", "cluster-environment"]
+  tfc_org            = local.tfc_org
+  tfc_oauth_token_id = var.tfc_oauth_token_id
+
+  secret-vars = {
+    k8s_endpoint       = local.k8sEndpoint
+    k8s_token          = local.k8sToken
+    k8s_cert           = local.k8sCert
+    cloudflare_zone_id = var.cloudflare_zone_id
+    cloudflare_token   = var.cloudflare_token
+    cloudflare_email   = var.cloudflare_email
+    tfc_email          = var.tfc_email
+    tfc_oauth_token_id = var.tfc_oauth_token_id
+    tfc_webhook_url    = var.tfc_webhook_url
+  }
+
+  env-vars = {
+    TFE_TOKEN = var.tfc_token
+  }
+}

terraform/platform/bootstrap/vault-gcs.tf

@@ -0,0 +1,26 @@
+locals {
+  vaultGcsSvcacctKey = google_service_account_key.vault-svcacct-key.private_key
+  vaultGcsUrl        = google_storage_bucket.vault-backend.url
+}
+
+resource "google_service_account" "vault-svcacct" {
+  account_id   = "vault-gcs"
+  display_name = "Vault Svcacct"
+}
+
+resource "google_service_account_key" "vault-svcacct-key" {
+  service_account_id = google_service_account.vault-svcacct.name
+}
+
+resource "google_storage_bucket" "vault-backend" {
+  name = "roleypoly-vault"
+}
+
+resource "google_storage_bucket_acl" "vault-backend-acl" {
+  bucket = google_storage_bucket.vault-backend.name
+
+  role_entity = [
+    "WRITER:user-${google_service_account.vault-svcacct.email}"
+  ]
+}
+

terraform/platform/bootstrap/vault-kms.tf

@@ -0,0 +1,42 @@
+resource "google_kms_key_ring" "vault-kms-ring" {
+  name     = "vault-keyring"
+  location = "global"
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+locals {
+  iam_members = [
+    "serviceAccount:${google_service_account.vault-svcacct.email}"
+  ]
+}
+
+data "google_iam_policy" "vault" {
+  binding {
+    role    = "roles/editor"
+    members = local.iam_members
+  }
+
+  binding {
+    role    = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+    members = local.iam_members
+  }
+}
+
+resource "google_kms_key_ring_iam_policy" "vault-binding" {
+  key_ring_id = google_kms_key_ring.vault-kms-ring.id
+  policy_data = data.google_iam_policy.vault.policy_data
+}
+
+resource "google_kms_crypto_key" "vault-key" {
+  name            = "vault-key"
+  key_ring        = google_kms_key_ring.vault-kms-ring.id
+  rotation_period = "100000s" // just over one day
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+

terraform/platform/services/ingress.tf

@@ -0,0 +1,13 @@
+module "ingress-controller" {
+  source                = "github.com/roleypoly/devops.git//terraform/modules/nginx-ingress-controller"
+  nginx-ingress-version = "0.32.0"
+}
+
+module "cluster-dns" {
+  source             = "github.com/roleypoly/devops.git//terraform/modules/cloudflare-cluster-dns"
+  ingress-name       = module.ingress-controller.service-name
+  ingress-namespace  = module.ingress-controller.service-namespace
+  ingress-endpoint   = module.ingress-controller.service-endpoint
+  cloudflare-zone-id = var.cloudflare_zone_id
+  record-name        = "roleypoly-nyc.kc"
+}

terraform/platform/services/provision.tf

@@ -0,0 +1,56 @@
+terraform {
+  required_version = ">=0.12.6"
+
+  backend "remote" {
+    organization = "Roleypoly"
+
+    workspaces {
+      name = "roleypoly-platform-services"
+    }
+  }
+}
+
+/*
+    DigitalOcean
+*/
+variable "digitalocean_token" { type = string }
+provider "digitalocean" {
+  version = ">=1.16.0"
+  token   = var.digitalocean_token
+}
+
+/*
+    Cloudflare
+*/
+variable "cloudflare_token" { type = string }
+variable "cloudflare_email" { type = string }
+variable "cloudflare_zone_id" { type = string }
+variable "cloudflare_origin_ca_token" { type = string }
+provider "cloudflare" {
+  version              = ">=2.0"
+  email                = var.cloudflare_email
+  api_token            = var.cloudflare_token
+  api_user_service_key = var.cloudflare_origin_ca_token
+}
+
+/*
+    Kubernetes
+*/
+variable "k8s_endpoint" { type = string }
+variable "k8s_token" { type = string }
+variable "k8s_cert" { type = string }
+provider "kubernetes" {
+  load_config_file       = false
+  token                  = var.k8s_token
+  host                   = var.k8s_endpoint
+  cluster_ca_certificate = var.k8s_cert
+}
+
+/*
+    Others
+*/
+variable "vault_gcs_token" { type = string }
+variable "vault_gcs_url" { type = string }
+variable "gcp_project" { type = string }
+variable "gcp_region" { type = string }
+

terraform/platform/services/vault.tf

@@ -0,0 +1,207 @@
+resource "kubernetes_namespace" "vault" {
+  metadata {
+    name = "vault"
+  }
+}
+
+locals {
+  vaultNs = kubernetes_namespace.vault.metadata.0.name
+  vaultLabels = {
+    "app.kubernetes.io/name"    = "vault"
+    "app.kubernetes.io/part-of" = "vault"
+  }
+}
+
+resource "kubernetes_secret" "vault-svcacct" {
+  metadata {
+    generate_name = "vault-svcacct"
+    namespace     = local.vaultNs
+    labels        = local.vaultLabels
+  }
+
+  data = {
+    "vault-service-account.json" = base64decode(var.vault_gcs_token)
+  }
+}
+
+resource "kubernetes_config_map" "vault-cm" {
+  metadata {
+    generate_name = "vault-config"
+    namespace     = local.vaultNs
+    labels        = local.vaultLabels
+  }
+
+  data = {
+    "vault-config.json" = jsonencode({
+      // Enables UI
+      ui = true,
+
+      // Storage with GCS
+      storage = {
+        gcs = {
+          bucket = "roleypoly-vault",
+        }
+      },
+
+      // Auto-seal setup with GCPKMS
+      seal = {
+        gcpckms = {
+          credentials = "/vault/mounted-secrets/vault-service-account.json",
+          project     = var.gcp_project
+          region      = "global"
+          key_ring    = "vault-keyring"
+          crypto_key  = "vault-key"
+        }
+      },
+
+      // TCP
+      listener = {
+        tcp = {
+          address = "0.0.0.0:8200"
+        }
+      },
+
+      // K8s service registration
+      service_registration = {
+        kubernetes = {}
+      }
+    })
+  }
+}
+
+
+
+resource "kubernetes_deployment" "vault" {
+  metadata {
+    name      = "vault"
+    namespace = local.vaultNs
+    labels    = local.vaultLabels
+  }
+
+  spec {
+    replicas = 1
+
+    selector {
+      match_labels = local.vaultLabels
+    }
+
+    template {
+      metadata {
+        labels = local.vaultLabels
+      }
+
+      spec {
+        service_account_name            = kubernetes_service_account.vault-sa.metadata.0.name
+        automount_service_account_token = true
+
+        container {
+          image = "vault:1.5.0"
+          name  = "vault"
+
+          env {
+            name  = "GOOGLE_APPLICATION_CREDENTIALS"
+            value = "/vault/mounted-secrets/vault-service-account.json"
+          }
+
+          env {
+            name = "VAULT_K8S_NAMESPACE"
+            value_from {
+              field_ref {
+                field_path = "metadata.namespace"
+              }
+            }
+          }
+
+          env {
+            name = "VAULT_K8S_POD_NAME"
+            value_from {
+              field_ref {
+                field_path = "metadata.name"
+              }
+            }
+          }
+
+          volume_mount {
+            mount_path = "/vault/mounted-secrets"
+            name       = "vault-secrets"
+            read_only  = true
+          }
+
+          volume_mount {
+            mount_path = "/vault/config/vault-config.json"
+            name       = "vault-config"
+            sub_path   = "vault-config.json"
+          }
+
+          security_context {
+            capabilities {
+              add = ["IPC_LOCK"]
+            }
+          }
+        }
+
+        node_selector = {
+          node_type = "static"
+        }
+
+        restart_policy = "Always"
+
+        volume {
+          name = "vault-secrets"
+          secret {
+            secret_name = kubernetes_secret.vault-svcacct.metadata.0.name
+          }
+        }
+
+        volume {
+          name = "vault-config"
+          config_map {
+            name = kubernetes_config_map.vault-cm.metadata.0.name
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service_account" "vault-sa" {
+  metadata {
+    namespace = local.vaultNs
+    name      = "vault"
+    labels    = local.vaultLabels
+  }
+}
+
+resource "kubernetes_role" "vault-sa-role" {
+  metadata {
+    namespace = local.vaultNs
+    name      = "vault"
+    labels    = local.vaultLabels
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["pods"]
+    verbs      = ["get", "update"]
+  }
+}
+
+resource "kubernetes_role_binding" "vault-sa-rb" {
+  metadata {
+    namespace = local.vaultNs
+    name      = "vault-rb"
+    labels    = local.vaultLabels
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "Role"
+    name      = kubernetes_role.vault-sa-role.metadata.0.name
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.vault-sa.metadata.0.name
+    namespace = local.vaultNs
+  }
+}