From 7f8a459ab31b64f54fba523741987d341ed70666 Mon Sep 17 00:00:00 2001 From: Kieran Patel Date: Tue, 8 Oct 2024 18:30:56 +0200 Subject: [PATCH 01/38] Fix crossplane script downloading --- .github/workflows/pr-crossplane.yaml | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pr-crossplane.yaml b/.github/workflows/pr-crossplane.yaml index 5d2d9877..70fe465d 100644 --- a/.github/workflows/pr-crossplane.yaml +++ b/.github/workflows/pr-crossplane.yaml @@ -76,11 +76,6 @@ jobs: path: ${{ github.workspace }}/dist zip_lambda_workflow_step: ${{ inputs.zip_lambda_workflow_step }} - - name: Display structure of downloaded files - run: ls -R - working-directory: ${{ steps.download.outputs.download-path }} - - - uses: hashicorp/setup-terraform@v1 with: terraform_version: ~1.4 @@ -89,26 +84,16 @@ jobs: id: downloadscripts with: name: scripts - path: ./scripts - - - name: copy scripts to scripts folder in working directory - run: | - mkdir ./scripts && ls - cp -r ${{ github.workspace }}/scripts/.github/workflows ./scripts - shell: sh + path: /tmp/scripts - name: Install 1Password Cli, patch claim run: | - curl https://cache.agilebits.com/dist/1P/op2/pkg/v2.18.0/op_linux_amd64_v2.18.0.zip > op.zip - unzip op.zip - sudo mv op /usr/local/bin - rm op.zip - ls + curl https://cache.agilebits.com/dist/1P/op2/pkg/v2.18.0/op_linux_amd64_v2.23.0.zip > op.zip + sudo unzip op.zip -d /usr/local/bin && rm op.zip python -m pip install "ruamel.yaml<0.18.0" - python scripts/workflows/scripts/patch.py - - + python /tmp/scripts/.github/workflows/scripts/patch.py + for file in *claims.yaml; do if [ -f "$file" ]; then cat $file From ca9ba6852544e62738f40f10d79ba8b763029cd0 Mon Sep 17 00:00:00 2001 From: Kieran Patel Date: Wed, 9 Oct 2024 16:03:40 +0200 Subject: [PATCH 02/38] rossplane new CI --- .github/workflows/crossplane.yaml | 85 +++++++++++++++++++++++++++++++ crossplane/.gitignore | 2 + crossplane/backend-config.hcl | 3 ++ crossplane/main.tf | 58 +++++++++++++++++++++ crossplane/modify-claims.sh | 81 +++++++++++++++++++++++++++++ crossplane/prod.tfvars | 2 + crossplane/staging.tfvars | 2 + crossplane/variables.tf | 19 +++++++ crossplane/versions.tf | 25 +++++++++ renovate.json | 6 --- 10 files changed, 277 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/crossplane.yaml create mode 100644 crossplane/.gitignore create mode 100644 crossplane/backend-config.hcl create mode 100644 crossplane/main.tf create mode 100755 crossplane/modify-claims.sh create mode 100644 crossplane/prod.tfvars create mode 100644 crossplane/staging.tfvars create mode 100644 crossplane/variables.tf create mode 100644 crossplane/versions.tf delete mode 100644 renovate.json diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml new file mode 100644 index 00000000..9aa58665 --- /dev/null +++ b/.github/workflows/crossplane.yaml @@ -0,0 +1,85 @@ +on: + workflow_call: + inputs: + environment: + required: true + type: string + service_name: + required: true + type: string + +permissions: + id-token: write + contents: read + pull-requests: write + statuses: write + +jobs: + crossplane-terraform: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - + name: Checkout shared-actions + uses: actions/checkout@v4 + with: + repository: vimeda/shared-actions + path: ./scripts + ref: feature/shared-crossplane + - + name: Checkout service + uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + path: ./scripts/${{ inputs.service_name }} + - + name: Install yq + uses: chrisdickinson/setup-yq@v1.0.1 + with: + yq-version: v4.25.3 + - + name: Install 1Password CLI + uses: 1password/install-cli-action@v1 + - + name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-${{ inputs.service_name }}-role + role-session-name: ga-${{ inputs.service_name }} + aws-region: eu-central-1 + - + name: Install Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ~1.9 + + - + name: Terraform Init + working-directory: ./scripts/crossplane + run: | + terraform init \ + -backend-config="region=eu-central-1" \ + -backend-config="bucket=terraform-eks" \ + -backend-config="key=crossplane/${{ inputs.service_name }}" + + - + name: Terraform Validate + working-directory: ./scripts/crossplane + run: | + terraform validate -no-color + - + name: Terraform + working-directory: ./scripts/crossplane + run: | + echo "${{ secrets.STAGING_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml + terraform apply -auto-approve -no-color \ + -var-file=${{ inputs.environment }}.tfvars \ + -var="commit_hash=${{ github.sha }}" \ + -var="service_name=${{ inputs.service_name }}" + env: + TF_WORKSPACE: ${{ inputs.environment }} + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.STAGING_ONEPASSWORD_SERVICEACCOUNT_TOKEN }} + + diff --git a/crossplane/.gitignore b/crossplane/.gitignore new file mode 100644 index 00000000..cc5778cf --- /dev/null +++ b/crossplane/.gitignore @@ -0,0 +1,2 @@ +.terraform/ +.terraform.lock.hcl diff --git a/crossplane/backend-config.hcl b/crossplane/backend-config.hcl new file mode 100644 index 00000000..2396500d --- /dev/null +++ b/crossplane/backend-config.hcl @@ -0,0 +1,3 @@ +region = "eu-central-1" +bucket = "terraform-eks" +key = "crossplane/${service_name}/state.tf" diff --git a/crossplane/main.tf b/crossplane/main.tf new file mode 100644 index 00000000..df1f6199 --- /dev/null +++ b/crossplane/main.tf @@ -0,0 +1,58 @@ +data "template_file" "claims" { + for_each = fileset("../${var.service_name}/configs/crossplane/${terraform.workspace}", "*.yaml") + template = file("../${var.service_name}/configs/crossplane/${terraform.workspace}/${each.value}") + + vars = { + commit_hash = var.commit_hash + } +} + +# +# Use external data source to run the bash script to modify the claims +data "external" "modified_yaml" { + for_each = data.template_file.claims + program = ["bash", "${path.module}/modify-claims.sh"] + + query = { + vault_id = var.vault_id + claim_yaml = each.value.rendered + env = terraform.workspace + } +} + +output "modified_yaml" { + value = data.external.modified_yaml +} + +# Locals for decoding the updated YAML from the external script output +locals { + # Define the path to the directory containing YAML files + yaml_dir = "${path.module}/tmp" # Adjust this to your module's relative path + yaml_files = fileset(local.yaml_dir, "*.yaml") # Get all YAML files in the specified directory +} + +# Parse the YAML content into Kubernetes documents using kubectl provider +data "kubectl_file_documents" "claims" { + depends_on = [data.external.modified_yaml] # Ensure this runs after the external data source + for_each = data.external.modified_yaml + content = yamlencode(jsondecode(each.value.result.manifest)) +} + +output "kubectl_manifest" { + value = data.kubectl_file_documents.claims +} + +locals { + # Collect all manifests into a flat list + manifests_array = flatten([ + for doc in data.kubectl_file_documents.claims : [ + for _, manifest in doc.manifests : manifest + ] + ]) +} + +resource "kubectl_manifest" "apply" { + depends_on = [data.kubectl_file_documents.claims] + for_each = toset(local.manifests_array) + yaml_body = each.value # Apply each manifest from the array +} diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh new file mode 100755 index 00000000..9bb4da48 --- /dev/null +++ b/crossplane/modify-claims.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -euov pipefail + +# Ensure the tmp/ folder exists in the current working directory +mkdir -p tmp/ + +# Extract variables using jq +eval "$(jq -r '@sh "ENV=\(.env) VAULT_ID=\(.vault_id) CLAIM_YAML=\(.claim_yaml)"')" + +# Generate a SHA256 hash from CLAIM_YAML and use part of it for the file name +hash=$(echo -n "$CLAIM_YAML" | sha256sum | cut -d' ' -f1) + +# Create a temporary file in the tmp/ folder, prefixed with 'tmpfile_' and suffixed with the hash +temp_yaml_file="tmp/tmpfile_${hash}.yaml" + +# Write the input YAML to the temporary file for processing +echo "$CLAIM_YAML" > "$temp_yaml_file" + +# Predefined arrays of claim types to process +CLAIM_TYPES_LAMBDA=("XLykonLambda" "XLykonLambdaDockerImage") +CLAIM_TYPES_GOAPP=("XLykonGoApp") + +# Check the kind of the YAML +kind=$(yq eval '.kind' "$temp_yaml_file") + +# Function to add VPC configuration based on environment +add_vpc_config() { + local env="$1" + local config + + if [[ "$env" == "staging" ]]; then + config='[{"securityGroupIds":["sg-03c24245575c1ebc0"],"subnetIds":["subnet-011cb6fe763310759","subnet-08deca209f9e46ebb","subnet-06e62ab1abfd70465"]}]' + elif [[ "$env" == "prod" ]]; then + config='[{"securityGroupIds":["sg-03c24245575c1ebc0"],"subnetIds":["subnet-011cb6fe763310759","subnet-08deca20d9f9e46ebb","subnet-06e62ab1abfd70465"]}]' + else + echo "Error: Unsupported environment $env" + exit 1 + fi + + # Set the vpcConfig directly as an array + yq eval ".spec.parameters.vpcConfig = $config" -i "$temp_yaml_file" +} + +if [[ " ${CLAIM_TYPES_LAMBDA[@]} " =~ " ${kind} " ]]; then + # Handle XLykonLambda and XLykonLambdaDockerImage + service_name=$(yq eval '.spec.parameters.service_name' "$temp_yaml_file") + + if [[ -z "$service_name" ]]; then + echo "Warning: service_name is not defined, skipping secret fetching." + else + secrets=$(op items get "$service_name" --vault="$VAULT_ID" --format=json | jq '.fields | map({(.label): .value}) | add') + + if [[ -z "$secrets" ]]; then + echo "Warning: Failed to fetch secrets for $service_name, skipping secret addition." + else + # Wrap secrets in an array with 'variables' + secrets_with_variables=$(jq -n --argjson secrets "$secrets" '[{"variables": $secrets}]') + + # Update the YAML file with the secrets under 'secrets' field + yq eval ".spec.parameters.secrets = $secrets_with_variables" -i "$temp_yaml_file" + fi + fi + add_vpc_config "$ENV" # Add VPC config only for Lambda types +elif [[ " ${CLAIM_TYPES_GOAPP[@]} " =~ " ${kind} " ]]; then + # Handle XLykonGoApp + if [[ "$ENV" == "staging" ]]; then + vault_id="errsir3kqd4gdjgaxliofyskey" + elif [[ "$ENV" == "prod" ]]; then + vault_id="37y43e5v2qd3iptgt7wgyk34ga" + else + echo "Error: Unsupported environment $ENV" + exit 1 + fi + + yq eval ".spec.parameters.vault_id = \"$vault_id\"" -i "$temp_yaml_file" +fi + +# Convert the final YAML to JSON for Terraform +manifest=$(yq eval -o=json "$temp_yaml_file") +jq -n --arg manifest "$manifest" '{ manifest: $manifest }' diff --git a/crossplane/prod.tfvars b/crossplane/prod.tfvars new file mode 100644 index 00000000..2d38f8e7 --- /dev/null +++ b/crossplane/prod.tfvars @@ -0,0 +1,2 @@ +cluster_name = "prod-eks-v2" +vault_id = "37y43e5v2qd3iptgt7wgyk34ga" diff --git a/crossplane/staging.tfvars b/crossplane/staging.tfvars new file mode 100644 index 00000000..18baf14d --- /dev/null +++ b/crossplane/staging.tfvars @@ -0,0 +1,2 @@ +cluster_name = "staging-eks-v2" +vault_id = "errsir3kqd4gdjgaxliofyskey" diff --git a/crossplane/variables.tf b/crossplane/variables.tf new file mode 100644 index 00000000..0b1db451 --- /dev/null +++ b/crossplane/variables.tf @@ -0,0 +1,19 @@ +variable "commit_hash" { + description = "git commit hash, which will be used to tag the docker image" + type = string +} + +variable "cluster_name" { + description = "name of the eks cluster" + type = string +} + +variable "service_name" { + description = "name of the service to deploy" + type = string +} + +variable "vault_id" { + description = "1password vault id" + type = string +} diff --git a/crossplane/versions.tf b/crossplane/versions.tf new file mode 100644 index 00000000..338feac8 --- /dev/null +++ b/crossplane/versions.tf @@ -0,0 +1,25 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + kubectl = { + source = "alekc/kubectl" + version = ">= 2.0.0" + } + local = { + source = "hashicorp/local" + version = "2.5.2" + } + aws = { + source = "hashicorp/aws" + version = "4.64.0" + } + template = { + source = "hashicorp/template" + version = "2.2.0" + } + } + backend "s3" { + } +} + diff --git a/renovate.json b/renovate.json deleted file mode 100644 index 39a2b6e9..00000000 --- a/renovate.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "config:base" - ] -} From b424626823b8bd7d52e6ba5b3a619ac8edad2db9 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Thu, 31 Oct 2024 23:48:07 +0100 Subject: [PATCH 03/38] update vpcConfig --- crossplane/modify-claims.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh index 9bb4da48..bbc7f291 100755 --- a/crossplane/modify-claims.sh +++ b/crossplane/modify-claims.sh @@ -30,16 +30,15 @@ add_vpc_config() { local config if [[ "$env" == "staging" ]]; then - config='[{"securityGroupIds":["sg-03c24245575c1ebc0"],"subnetIds":["subnet-011cb6fe763310759","subnet-08deca209f9e46ebb","subnet-06e62ab1abfd70465"]}]' + config='{"vpcConfig":[{"securityGroupIds":["sg-03c24245575c1ebc0"],"subnetIds":["subnet-011cb6fe763310759","subnet-08deca209f9e46ebb","subnet-06e62ab1abfd70465"]}]}' elif [[ "$env" == "prod" ]]; then - config='[{"securityGroupIds":["sg-03c24245575c1ebc0"],"subnetIds":["subnet-011cb6fe763310759","subnet-08deca20d9f9e46ebb","subnet-06e62ab1abfd70465"]}]' + config='{"vpcConfig":[{"securityGroupIds":["sg-03c24245575c1ebc0"],"subnetIds":["subnet-011cb6fe763310759","subnet-08deca20d9f9e46ebb","subnet-06e62ab1abfd70465"]}]}' else echo "Error: Unsupported environment $env" exit 1 fi - # Set the vpcConfig directly as an array - yq eval ".spec.parameters.vpcConfig = $config" -i "$temp_yaml_file" + yq eval ".spec.parameters += $config" -i "$temp_yaml_file" } if [[ " ${CLAIM_TYPES_LAMBDA[@]} " =~ " ${kind} " ]]; then From ec4a1103520372730e2b76b99271af2db3ea6b6a Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Tue, 5 Nov 2024 01:28:28 +0100 Subject: [PATCH 04/38] update backend config --- .github/workflows/crossplane.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml index 9aa58665..70271306 100644 --- a/.github/workflows/crossplane.yaml +++ b/.github/workflows/crossplane.yaml @@ -61,7 +61,7 @@ jobs: terraform init \ -backend-config="region=eu-central-1" \ -backend-config="bucket=terraform-eks" \ - -backend-config="key=crossplane/${{ inputs.service_name }}" + -backend-config="key=crossplane/${{ inputs.environment }}/${{ inputs.service_name }}" - name: Terraform Validate @@ -77,7 +77,8 @@ jobs: terraform apply -auto-approve -no-color \ -var-file=${{ inputs.environment }}.tfvars \ -var="commit_hash=${{ github.sha }}" \ - -var="service_name=${{ inputs.service_name }}" + -var="service_name=${{ inputs.service_name }}" \ + -backend-config="key=crossplane/${{ inputs.environment }}/${{ inputs.service_name }}" env: TF_WORKSPACE: ${{ inputs.environment }} OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.STAGING_ONEPASSWORD_SERVICEACCOUNT_TOKEN }} From 59df2ca0a1efd86f7b93036f6a880427c2680016 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Tue, 5 Nov 2024 02:05:12 +0100 Subject: [PATCH 05/38] update backend config --- .github/workflows/crossplane.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml index 70271306..5a757ca9 100644 --- a/.github/workflows/crossplane.yaml +++ b/.github/workflows/crossplane.yaml @@ -77,8 +77,7 @@ jobs: terraform apply -auto-approve -no-color \ -var-file=${{ inputs.environment }}.tfvars \ -var="commit_hash=${{ github.sha }}" \ - -var="service_name=${{ inputs.service_name }}" \ - -backend-config="key=crossplane/${{ inputs.environment }}/${{ inputs.service_name }}" + -var="service_name=${{ inputs.service_name }}" env: TF_WORKSPACE: ${{ inputs.environment }} OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.STAGING_ONEPASSWORD_SERVICEACCOUNT_TOKEN }} From abc4a5b5aa3fb8bd0f66c8e408adbe3d12710753 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Tue, 26 Nov 2024 16:51:47 +0100 Subject: [PATCH 06/38] crossplane for prod --- .github/workflows/crossplane-release.yaml | 85 +++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/crossplane-release.yaml diff --git a/.github/workflows/crossplane-release.yaml b/.github/workflows/crossplane-release.yaml new file mode 100644 index 00000000..0075e3dd --- /dev/null +++ b/.github/workflows/crossplane-release.yaml @@ -0,0 +1,85 @@ +on: + workflow_call: + inputs: + environment: + required: true + type: string + service_name: + required: true + type: string + +permissions: + id-token: write + contents: read + pull-requests: write + statuses: write + +jobs: + crossplane-terraform: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - + name: Checkout shared-actions + uses: actions/checkout@v4 + with: + repository: vimeda/shared-actions + path: ./scripts + ref: feature/shared-crossplane + - + name: Checkout service + uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + path: ./scripts/${{ inputs.service_name }} + - + name: Install yq + uses: chrisdickinson/setup-yq@v1.0.1 + with: + yq-version: v4.25.3 + - + name: Install 1Password CLI + uses: 1password/install-cli-action@v1 + - + name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-${{ inputs.service_name }}-role + role-session-name: ga-${{ inputs.service_name }} + aws-region: eu-central-1 + - + name: Install Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ~1.9 + + - + name: Terraform Init + working-directory: ./scripts/crossplane + run: | + terraform init \ + -backend-config="region=eu-central-1" \ + -backend-config="bucket=terraform-eks" \ + -backend-config="key=crossplane/${{ inputs.environment }}/${{ inputs.service_name }}" + + - + name: Terraform Validate + working-directory: ./scripts/crossplane + run: | + terraform validate -no-color + - + name: Terraform + working-directory: ./scripts/crossplane + run: | + echo "${{ secrets.PROD_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml + terraform apply -auto-approve -no-color \ + -var-file=${{ inputs.environment }}.tfvars \ + -var="commit_hash=${{ github.sha }}" \ + -var="service_name=${{ inputs.service_name }}" + env: + TF_WORKSPACE: ${{ inputs.environment }} + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.PROD_ONEPASSWORD_SERVICEACCOUNT_TOKEN }} + + From f19fcdac71f91a199a291335ad0b093c2ed2a7b8 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Thu, 28 Nov 2024 13:21:45 +0100 Subject: [PATCH 07/38] update subnets --- crossplane/modify-claims.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh index bbc7f291..ed24fd72 100755 --- a/crossplane/modify-claims.sh +++ b/crossplane/modify-claims.sh @@ -32,7 +32,7 @@ add_vpc_config() { if [[ "$env" == "staging" ]]; then config='{"vpcConfig":[{"securityGroupIds":["sg-03c24245575c1ebc0"],"subnetIds":["subnet-011cb6fe763310759","subnet-08deca209f9e46ebb","subnet-06e62ab1abfd70465"]}]}' elif [[ "$env" == "prod" ]]; then - config='{"vpcConfig":[{"securityGroupIds":["sg-03c24245575c1ebc0"],"subnetIds":["subnet-011cb6fe763310759","subnet-08deca20d9f9e46ebb","subnet-06e62ab1abfd70465"]}]}' + config='{"vpcConfig":[{"securityGroupIds":["sg-03c24245575c1ebc0"],"subnetIds":["subnet-011cb6fe763310759","subnet-08deca209f9e46ebb","subnet-06e62ab1abfd70465"]}]}' else echo "Error: Unsupported environment $env" exit 1 From fc8d1e5a46cbf88e4938c28eba34fee9c3cbad93 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Fri, 29 Nov 2024 01:05:57 +0100 Subject: [PATCH 08/38] run_zip_lambda_workflow_step set to false --- .github/workflows/crossplane.yaml | 82 +++++++++++++++++++++++++++++++ crossplane/main.tf | 3 ++ crossplane/modify-claims.sh | 2 +- 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml index 5a757ca9..77af52f4 100644 --- a/.github/workflows/crossplane.yaml +++ b/.github/workflows/crossplane.yaml @@ -7,6 +7,10 @@ on: service_name: required: true type: string + run_zip_lambda_workflow_step: + type: boolean + default: false + permissions: id-token: write @@ -53,6 +57,84 @@ jobs: uses: hashicorp/setup-terraform@v3 with: terraform_version: ~1.9 + + + + + + + + + + + + + + - name: Build lambdas + if: ${{inputs.run_zip_lambda_workflow_step == true}} + run: cd ${{ github.workspace }} && make lambda + env: + GITHUB_TOKEN: ${{ secrets.GB_TOKEN_PRIVATE }} + GOPRIVATE: "github.com/vimeda/*" + + - name: Upload Build Artifacts + if: ${{inputs.run_zip_lambda_workflow_step == 'true'}} + uses: actions/upload-artifact@v4 + with: + name: srv-lambdas + path: ${{ github.workspace }}/dist/* + + - uses: actions/download-artifact@v4 + id: download + if: ${{inputs.run_zip_lambda_workflow_step == 'true'}} + with: + name: srv-lambdas + path: ${{ github.workspace }}/dist + + - name: Display structure of downloaded files + if: ${{env.run_zip_lambda_workflow_step == true}} + run: ls -R + working-directory: ${{ steps.download.outputs.download-path }} + env: + run_zip_lambda_workflow_step: ${{ inputs.run_zip_lambda_workflow_step }} + + - name: Push all functions to Bucket + if: ${{env.run_zip_lambda_workflow_step == true}} + run: | + cd ${{ github.workspace }}/dist && ls + for file in "./"/*lambda.zip + do + filename=$(basename "$file" .zip) + function_name=${filename%_lambda} + aws s3 cp "$file" "s3://${{inputs.env}}-lykon-lambdas/${{ github.event.repository.name}}/$function_name.zip" + done + env: + run_zip_lambda_workflow_step: ${{ inputs.run_zip_lambda_workflow_step }} + + + + + + + + + + + + + + + + + + + + + + + + + - name: Terraform Init diff --git a/crossplane/main.tf b/crossplane/main.tf index df1f6199..5bd52471 100644 --- a/crossplane/main.tf +++ b/crossplane/main.tf @@ -55,4 +55,7 @@ resource "kubectl_manifest" "apply" { depends_on = [data.kubectl_file_documents.claims] for_each = toset(local.manifests_array) yaml_body = each.value # Apply each manifest from the array + lifecycle { + create_before_destroy = true # recreate the resource each time + } } diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh index ed24fd72..d1da1f99 100755 --- a/crossplane/modify-claims.sh +++ b/crossplane/modify-claims.sh @@ -32,7 +32,7 @@ add_vpc_config() { if [[ "$env" == "staging" ]]; then config='{"vpcConfig":[{"securityGroupIds":["sg-03c24245575c1ebc0"],"subnetIds":["subnet-011cb6fe763310759","subnet-08deca209f9e46ebb","subnet-06e62ab1abfd70465"]}]}' elif [[ "$env" == "prod" ]]; then - config='{"vpcConfig":[{"securityGroupIds":["sg-03c24245575c1ebc0"],"subnetIds":["subnet-011cb6fe763310759","subnet-08deca209f9e46ebb","subnet-06e62ab1abfd70465"]}]}' + config='{"vpcConfig":[{"securityGroupIds":["sg-0c928b162190fd686"],"subnetIds":["subnet-0a03a55b3efec8bc5","subnet-074c6060a0d7ebe56","subnet-0ee3dbbbc8db2762e"]}]}' else echo "Error: Unsupported environment $env" exit 1 From 3b2a0c0b86e9f392810cd8dd58be7603069b13c3 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:10:35 +0100 Subject: [PATCH 09/38] run_zip_lambda_workflow_step set to false --- .github/workflows/crossplane.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml index 77af52f4..df473585 100644 --- a/.github/workflows/crossplane.yaml +++ b/.github/workflows/crossplane.yaml @@ -78,7 +78,7 @@ jobs: GOPRIVATE: "github.com/vimeda/*" - name: Upload Build Artifacts - if: ${{inputs.run_zip_lambda_workflow_step == 'true'}} + if: ${{inputs.run_zip_lambda_workflow_step == true}} uses: actions/upload-artifact@v4 with: name: srv-lambdas @@ -86,7 +86,7 @@ jobs: - uses: actions/download-artifact@v4 id: download - if: ${{inputs.run_zip_lambda_workflow_step == 'true'}} + if: ${{inputs.run_zip_lambda_workflow_step == true}} with: name: srv-lambdas path: ${{ github.workspace }}/dist From f2a6927ae97b02b8f466bb9c9570640c5441c5c7 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Mon, 9 Dec 2024 12:39:28 +0100 Subject: [PATCH 10/38] update PR role --- .github/workflows/pr-terraform.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-terraform.yaml b/.github/workflows/pr-terraform.yaml index 82f8af07..e6845e5b 100644 --- a/.github/workflows/pr-terraform.yaml +++ b/.github/workflows/pr-terraform.yaml @@ -39,9 +39,9 @@ jobs: id: aws uses: aws-actions/configure-aws-credentials@v2 with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ inputs.role_name }} - role-session-name: ${{ inputs.role_session_name }} - aws-region: ${{ inputs.aws_region }} + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-${{ inputs.service_name }}-role + role-session-name: ga-${{ inputs.service_name }} + aws-region: eu-central-1 - name: Download Staging Variables uses: actions/download-artifact@v4 From 16a7e1f0fba0f28e6af6ce83657b17946893932d Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:06:10 +0100 Subject: [PATCH 11/38] add support for lambda in release ci --- .github/workflows/crossplane-release.yaml | 45 +++++++++++++++++++++++ .github/workflows/crossplane.yaml | 39 ++------------------ 2 files changed, 48 insertions(+), 36 deletions(-) diff --git a/.github/workflows/crossplane-release.yaml b/.github/workflows/crossplane-release.yaml index 0075e3dd..f5035d72 100644 --- a/.github/workflows/crossplane-release.yaml +++ b/.github/workflows/crossplane-release.yaml @@ -54,6 +54,51 @@ jobs: with: terraform_version: ~1.9 + #build lambdas zip or lambda docker images + + - name: Build lambdas + if: ${{inputs.run_zip_lambda_workflow_step == true}} + run: cd ${{ github.workspace }} && make lambda + env: + GITHUB_TOKEN: ${{ secrets.GB_TOKEN_PRIVATE }} + GOPRIVATE: "github.com/vimeda/*" + + - name: Upload Build Artifacts + if: ${{inputs.run_zip_lambda_workflow_step == true}} + uses: actions/upload-artifact@v4 + with: + name: srv-lambdas + path: ${{ github.workspace }}/dist/* + + - uses: actions/download-artifact@v4 + id: download + if: ${{inputs.run_zip_lambda_workflow_step == true}} + with: + name: srv-lambdas + path: ${{ github.workspace }}/dist + + - name: Display structure of downloaded files + if: ${{env.run_zip_lambda_workflow_step == true}} + run: ls -R + working-directory: ${{ steps.download.outputs.download-path }} + env: + run_zip_lambda_workflow_step: ${{ inputs.run_zip_lambda_workflow_step }} + + - name: Push all functions to Bucket + if: ${{env.run_zip_lambda_workflow_step == true}} + run: | + cd ${{ github.workspace }}/dist && ls + for file in "./"/*lambda.zip + do + filename=$(basename "$file" .zip) + function_name=${filename%_lambda} + aws s3 cp "$file" "s3://${{inputs.env}}-lykon-lambdas/${{ github.event.repository.name}}/$function_name.zip" + done + env: + run_zip_lambda_workflow_step: ${{ inputs.run_zip_lambda_workflow_step }} + + #apply terraform + - name: Terraform Init working-directory: ./scripts/crossplane diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml index df473585..b6b20c16 100644 --- a/.github/workflows/crossplane.yaml +++ b/.github/workflows/crossplane.yaml @@ -58,17 +58,7 @@ jobs: with: terraform_version: ~1.9 - - - - - - - - - - - + #build lambdas zip or lambda docker images - name: Build lambdas if: ${{inputs.run_zip_lambda_workflow_step == true}} @@ -110,31 +100,8 @@ jobs: done env: run_zip_lambda_workflow_step: ${{ inputs.run_zip_lambda_workflow_step }} - - - - - - - - - - - - - - - - - - - - - - - - - + + #apply terraform - name: Terraform Init From fbaf2113a8a3ceaa412d7a58892b29067fd311b2 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:09:23 +0100 Subject: [PATCH 12/38] upgrade PR ci for crossplane --- .github/workflows/pr-crossplane.yaml | 162 ++++++++++++++------------- 1 file changed, 87 insertions(+), 75 deletions(-) diff --git a/.github/workflows/pr-crossplane.yaml b/.github/workflows/pr-crossplane.yaml index 70fe465d..d33ff3fe 100644 --- a/.github/workflows/pr-crossplane.yaml +++ b/.github/workflows/pr-crossplane.yaml @@ -1,28 +1,16 @@ on: workflow_call: inputs: - role_name: + environment: required: true type: string - role_session_name: + service_name: required: true type: string - aws_region: - required: false - type: string - default: eu-central-1 - working_directory: - required: true - type: string - env: - required: true - type: string - zip_lambda_workflow_step: - required: true + run_zip_lambda_workflow_step: type: boolean - commit_hash: - required: true - type: string + default: false + permissions: id-token: write @@ -31,93 +19,117 @@ permissions: statuses: write jobs: - terraform: - name: Terraform + crossplane-terraform: runs-on: ubuntu-latest - timeout-minutes: 15 - - defaults: - run: - working-directory: ${{ inputs.working_directory }} + timeout-minutes: 10 steps: - - name: Checkout code + - + name: Checkout shared-actions + uses: actions/checkout@v4 + with: + repository: vimeda/shared-actions + path: ./scripts + ref: feature/shared-crossplane + - + name: Checkout service uses: actions/checkout@v4 - - - name: Configure AWS Credentials - id: aws - uses: aws-actions/configure-aws-credentials@v2 with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ inputs.role_name }} - role-session-name: ${{ inputs.role_session_name }} - aws-region: ${{ inputs.aws_region }} + ref: ${{ github.ref }} + path: ./scripts/${{ inputs.service_name }} + - + name: Install yq + uses: chrisdickinson/setup-yq@v1.0.1 + with: + yq-version: v4.25.3 + - + name: Install 1Password CLI + uses: 1password/install-cli-action@v1 + - + name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-${{ inputs.service_name }}-role + role-session-name: ga-${{ inputs.service_name }} + aws-region: eu-central-1 + - + name: Install Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: ~1.9 + + #build lambdas zip or lambda docker images - name: Build lambdas + if: ${{inputs.run_zip_lambda_workflow_step == true}} run: cd ${{ github.workspace }} && make lambda - if: ${{env.zip_lambda_workflow_step==true}} env: GITHUB_TOKEN: ${{ secrets.GB_TOKEN_PRIVATE }} GOPRIVATE: "github.com/vimeda/*" - zip_lambda_workflow_step: ${{ inputs.zip_lambda_workflow_step }} - name: Upload Build Artifacts + if: ${{inputs.run_zip_lambda_workflow_step == true}} uses: actions/upload-artifact@v4 - if: ${{env.zip_lambda_workflow_step==true}} with: name: srv-lambdas path: ${{ github.workspace }}/dist/* - zip_lambda_workflow_step: ${{ inputs.zip_lambda_workflow_step }} - uses: actions/download-artifact@v4 id: download - if: ${{env.zip_lambda_workflow_step==true}} + if: ${{inputs.run_zip_lambda_workflow_step == true}} with: name: srv-lambdas - path: ${{ github.workspace }}/dist - zip_lambda_workflow_step: ${{ inputs.zip_lambda_workflow_step }} + path: ${{ github.workspace }}/dist - - uses: hashicorp/setup-terraform@v1 - with: - terraform_version: ~1.4 + - name: Display structure of downloaded files + if: ${{env.run_zip_lambda_workflow_step == true}} + run: ls -R + working-directory: ${{ steps.download.outputs.download-path }} + env: + run_zip_lambda_workflow_step: ${{ inputs.run_zip_lambda_workflow_step }} - - uses: actions/download-artifact@v4 - id: downloadscripts - with: - name: scripts - path: /tmp/scripts + #apply terraform - - name: Install 1Password Cli, patch claim + - + name: Terraform Init + working-directory: ./scripts/crossplane run: | - curl https://cache.agilebits.com/dist/1P/op2/pkg/v2.18.0/op_linux_amd64_v2.23.0.zip > op.zip - sudo unzip op.zip -d /usr/local/bin && rm op.zip - python -m pip install "ruamel.yaml<0.18.0" - - python /tmp/scripts/.github/workflows/scripts/patch.py - - for file in *claims.yaml; do - if [ -f "$file" ]; then - cat $file - fi - done - - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.STAGING_ONEPASSWORD_SERVICEACCOUNT_TOKEN }} - VAULT_ID: "errsir3kqd4gdjgaxliofyskey" - ENV: ${{ inputs.env }} - - - name: patch claim image uri with commit hash - id: patch_image_uri_with_commit_hash + terraform init \ + -backend-config="region=eu-central-1" \ + -backend-config="bucket=terraform-eks" \ + -backend-config="key=crossplane/${{ inputs.environment }}/${{ inputs.service_name }}" + + - + name: Terraform Validate + working-directory: ./scripts/crossplane + run: | + terraform validate -no-color + - + name: Terraform Staging + working-directory: ./scripts/crossplane run: | - find ./ -type f -name "*.yaml" -exec sed -i -e 's/COMMIT_HASH/${{ env.commit_hash }}/g' -e 's/ECR_BASE_URL/279707217826.dkr.ecr.eu-central-1.amazonaws.com\/lykon/g' {} \; + echo "${{ secrets.STAGING_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml + terraform plan -no-color \ + -var-file=${{ inputs.environment }}.tfvars \ + -var="commit_hash=${{ github.sha }}" \ + -var="service_name=${{ inputs.service_name }}" env: - TF_WORKSPACE: ${{ inputs.env }} - commit_hash: ${{ inputs.commit_hash }} + TF_WORKSPACE: ${{ inputs.environment }} + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.STAGING_ONEPASSWORD_SERVICEACCOUNT_TOKEN }} - - name: Terraform Plan - id: plan + - + name: Terraform Prod + working-directory: ./scripts/crossplane run: | - echo "${{ secrets.STAGING_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + echo "${{ secrets.PROD_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml - cd ${{ github.workspace }}/configs/crossplane && terraform init && terraform -chdir=${{ github.workspace }}/configs/crossplane plan + terraform plan -no-color \ + -var-file=${{ inputs.environment }}.tfvars \ + -var="commit_hash=${{ github.sha }}" \ + -var="service_name=${{ inputs.service_name }}" env: - TF_WORKSPACE: ${{ inputs.env }} + TF_WORKSPACE: ${{ inputs.environment }} + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.PROD_ONEPASSWORD_SERVICEACCOUNT_TOKEN }} + + From 159347a27739a5542bffdc97f2eaf59bbd63826e Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:09:59 +0100 Subject: [PATCH 13/38] upgrade PR ci for crossplane --- .github/workflows/pr-crossplane.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-crossplane.yaml b/.github/workflows/pr-crossplane.yaml index d33ff3fe..32d76f01 100644 --- a/.github/workflows/pr-crossplane.yaml +++ b/.github/workflows/pr-crossplane.yaml @@ -88,7 +88,7 @@ jobs: env: run_zip_lambda_workflow_step: ${{ inputs.run_zip_lambda_workflow_step }} - #apply terraform + # terraform plan - name: Terraform Init From c64def0e20b65522485285049530ded3cb5f5195 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:31:38 +0100 Subject: [PATCH 14/38] add aws account id --- .github/workflows/pr-crossplane.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-crossplane.yaml b/.github/workflows/pr-crossplane.yaml index 32d76f01..886f0f31 100644 --- a/.github/workflows/pr-crossplane.yaml +++ b/.github/workflows/pr-crossplane.yaml @@ -49,7 +49,7 @@ jobs: name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-${{ inputs.service_name }}-role + role-to-assume: arn:aws:iam::279707217826:role/github-actions-${{ inputs.service_name }}-role role-session-name: ga-${{ inputs.service_name }} aws-region: eu-central-1 - From 4f55b10d34768d297de63fd51d184b364fdf0189 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:35:26 +0100 Subject: [PATCH 15/38] add aws account id From db1dad26b7e7ab0655e47dbc3a95eff809375edc Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Mon, 13 Jan 2025 17:57:32 +0100 Subject: [PATCH 16/38] handle LykonEventSourceMapping --- .github/workflows/pr-crossplane.yaml | 135 --------------------------- crossplane/modify-claims.sh | 21 +++++ 2 files changed, 21 insertions(+), 135 deletions(-) delete mode 100644 .github/workflows/pr-crossplane.yaml diff --git a/.github/workflows/pr-crossplane.yaml b/.github/workflows/pr-crossplane.yaml deleted file mode 100644 index 886f0f31..00000000 --- a/.github/workflows/pr-crossplane.yaml +++ /dev/null @@ -1,135 +0,0 @@ -on: - workflow_call: - inputs: - environment: - required: true - type: string - service_name: - required: true - type: string - run_zip_lambda_workflow_step: - type: boolean - default: false - - -permissions: - id-token: write - contents: read - pull-requests: write - statuses: write - -jobs: - crossplane-terraform: - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - - name: Checkout shared-actions - uses: actions/checkout@v4 - with: - repository: vimeda/shared-actions - path: ./scripts - ref: feature/shared-crossplane - - - name: Checkout service - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - path: ./scripts/${{ inputs.service_name }} - - - name: Install yq - uses: chrisdickinson/setup-yq@v1.0.1 - with: - yq-version: v4.25.3 - - - name: Install 1Password CLI - uses: 1password/install-cli-action@v1 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::279707217826:role/github-actions-${{ inputs.service_name }}-role - role-session-name: ga-${{ inputs.service_name }} - aws-region: eu-central-1 - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: ~1.9 - - #build lambdas zip or lambda docker images - - - name: Build lambdas - if: ${{inputs.run_zip_lambda_workflow_step == true}} - run: cd ${{ github.workspace }} && make lambda - env: - GITHUB_TOKEN: ${{ secrets.GB_TOKEN_PRIVATE }} - GOPRIVATE: "github.com/vimeda/*" - - - name: Upload Build Artifacts - if: ${{inputs.run_zip_lambda_workflow_step == true}} - uses: actions/upload-artifact@v4 - with: - name: srv-lambdas - path: ${{ github.workspace }}/dist/* - - - uses: actions/download-artifact@v4 - id: download - if: ${{inputs.run_zip_lambda_workflow_step == true}} - with: - name: srv-lambdas - path: ${{ github.workspace }}/dist - - - name: Display structure of downloaded files - if: ${{env.run_zip_lambda_workflow_step == true}} - run: ls -R - working-directory: ${{ steps.download.outputs.download-path }} - env: - run_zip_lambda_workflow_step: ${{ inputs.run_zip_lambda_workflow_step }} - - # terraform plan - - - - name: Terraform Init - working-directory: ./scripts/crossplane - run: | - terraform init \ - -backend-config="region=eu-central-1" \ - -backend-config="bucket=terraform-eks" \ - -backend-config="key=crossplane/${{ inputs.environment }}/${{ inputs.service_name }}" - - - - name: Terraform Validate - working-directory: ./scripts/crossplane - run: | - terraform validate -no-color - - - name: Terraform Staging - working-directory: ./scripts/crossplane - run: | - echo "${{ secrets.STAGING_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml - export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml - terraform plan -no-color \ - -var-file=${{ inputs.environment }}.tfvars \ - -var="commit_hash=${{ github.sha }}" \ - -var="service_name=${{ inputs.service_name }}" - env: - TF_WORKSPACE: ${{ inputs.environment }} - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.STAGING_ONEPASSWORD_SERVICEACCOUNT_TOKEN }} - - - - name: Terraform Prod - working-directory: ./scripts/crossplane - run: | - echo "${{ secrets.PROD_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml - export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml - terraform plan -no-color \ - -var-file=${{ inputs.environment }}.tfvars \ - -var="commit_hash=${{ github.sha }}" \ - -var="service_name=${{ inputs.service_name }}" - env: - TF_WORKSPACE: ${{ inputs.environment }} - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.PROD_ONEPASSWORD_SERVICEACCOUNT_TOKEN }} - - diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh index d1da1f99..8e58544b 100755 --- a/crossplane/modify-claims.sh +++ b/crossplane/modify-claims.sh @@ -75,6 +75,27 @@ elif [[ " ${CLAIM_TYPES_GOAPP[@]} " =~ " ${kind} " ]]; then yq eval ".spec.parameters.vault_id = \"$vault_id\"" -i "$temp_yaml_file" fi +if [[ "$kind" == "LykonEventSourceMapping" ]]; then + # Get the current stream value + stream=$(yq eval '.spec.parameters.stream' "$temp_yaml_file") + + # Modify the stream based on whether it's S3 or Kinesis + if [[ "$stream" == s3:* ]]; then + # Extract bucket name from the stream and form the S3 ARN + bucket_name=$(echo "$stream" | cut -d':' -f2) + new_stream="arn:aws:s3:::$bucket_name" + yq eval ".spec.parameters.stream = \"$new_stream\"" -i "$temp_yaml_file" + elif [[ "$stream" == kinesis:* ]]; then + # Extract stream name from the stream and form the Kinesis ARN + stream_name=$(echo "$stream" | cut -d':' -f2) + new_stream="arn:aws:kinesis:279707217826:stream/$stream_name" + yq eval ".spec.parameters.stream = \"$new_stream\"" -i "$temp_yaml_file" + else + echo "Error: Unsupported stream format: $stream" + exit 1 + fi +fi + # Convert the final YAML to JSON for Terraform manifest=$(yq eval -o=json "$temp_yaml_file") jq -n --arg manifest "$manifest" '{ manifest: $manifest }' From a591ccd740aa47422d4e8c25df6a1a96d380fc44 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Mon, 13 Jan 2025 18:01:50 +0100 Subject: [PATCH 17/38] handles all other as kinesis --- crossplane/modify-claims.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh index 8e58544b..8b9f0bf7 100755 --- a/crossplane/modify-claims.sh +++ b/crossplane/modify-claims.sh @@ -79,7 +79,7 @@ if [[ "$kind" == "LykonEventSourceMapping" ]]; then # Get the current stream value stream=$(yq eval '.spec.parameters.stream' "$temp_yaml_file") - # Modify the stream based on whether it's S3 or Kinesis + # Modify the stream based on whether it's S3, Kinesis, or a simple Kinesis stream name if [[ "$stream" == s3:* ]]; then # Extract bucket name from the stream and form the S3 ARN bucket_name=$(echo "$stream" | cut -d':' -f2) @@ -91,8 +91,9 @@ if [[ "$kind" == "LykonEventSourceMapping" ]]; then new_stream="arn:aws:kinesis:279707217826:stream/$stream_name" yq eval ".spec.parameters.stream = \"$new_stream\"" -i "$temp_yaml_file" else - echo "Error: Unsupported stream format: $stream" - exit 1 + # Treat any other value as a simple Kinesis stream name and form the Kinesis ARN + new_stream="arn:aws:kinesis:279707217826:stream/$stream" + yq eval ".spec.parameters.stream = \"$new_stream\"" -i "$temp_yaml_file" fi fi From 4231ae28b7de97e94c3f16b620ea2d530a817a5f Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Mon, 13 Jan 2025 18:24:10 +0100 Subject: [PATCH 18/38] update kind to XLykonLambdaDockerImage --- crossplane/modify-claims.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh index 8b9f0bf7..87833ed8 100755 --- a/crossplane/modify-claims.sh +++ b/crossplane/modify-claims.sh @@ -75,7 +75,7 @@ elif [[ " ${CLAIM_TYPES_GOAPP[@]} " =~ " ${kind} " ]]; then yq eval ".spec.parameters.vault_id = \"$vault_id\"" -i "$temp_yaml_file" fi -if [[ "$kind" == "LykonEventSourceMapping" ]]; then +if [[ "$kind" == "XLykonLambdaDockerImage" ]]; then # Get the current stream value stream=$(yq eval '.spec.parameters.stream' "$temp_yaml_file") From b01c694ca8bf7b3f04dce6bfadd27bac908c6581 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Mon, 13 Jan 2025 18:44:27 +0100 Subject: [PATCH 19/38] update kind to XLykonLambdaDockerImage --- crossplane/modify-claims.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh index 87833ed8..84ccb1e8 100755 --- a/crossplane/modify-claims.sh +++ b/crossplane/modify-claims.sh @@ -88,11 +88,11 @@ if [[ "$kind" == "XLykonLambdaDockerImage" ]]; then elif [[ "$stream" == kinesis:* ]]; then # Extract stream name from the stream and form the Kinesis ARN stream_name=$(echo "$stream" | cut -d':' -f2) - new_stream="arn:aws:kinesis:279707217826:stream/$stream_name" + new_stream="arn:aws:kinesis:eu-central-1:279707217826:stream/$stream_name" yq eval ".spec.parameters.stream = \"$new_stream\"" -i "$temp_yaml_file" else # Treat any other value as a simple Kinesis stream name and form the Kinesis ARN - new_stream="arn:aws:kinesis:279707217826:stream/$stream" + new_stream="arn:aws:kinesis:eu-central-1:279707217826:stream/$stream" yq eval ".spec.parameters.stream = \"$new_stream\"" -i "$temp_yaml_file" fi fi From 0b77b38b384208d27438ef812b6f084ff9c24d16 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Thu, 23 Jan 2025 13:31:39 +0100 Subject: [PATCH 20/38] updated staging kubeconfig --- .github/workflows/crossplane.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml index b6b20c16..13fe9d4e 100644 --- a/.github/workflows/crossplane.yaml +++ b/.github/workflows/crossplane.yaml @@ -121,7 +121,7 @@ jobs: name: Terraform working-directory: ./scripts/crossplane run: | - echo "${{ secrets.STAGING_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + echo "${{ secrets.STG_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml terraform apply -auto-approve -no-color \ -var-file=${{ inputs.environment }}.tfvars \ From 9636d0c78efee9a25762951cd12248f64852e104 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Thu, 23 Jan 2025 13:33:27 +0100 Subject: [PATCH 21/38] updated staging kubeconfig --- .github/workflows/crossplane.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml index 13fe9d4e..b6b20c16 100644 --- a/.github/workflows/crossplane.yaml +++ b/.github/workflows/crossplane.yaml @@ -121,7 +121,7 @@ jobs: name: Terraform working-directory: ./scripts/crossplane run: | - echo "${{ secrets.STG_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + echo "${{ secrets.STAGING_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml terraform apply -auto-approve -no-color \ -var-file=${{ inputs.environment }}.tfvars \ From ca7c7211e939fa54e124daecbd569efe0d547e07 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Thu, 23 Jan 2025 13:52:59 +0100 Subject: [PATCH 22/38] updated staging kubeconfig --- .github/workflows/crossplane.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml index b6b20c16..13fe9d4e 100644 --- a/.github/workflows/crossplane.yaml +++ b/.github/workflows/crossplane.yaml @@ -121,7 +121,7 @@ jobs: name: Terraform working-directory: ./scripts/crossplane run: | - echo "${{ secrets.STAGING_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + echo "${{ secrets.STG_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml terraform apply -auto-approve -no-color \ -var-file=${{ inputs.environment }}.tfvars \ From 7e8b51917809465011a8167a7e53683558376568 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Thu, 23 Jan 2025 14:17:27 +0100 Subject: [PATCH 23/38] updated staging kubeconfig --- .github/workflows/crossplane.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml index 13fe9d4e..f990f657 100644 --- a/.github/workflows/crossplane.yaml +++ b/.github/workflows/crossplane.yaml @@ -121,7 +121,7 @@ jobs: name: Terraform working-directory: ./scripts/crossplane run: | - echo "${{ secrets.STG_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + echo "${{ secrets.STG23_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml terraform apply -auto-approve -no-color \ -var-file=${{ inputs.environment }}.tfvars \ From 3583b1d95683866a5322f1117725cfc5d9ab4905 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Thu, 23 Jan 2025 14:23:20 +0100 Subject: [PATCH 24/38] updated staging kubeconfig --- .github/workflows/crossplane.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml index f990f657..13fe9d4e 100644 --- a/.github/workflows/crossplane.yaml +++ b/.github/workflows/crossplane.yaml @@ -121,7 +121,7 @@ jobs: name: Terraform working-directory: ./scripts/crossplane run: | - echo "${{ secrets.STG23_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + echo "${{ secrets.STG_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml terraform apply -auto-approve -no-color \ -var-file=${{ inputs.environment }}.tfvars \ From b1e7ba6be297c1807277feeb1df333f85844c650 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Thu, 23 Jan 2025 15:19:51 +0100 Subject: [PATCH 25/38] updated staging kubeconfig --- .github/workflows/main-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main-deploy.yaml b/.github/workflows/main-deploy.yaml index bccb7422..16429eca 100644 --- a/.github/workflows/main-deploy.yaml +++ b/.github/workflows/main-deploy.yaml @@ -78,4 +78,4 @@ jobs: release: ${{ inputs.release_name }} namespace: ${{ inputs.namespace }} env: - KUBECONFIG_FILE: ${{ secrets.STAGING_KUBECONFIG }} + KUBECONFIG_FILE: ${{ secrets.STG_KUBECONFIG }} From 467689003b1e5e9c4c5c689bc8ca0721a4dfb541 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Thu, 23 Jan 2025 15:56:46 +0100 Subject: [PATCH 26/38] updated staging kubeconfig --- .github/workflows/crossplane.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml index 13fe9d4e..40530dc0 100644 --- a/.github/workflows/crossplane.yaml +++ b/.github/workflows/crossplane.yaml @@ -49,8 +49,8 @@ jobs: name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-${{ inputs.service_name }}-role - role-session-name: ga-${{ inputs.service_name }} + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-kubernetes-role + role-session-name: ga-kubernetes aws-region: eu-central-1 - name: Install Terraform From 41fc5e8bfecbf52958698513b8df6ce8df9a90c0 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Mon, 3 Feb 2025 18:41:09 +0100 Subject: [PATCH 27/38] update XEventSourceMapping stream --- crossplane/modify-claims.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh index 84ccb1e8..89fe4a09 100755 --- a/crossplane/modify-claims.sh +++ b/crossplane/modify-claims.sh @@ -75,7 +75,7 @@ elif [[ " ${CLAIM_TYPES_GOAPP[@]} " =~ " ${kind} " ]]; then yq eval ".spec.parameters.vault_id = \"$vault_id\"" -i "$temp_yaml_file" fi -if [[ "$kind" == "XLykonLambdaDockerImage" ]]; then +if [[ "$kind" == "XLykonLambdaDockerImage" || "$kind" == "XEventSourceMapping" ]]; then # Get the current stream value stream=$(yq eval '.spec.parameters.stream' "$temp_yaml_file") From c53622916e225e78b458736aaeae1e4409e3ea2e Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Tue, 4 Feb 2025 14:54:30 +0100 Subject: [PATCH 28/38] add XLykonS3EventSourceMapping --- crossplane/modify-claims.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh index 89fe4a09..885ea638 100755 --- a/crossplane/modify-claims.sh +++ b/crossplane/modify-claims.sh @@ -75,7 +75,7 @@ elif [[ " ${CLAIM_TYPES_GOAPP[@]} " =~ " ${kind} " ]]; then yq eval ".spec.parameters.vault_id = \"$vault_id\"" -i "$temp_yaml_file" fi -if [[ "$kind" == "XLykonLambdaDockerImage" || "$kind" == "XEventSourceMapping" ]]; then +if [[ "$kind" == "XLykonLambdaDockerImage" || "$kind" == "XEventSourceMapping"|| "$kind" == "XLykonS3EventSourceMapping" ]]; then # Get the current stream value stream=$(yq eval '.spec.parameters.stream' "$temp_yaml_file") From c395df35681e06693044f443e7ed78efb720d07f Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Tue, 4 Feb 2025 14:55:04 +0100 Subject: [PATCH 29/38] remove XLykonS3EventSourceMapping --- crossplane/modify-claims.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh index 885ea638..89fe4a09 100755 --- a/crossplane/modify-claims.sh +++ b/crossplane/modify-claims.sh @@ -75,7 +75,7 @@ elif [[ " ${CLAIM_TYPES_GOAPP[@]} " =~ " ${kind} " ]]; then yq eval ".spec.parameters.vault_id = \"$vault_id\"" -i "$temp_yaml_file" fi -if [[ "$kind" == "XLykonLambdaDockerImage" || "$kind" == "XEventSourceMapping"|| "$kind" == "XLykonS3EventSourceMapping" ]]; then +if [[ "$kind" == "XLykonLambdaDockerImage" || "$kind" == "XEventSourceMapping" ]]; then # Get the current stream value stream=$(yq eval '.spec.parameters.stream' "$temp_yaml_file") From 40667802534b70c6bd1581173b1c1575b19d9cb4 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Wed, 5 Feb 2025 17:33:06 +0100 Subject: [PATCH 30/38] fix modify-claims.sh --- crossplane/modify-claims.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh index 89fe4a09..0368a4a7 100755 --- a/crossplane/modify-claims.sh +++ b/crossplane/modify-claims.sh @@ -90,11 +90,6 @@ if [[ "$kind" == "XLykonLambdaDockerImage" || "$kind" == "XEventSourceMapping" ] stream_name=$(echo "$stream" | cut -d':' -f2) new_stream="arn:aws:kinesis:eu-central-1:279707217826:stream/$stream_name" yq eval ".spec.parameters.stream = \"$new_stream\"" -i "$temp_yaml_file" - else - # Treat any other value as a simple Kinesis stream name and form the Kinesis ARN - new_stream="arn:aws:kinesis:eu-central-1:279707217826:stream/$stream" - yq eval ".spec.parameters.stream = \"$new_stream\"" -i "$temp_yaml_file" - fi fi # Convert the final YAML to JSON for Terraform From dd5ba49efaeac379408c3721f2543b47817abdbb Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Wed, 5 Feb 2025 17:44:49 +0100 Subject: [PATCH 31/38] fix modify-claims.sh --- crossplane/modify-claims.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/crossplane/modify-claims.sh b/crossplane/modify-claims.sh index 0368a4a7..1a040a4d 100755 --- a/crossplane/modify-claims.sh +++ b/crossplane/modify-claims.sh @@ -90,6 +90,7 @@ if [[ "$kind" == "XLykonLambdaDockerImage" || "$kind" == "XEventSourceMapping" ] stream_name=$(echo "$stream" | cut -d':' -f2) new_stream="arn:aws:kinesis:eu-central-1:279707217826:stream/$stream_name" yq eval ".spec.parameters.stream = \"$new_stream\"" -i "$temp_yaml_file" + fi fi # Convert the final YAML to JSON for Terraform From dc8b911d172b331b73fc67a29bbdca477439b503 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Sun, 6 Apr 2025 20:13:44 +0100 Subject: [PATCH 32/38] prd kubeconfig --- .github/workflows/crossplane-release.yaml | 24 +++++++++++++++--- .github/workflows/kubeconfig | 30 +++++++++++++++++++++++ .github/workflows/release-crossplane.yaml | 2 +- .github/workflows/release-deploy.yaml | 2 +- .github/workflows/test-crossplane.yaml | 2 +- crossplane/prod.tfvars | 2 +- crossplane/variables.tf | 1 + crossplane/versions.tf | 5 ++++ 8 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/kubeconfig diff --git a/.github/workflows/crossplane-release.yaml b/.github/workflows/crossplane-release.yaml index f5035d72..d3ad362f 100644 --- a/.github/workflows/crossplane-release.yaml +++ b/.github/workflows/crossplane-release.yaml @@ -45,7 +45,7 @@ jobs: name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-${{ inputs.service_name }}-role + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-kubernetes-role role-session-name: ga-${{ inputs.service_name }} aws-region: eu-central-1 - @@ -114,10 +114,28 @@ jobs: run: | terraform validate -no-color - - name: Terraform + name: Verify Kubeconfig working-directory: ./scripts/crossplane run: | - echo "${{ secrets.PROD_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + if [ -z "$KUBECONFIG" ]; then + echo "Error: KUBECONFIG environment variable is not set" + exit 1 + fi + + # Test if we can connect to the cluster using the kubeconfig directly + if ! echo "$KUBECONFIG" | base64 -d | kubectl --kubeconfig=/dev/stdin cluster-info; then + echo "Error: Unable to connect to Kubernetes cluster" + exit 1 + fi + + echo "Kubeconfig verification successful" + env: + KUBECONFIG: ${{ secrets.PRD_KUBECONFIG }} + + - name: Terraform + working-directory: ./scripts/crossplane + run: | + echo "${{ secrets.PRD_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml terraform apply -auto-approve -no-color \ -var-file=${{ inputs.environment }}.tfvars \ diff --git a/.github/workflows/kubeconfig b/.github/workflows/kubeconfig new file mode 100644 index 00000000..9ecd6518 --- /dev/null +++ b/.github/workflows/kubeconfig @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Config +preferences: {} +current-context: arn:aws:eks:eu-central-1:279707217826:cluster/prd-eks-v2 + +clusters: +- name: arn:aws:eks:eu-central-1:279707217826:cluster/prd-eks-v2 + cluster: + server: https://6FAE7EA31F6ABDC83D35085CD36856A9.gr7.eu-central-1.eks.amazonaws.com + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJZXBDbzlyUGFBN1V3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TlRBME1ETXhOekk1TlRGYUZ3MHpOVEEwTURFeE56TTBOVEZhTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUURsYXRxcENFaHhjNHZuR1Q2U3htbzcwY3pFZ0M2Nm5TR2NwaDdNeTl4Njg4UTZTSWV6MFFyTnNPM3IKdExjaGRsaWtpREdZU1RwK2dmUlVNWDN5UjdYVTVyTHVEaktWWkh1N21XTTR4RFEwenhkNlVKcGd5NFRzK0kvbwpmc0NESnZoOUdtRjk4Z2kxdTdvTnorU0d3OXdFd3U1MzJsbEVtUXRTclpSemZuNHFYOUk1b1hRSXB0dkd3ekxXCk1ON0RmSjVTTGZIWlloblZPTmJUeCtQOWN4QW0yMXhVdGRYUlFIaXRJbklxZnFsTlhsYWlLbWlwQUNNRlJtazQKeTFETElVM1lWZDk0eGtJcDlVTFdMTnRjZ2NtWEdvZUMyMGhDUXJnRkJWaC9LRlNlQmNkOWQ2eVdySjFhVCtKUgovWEJTa1BvMWNBMDBjYy9jajNjWjVhZ3ozSXEvQWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJSeUF5aGd2Rm1LY2plTFcvNUNia0M0R2hYUFVEQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ0VQNUVzc0pnMAo5VW03TkZId0FRN0JWdCs0VWJkR3VQZEdaU0JDVWFwS1pCN05tUlFKdGtac05oRHJkQngydlpGZjVDWm1iUzB1CkhrRVJpSjc3K2hweWtuSEVKZmdyMjV0NWZ1cFBUdXRWU3V6ZTBicEJFMmZheEZpQy9kQlQ5bjBlK2lRVnd0SUkKemxGbVZBQUVRU2VSQjNqdFhFTk1CSUdOSXhoSUw4ZGRPVnFLQmdRWm9tZGlvdzhRRVRYV29iSzZLM3ZUVmRKcwozYmVUZ204NTZYdnRuSGtWM0JmTTFRK1dMTm1DYlYzeS9SODBHd3drQXJaK01VN2J1Q1hOUkhqem9qTkx0UEdTCmk3cjdZVmlUenlvTHFubVBub3N6VFZzTVh4YWxLelNnSGlQOGxFNUhWM2xQYm9TeUFnMm13ZEVVRFQ2ZkFzb0cKK2ROY1BBUnNwVUpvCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + +contexts: +- name: arn:aws:eks:eu-central-1:279707217826:cluster/prd-eks-v2 + context: + cluster: arn:aws:eks:eu-central-1:279707217826:cluster/prd-eks-v2 + user: arn:aws:eks:eu-central-1:279707217826:cluster/prd-eks-v2 + +users: +- name: arn:aws:eks:eu-central-1:279707217826:cluster/prd-eks-v2 + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + command: aws + args: + - --region + - eu-central-1 + - eks + - get-token + - --cluster-name + - prd-eks-v2 \ No newline at end of file diff --git a/.github/workflows/release-crossplane.yaml b/.github/workflows/release-crossplane.yaml index 32099230..de74b82c 100644 --- a/.github/workflows/release-crossplane.yaml +++ b/.github/workflows/release-crossplane.yaml @@ -120,7 +120,7 @@ jobs: - name: Terraform apply id: apply run: | - echo "${{ secrets.PROD_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + echo "${{ secrets.PRD_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml cd ${{ github.workspace }}/configs/crossplane && terraform init && terraform apply -auto-approve env: diff --git a/.github/workflows/release-deploy.yaml b/.github/workflows/release-deploy.yaml index 3f47e8bf..a301dc54 100644 --- a/.github/workflows/release-deploy.yaml +++ b/.github/workflows/release-deploy.yaml @@ -78,4 +78,4 @@ jobs: release: ${{ inputs.release_name }} namespace: ${{ inputs.namespace }} env: - KUBECONFIG_FILE: ${{ secrets.PROD_KUBECONFIG }} + KUBECONFIG_FILE: ${{ secrets.PRD_KUBECONFIG }} diff --git a/.github/workflows/test-crossplane.yaml b/.github/workflows/test-crossplane.yaml index f98dd4e4..c8a6b0ba 100644 --- a/.github/workflows/test-crossplane.yaml +++ b/.github/workflows/test-crossplane.yaml @@ -49,7 +49,7 @@ jobs: - name: Terraform Apply - Prod Crossplane id: apply-crossplane run: | - echo "${{ secrets.PROD_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + echo "${{ secrets.PRD_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml cd ${{ github.workspace }}/configs/crossplane ls diff --git a/crossplane/prod.tfvars b/crossplane/prod.tfvars index 2d38f8e7..674312de 100644 --- a/crossplane/prod.tfvars +++ b/crossplane/prod.tfvars @@ -1,2 +1,2 @@ -cluster_name = "prod-eks-v2" +cluster_name = "prd-eks-v2" vault_id = "37y43e5v2qd3iptgt7wgyk34ga" diff --git a/crossplane/variables.tf b/crossplane/variables.tf index 0b1db451..fce241c0 100644 --- a/crossplane/variables.tf +++ b/crossplane/variables.tf @@ -17,3 +17,4 @@ variable "vault_id" { description = "1password vault id" type = string } + diff --git a/crossplane/versions.tf b/crossplane/versions.tf index 338feac8..251c6139 100644 --- a/crossplane/versions.tf +++ b/crossplane/versions.tf @@ -23,3 +23,8 @@ terraform { } } +provider "kubectl" { + apply_retry_count = 5 + load_config_file = false +} + From 43409a1f3a730e6dab54342ce0eba261be88365e Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Mon, 7 Apr 2025 12:06:35 +0100 Subject: [PATCH 33/38] fix --- .github/workflows/crossplane-release.yaml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/crossplane-release.yaml b/.github/workflows/crossplane-release.yaml index d3ad362f..6c2b66e7 100644 --- a/.github/workflows/crossplane-release.yaml +++ b/.github/workflows/crossplane-release.yaml @@ -135,14 +135,26 @@ jobs: - name: Terraform working-directory: ./scripts/crossplane run: | - echo "${{ secrets.PRD_KUBECONFIG }}" > ${{ github.workspace }}/kubeconfig.yaml + # Create and decode kubeconfig + echo "${{ secrets.PRD_KUBECONFIG }}" | base64 -d > ${{ github.workspace }}/kubeconfig.yaml + chmod 600 ${{ github.workspace }}/kubeconfig.yaml + + # Set KUBECONFIG environment variable export KUBECONFIG=${{ github.workspace }}/kubeconfig.yaml + + # Verify kubectl can connect to the cluster + kubectl cluster-info + + # Run terraform apply terraform apply -auto-approve -no-color \ - -var-file=${{ inputs.environment }}.tfvars \ - -var="commit_hash=${{ github.sha }}" \ - -var="service_name=${{ inputs.service_name }}" + -var-file=${{ inputs.environment }}.tfvars \ + -var="commit_hash=${{ github.sha }}" \ + -var="service_name=${{ inputs.service_name }}" env: TF_WORKSPACE: ${{ inputs.environment }} OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.PROD_ONEPASSWORD_SERVICEACCOUNT_TOKEN }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: eu-central-1 From fd58327fb1330a4a0692222f38c6d2a2b0701af1 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Mon, 7 Apr 2025 12:12:29 +0100 Subject: [PATCH 34/38] fix --- .github/workflows/crossplane-release.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/crossplane-release.yaml b/.github/workflows/crossplane-release.yaml index 6c2b66e7..401f30d9 100644 --- a/.github/workflows/crossplane-release.yaml +++ b/.github/workflows/crossplane-release.yaml @@ -153,8 +153,6 @@ jobs: env: TF_WORKSPACE: ${{ inputs.environment }} OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.PROD_ONEPASSWORD_SERVICEACCOUNT_TOKEN }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: eu-central-1 + AWS_DEFAULT_REGION: eu-central-1 From 68aef01e02fed8b012dccaace5ebe2415094989e Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Mon, 7 Apr 2025 12:16:09 +0100 Subject: [PATCH 35/38] fix --- .github/workflows/crossplane-release.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/crossplane-release.yaml b/.github/workflows/crossplane-release.yaml index 401f30d9..8783ff75 100644 --- a/.github/workflows/crossplane-release.yaml +++ b/.github/workflows/crossplane-release.yaml @@ -145,6 +145,13 @@ jobs: # Verify kubectl can connect to the cluster kubectl cluster-info + # Create provider configuration file + cat > provider.tf << EOF + provider "kubectl" { + config_path = "${{ github.workspace }}/kubeconfig.yaml" + } + EOF + # Run terraform apply terraform apply -auto-approve -no-color \ -var-file=${{ inputs.environment }}.tfvars \ From 2a5c9a3bcc1fa20f0c218e220ab8ced743e9d279 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Mon, 7 Apr 2025 12:21:17 +0100 Subject: [PATCH 36/38] fix --- .github/workflows/crossplane-release.yaml | 2 ++ crossplane/versions.tf | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/crossplane-release.yaml b/.github/workflows/crossplane-release.yaml index 8783ff75..4bf467b2 100644 --- a/.github/workflows/crossplane-release.yaml +++ b/.github/workflows/crossplane-release.yaml @@ -149,6 +149,8 @@ jobs: cat > provider.tf << EOF provider "kubectl" { config_path = "${{ github.workspace }}/kubeconfig.yaml" + apply_retry_count = 5 + load_config_file = false } EOF diff --git a/crossplane/versions.tf b/crossplane/versions.tf index 251c6139..338feac8 100644 --- a/crossplane/versions.tf +++ b/crossplane/versions.tf @@ -23,8 +23,3 @@ terraform { } } -provider "kubectl" { - apply_retry_count = 5 - load_config_file = false -} - From d1235bcd5cf3f09daf3e75e77083fe370886fae0 Mon Sep 17 00:00:00 2001 From: Precious Okwu <32944039+libracoder@users.noreply.github.com> Date: Mon, 7 Apr 2025 12:21:28 +0100 Subject: [PATCH 37/38] fix --- .github/workflows/crossplane-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/crossplane-release.yaml b/.github/workflows/crossplane-release.yaml index 4bf467b2..4358a957 100644 --- a/.github/workflows/crossplane-release.yaml +++ b/.github/workflows/crossplane-release.yaml @@ -150,7 +150,7 @@ jobs: provider "kubectl" { config_path = "${{ github.workspace }}/kubeconfig.yaml" apply_retry_count = 5 - load_config_file = false + load_config_file = true } EOF From dbe9314ef5147099dc9e5fd09058b27e09716f3e Mon Sep 17 00:00:00 2001 From: Kieran Patel Date: Fri, 11 Apr 2025 15:26:46 +0200 Subject: [PATCH 38/38] Allow passing in role --- .github/workflows/crossplane.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/crossplane.yaml b/.github/workflows/crossplane.yaml index 40530dc0..c06d22a6 100644 --- a/.github/workflows/crossplane.yaml +++ b/.github/workflows/crossplane.yaml @@ -10,6 +10,12 @@ on: run_zip_lambda_workflow_step: type: boolean default: false + role_name: + type: string + default: 'github-actions-kubernetes-role' + role_session_name: + type: string + default: 'ga-kubernetes' permissions: @@ -49,8 +55,8 @@ jobs: name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-actions-kubernetes-role - role-session-name: ga-kubernetes + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ inputs.role_name }} + role-session-name: ${{ inputs.role_session_name }} aws-region: eu-central-1 - name: Install Terraform @@ -130,5 +136,3 @@ jobs: env: TF_WORKSPACE: ${{ inputs.environment }} OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.STAGING_ONEPASSWORD_SERVICEACCOUNT_TOKEN }} - -