Kubernetes EKS Gitlab Notes


helm install gitlab gitlab/gitlab --set global.hosts.domain=gitlab.digitaldevops.in --set certmanager.install=false --set global.ingress.configureCertmanager=false
helm install gitlab gitlab/gitlab --set global.hosts.domain=gitlab.digitaldevops.in --set certmanager-issuer.email=devops@rajeshkumar.xyz
helm install gitlab gitlab/gitlab \
--set certmanager.install=false \
--set global.ingress.configureCertmanager=false \
--set gitlab-runner.install=false
helm install gitlab gitlab/gitlab \
--set global.hosts.domain=gitlab.site.com \
--set certmanager.install=false \
--set global.ingress.configureCertmanager=false 
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: CUSTOM_STORAGE_CLASS_NAME
provisioner: kubernetes.io/aws-ebs
reclaimPolicy: Retain
parameters:
type: gp2
zone: '*AWS_ZONE*'
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: CUSTOM_STORAGE_CLASS_NAME
provisioner: kubernetes.io/aws-ebs
reclaimPolicy: Retain
parameters:
type: gp2
zone: '*AWS_ZONE*'
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: standard
provisioner: kubernetes.io/aws-ebs
parameters:
type: gp2
reclaimPolicy: Retain
allowVolumeExpansion: true
mountOptions:
- debug
volumeBindingMode: Immediate
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv1
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
awsElasticBlockStore:
volumeID: vol-027da2b8974bf4726
fsType: ext4
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv2
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
awsElasticBlockStore:
volumeID: 
fsType: ext4
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv3
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
awsElasticBlockStore:
volumeID: 
fsType: ext4
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv4
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
awsElasticBlockStore:
volumeID: vol-01bb15c5ebd8cf0fe
fsType: ext4
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv5
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
awsElasticBlockStore:
volumeID: vol-063ce825bcd5f2bfc
fsType: ext4
gitlab-postgresql
data-gitlab-postgresql-0
https://docs.gitlab.com/ee/install/requirements.html
oidc_id=$(aws eks describe-cluster --name eks-cluster1 --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5)
aws iam list-open-id-connect-providers | grep $oidc_id | cut -d "/" -f4
Pod and Persistent volume with existing EBS in EKS
eksctl utils associate-iam-oidc-provider --cluster eks-cluster1 --approve https://docs.gitlab.com/ee/install/requirements.html https://aws.amazon.com/premiumsupport/knowledge-center/eks-persistent-storage/ https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html
Pod and Persistent volume with existing EBS in EKS
https://docs.gitlab.com/charts/installation/cloud/eks.html https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html https://aws.amazon.com/premiumsupport/knowledge-center/eks-persistent-storage/ https://docs.gitlab.com/charts/installation/storage.html https://docs.aws.amazon.com/eks/latest/userguide/csi-iam-role.html https://docs.gitlab.com/charts/installation/deployment.html#persistence https://aws.amazon.com/premiumsupport/knowledge-center/eks-troubleshoot-ebs-volume-mounts/ https://aws.amazon.com/blogs/containers/introducing-efs-csi-dynamic-provisioning/ https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html https://docs.gitlab.com/charts/troubleshooting/ https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/examples/kubernetes/dynamic-provisioning/manifests/claim.yaml https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html https://aws.amazon.com/premiumsupport/knowledge-center/eks-troubleshoot-ebs-volume-mounts/ https://gitlab.com/gitlab-org/charts/gitlab/-/blob/master/examples/storage/eks_storage_class.yml https://docs.gitlab.com/charts/installation/storage.html https://docs.gitlab.com/charts/troubleshooting/ https://kubernetes.io/docs/concepts/storage/storage-classes/#the-storageclass-resource https://github.com/xinity/custom-gitlab/blob/master/doc/installation/storage.md https://stackoverflow.com/questions/51946393/kubernetes-pod-warning-1-nodes-had-volume-node-affinity-conflict https://github.com/kubernetes-sigs/aws-ebs-csi-driver https://aws-quickstart.github.io/quickstart-eks-gitlab/ https://aws-quickstart.github.io/quickstart-eks-gitlab/ https://dev.to/stack-labs/deploying-production-ready-gitlab-on-amazon-eks-with-terraform-3coh https://polaris.cse.unr.edu/gitlab/help/install/kubernetes/preparation/eks.md https://polaris.cse.unr.edu/gitlab https://gitlab.com/gitlab-org/charts/gitlab/blob/master/doc/installation/storage.md https://gitlab.com/gitlab-org/charts/gitlab/-/issues/3385 https://gitlab.com/gitlab-org/charts/gitlab/-/issues/3318 https://github.com/xinity/custom-gitlab/blob/master/doc/installation/storage.md https://polaris.cse.unr.edu/gitlab/help/install/kubernetes/preparation/eks.md https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/readonlymany-disks https://www.linode.com/community/questions/20215/how-to-re-attach-persistent-volume-to-pod-when-claim-is-deleted https://stackoverflow.com/questions/54629660/kubernetes-how-do-i-delete-pv-in-the-correct-manner https://gitlab.com/gitlab-org/charts/gitlab/-/issues/2709 https://gitlab.com/gitlab-org/charts/gitlab/-/issues/1692 https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/readonlymany-disks https://dev.to/stack-labs/deploying-production-ready-gitlab-on-amazon-eks-with-terraform-3coh https://gitlab.com/gitlab-org/charts/gitlab/-/issues/3935 https://docs.aws.amazon.com/eks/latest/userguide/install-aws-iam-authenticator.html https://docs.gitlab.com/charts/ https://docs.aws.amazon.com/eks/latest/userguide/create-node-role.html https://docs.securestate.vmware.com/rule-docs/eks-nodegroup-configured-with-admin-iam-role https://stackoverflow.com/questions/50667437/what-to-do-with-released-persistent-volume https://gitlab.com/gitlab-org/charts/gitlab/-/issues/3935 https://docs.aws.amazon.com/eks/latest/userguide/create-cluster.html https://kubernetes.io/blog/2019/04/04/kubernetes-1.14-local-persistent-volumes-ga/ https://stackoverflow.com/questions/72262623/kubernetes-pod-fails-with-unable-to-attach-or-mount-volumes https://raw.githubusercontent.com/kubernetes-sigs/aws-ebs-csi-driver/release-1.3/docs/example-iam-policy.json https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html https://aws.amazon.com/premiumsupport/knowledge-center/eks-troubleshoot-ebs-volume-mounts/ ubuntu@ip-172-31-16-250:~/rajesh$ eksctl utils associate-iam-oidc-provider --cluster eks-cluster1 --approve 2023-02-09 04:22:52 [ℹ] will create IAM Open ID Connect provider for cluster "eks-cluster1" in "ap-northeast-1" 2023-02-09 04:22:52 [✔] created IAM Open ID Connect provider for cluster "eks-cluster1" in "ap-northeast-1" https://docs.gitlab.com/charts/installation/storage.html List of prerequisites before setting up EKS cluster Before setting up an Amazon Elastic Container Service for Kubernetes (EKS) cluster, there are several prerequisites that must be met: AWS Account: You need an AWS account to access AWS services, including EKS. AWS CLI and AWS IAM Authenticator: You need to have the AWS CLI installed and configured on your machine to create and manage an EKS cluster. Additionally, you need to install the AWS IAM Authenticator for Kubernetes to manage authentication between your local machine and the EKS cluster. VPC and Subnets: You need to create a Virtual Private Cloud (VPC) and subnets in which to run your EKS cluster. Security Groups: You need to create security groups that control access to the nodes in your EKS cluster and to the cluster itself. IAM Roles: You need to create IAM roles to allow the EKS control plane to manage the nodes in your cluster. Kubernetes CLI (kubectl): You need to install the Kubernetes CLI (kubectl) on your local machine to manage your EKS cluster. AWS Resources: You need to create additional AWS resources, such as an S3 bucket, to store configuration data for your EKS cluster. Kubernetes Troubleshooting with Volume https://stackoverflow.com/questions/72262623/kubernetes-pod-fails-with-unable-to-attach-or-mount-volumes ----------------------- I figured what my issue was. My AWS EBS CSI controllers were running on nodes with IAM roles having insufficient permissions. As a result I was seeing these messages in the logs: $ kubectl logs deployment/ebs-csi-controller -n kube-system -c ebs-plugin status code: 403, request id: f4bdbecb-40d5-4eeb-bcef-d0b734a94c2a E0212 21:04:38.366854 1 driver.go:120] GRPC error: rpc error: code = Internal desc = Could not attach volume "vol-0b10c235246e76523" to node "i-0bceabf074ee5f7c7": could not attach volume "vol-0b10c235246e76523" to node "i-0bceabf074ee5f7c7": UnauthorizedOperation: You are not authorized to perform this operation. Encoded authorization failure message: 1rf720y-vwEYGFNwphni8ZXNr42fkuH3Vx7GWJgExmOd58-tN0S4nkAG6RHWPjHCl_ODo4ripUzogFRKRyPbFOROFCzl7uyTgs3RcWrVVWX0Ug6scvyKRvO7SPMhXsWH0HpDPXWJhqo1_9hJzgP13hE1ecfqCsN204zQQNYziNf3dmELgHnW24XQMdDEF_TOzY0u82xBRJUIVvb7W-w7E1PWbYCW0pT_D8AuEIeoRY-fXfmGZb11-SqY35GB1wFBt-06s0tqphQbthMuRLT5ios33FcyJE3PqI2o6FHF09CGnbFcoxCR1BaDKZ7RAIxM_qHP87JuOSZvQxk3lYa45rlqhj3p0dI4ByTVO1sNX6EJFLkffAnLa0-GSbRhWubUlj1bPQ_UqYnkK5iII2h4IBIUvrPu0vHR0tAkdb2BIM1r7vl1vx9KPFUfjXMhu_KA7thujWYwb7_9N3pj-VC4nn8SL5gmtWqB9NdUziSLh76WlA9xmuB59fJOoFVFdsvmawMxFM3rKCrmHFJUiot9-ZcrC9adZe6wPu4CVqA_Coqm_IIuPc6haySr6P_EylT4k51Bo08eUWCaSQilRFYwEh0GlN4cqOSaiEJ6hGhRg1ID_Qgxt1Iz3kM00hlRBPO3JIYzQY3k-24vvhBZShUmO8fa2MkAIhBArdSwTVnhb0kt3R-unLNkyguWJ8A status code: 403, request id: c6f0488d-0a45-4e70-bb99-35c3635418a6 --------------------------------------------- data-gitlab-postgresql-0 data-gitlab-postgresql-0 kubectl describe pvc data-gitlab-postgresql-0 -n <namespace> oidc_id=$(aws eks describe-cluster --name eks-cluster1 --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5) aws-iam-authenticator EKS EC2 VPC ROUTE53 \ What is OIDC What is aws-iam-authenticator kubectl patch pv imReannotations: storageclass.kubernetes.io/is-default-class: "true" allowVolumeExpansion: true provisioner: ebs.csi.aws.com volumeBindingMode: WaitForFirstConsumer reclaimPolicy: Delete parameters: type: gp3 fsType: ext4 allowedTopologies: - matchLabelExpressions: - key: topology.ebs.csi.aws.com/zone values: - us-east-2a - us-east-2b - us-east-2c https://github.com/aws/karpenter/issues/1775 ubuntu@ip-172-31-16-250:~/rajesh$ kubectl logs gitlab-sidekiq-all-in-1-v2-544b887df7-fs8wz Defaulted container "sidekiq" out of: sidekiq, certificates (init), configure (init), dependencies (init) Error from server (BadRequest): container "sidekiq" in pod "gitlab-sidekiq-all-in-1-v2-544b887df7-fs8wz" is waiting to start: PodInitializing Warning FailedScheduling 3m15s default-scheduler 0/2 nodes are available: 2 Too many pods. preemption: 0/2 nodes are available: 2 No preemption victims found for incoming pod. kubectl patch pv pv1 -p '{"spec":{"claimRef": null}}' kubectl patch pv pv2 -p '{"spec":{"claimRef": null}}' kubectl patch pv pv3 -p '{"spec":{"claimRef": null}}' kubectl patch pv pv4 -p '{"spec":{"claimRef": null}}' kubectl patch pv pv5 -p '{"spec":{"claimRef": null}}' https://github.com/aws/karpenter/issues/1775 kubectl get pod gitlab-sidekiq-all-in-1-v2-544b887df7-glbh7 --template '{{.status.initContainerStatuses}}' kubectl get pod gitlab-webservice-default-64568bbf56-8mst6 --template '{{.status.initContainerStatuses}}' kubectl get pod gitlab-webservice-default-64568bbf56-wkcrc --template '{{.status.initContainerStatuses}}' kubectl logs gitlab-webservice-default-64568bbf56-8mst6 -c certificates kubectl logs gitlab-webservice-default-64568bbf56-wkcrc -c certificates kubectl get deployment -lapp=webservice -ojsonpath='{.items[0].spec.template.spec.initContainers[0].image}' apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: CUSTOM_STORAGE_CLASS_NAME provisioner: kubernetes.io/aws-ebs reclaimPolicy: Retain parameters: type: gp2 zone: '' kubectl patch storageclass gitlab -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: slow provisioner: kubernetes.io/aws-ebs parameters: type: io1 iopsPerGB: "10" fsType: ext4 apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: standard provisioner: kubernetes.io/gce-pd parameters: type: pd-standard volumeBindingMode: WaitForFirstConsumer allowedTopologies: - matchLabelExpressions: - key: failure-domain.beta.kubernetes.io/zone values: - us-central-1a - us-central-1b Before setting up an Amazon Elastic Container Service for Kubernetes (EKS) cluster, there are several prerequisites that must be met: AWS Account: You need an AWS account to access AWS services, including EKS. AWS CLI and AWS IAM Authenticator: You need to have the AWS CLI installed and configured on your machine to create and manage an EKS cluster. Additionally, you need to install the AWS IAM Authenticator for Kubernetes to manage authentication between your local machine and the EKS cluster. VPC and Subnets: You need to create a Virtual Private Cloud (VPC) and subnets in which to run your EKS cluster. Security Groups: You need to create security groups that control access to the nodes in your EKS cluster and to the cluster itself. IAM Roles: You need to create IAM roles to allow the EKS control plane to manage the nodes in your cluster. Kubernetes CLI (kubectl): You need to install the Kubernetes CLI (kubectl) on your local machine to manage your EKS cluster. AWS Resources: You need to create additional AWS resources, such as an S3 bucket, to store configuration data for your EKS cluster.

Kubernetes EKS Gitlab Database issues –

Error

ubuntu@ip-172-31-16-250:~/rajesh$ kubectl logs gitlab-postgresql-0
Defaulted container "gitlab-postgresql" out of: gitlab-postgresql, metrics
postgresql 00:13:21.83
postgresql 00:13:21.83 Welcome to the Bitnami postgresql container
postgresql 00:13:21.84 Subscribe to project updates by watching https://github.com/bitnami/bitnami-docker-postgresql
postgresql 00:13:21.84 Submit issues and feature requests at https://github.com/bitnami/bitnami-docker-postgresql/issues
postgresql 00:13:21.85
postgresql 00:13:21.87 INFO  ==> ** Starting PostgreSQL setup **
postgresql 00:13:21.91 INFO  ==> Validating settings in POSTGRESQL_* env vars..
postgresql 00:13:21.92 INFO  ==> Loading custom pre-init scripts...
postgresql 00:13:21.93 INFO  ==> Loading user's custom files from /docker-entrypoint-preinitdb.d ...
postgresql 00:13:21.94 INFO  ==> Initializing PostgreSQL database...
postgresql 00:13:21.96 INFO  ==> pg_hba.conf file not detected. Generating it...
postgresql 00:13:21.97 INFO  ==> Generating local authentication configuration
postgresql 00:13:23.46 INFO  ==> Starting PostgreSQL in background...
postgresql 00:13:23.70 INFO  ==> Changing password of postgres
postgresql 00:13:23.71 INFO  ==> Creating user gitlab
postgresql 00:13:23.73 INFO  ==> Granting access to "gitlab" to the database "gitlabhq_production"
postgresql 00:13:23.76 INFO  ==> Setting ownership for the 'public' schema database "gitlabhq_production" to "gitlab"
postgresql 00:13:23.79 INFO  ==> Configuring replication parameters
postgresql 00:13:23.84 INFO  ==> Configuring fsync
postgresql 00:13:23.88 INFO  ==> Loading custom scripts...
postgresql 00:13:23.89 INFO  ==> Loading user's custom files from /docker-entrypoint-initdb.d ...
postgresql 00:13:23.89 INFO  ==> Starting PostgreSQL in background...
CREATE EXTENSION
postgresql 00:13:24.07 INFO  ==> Enabling remote connections
postgresql 00:13:24.09 INFO  ==> Stopping PostgreSQL...
waiting for server to shut down.... done
server stopped
postgresql 00:13:24.21 INFO  ==> ** PostgreSQL setup finished! **
postgresql 00:13:24.26 INFO  ==> ** Starting PostgreSQL **
2023-02-13 00:13:24.294 GMT [1] LOG:  pgaudit extension initialized
2023-02-13 00:13:24.294 GMT [1] LOG:  starting PostgreSQL 12.7 on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit
2023-02-13 00:13:24.295 GMT [1] LOG:  listening on IPv4 address "0.0.0.0", port 5432
2023-02-13 00:13:24.295 GMT [1] LOG:  listening on IPv6 address "::", port 5432
2023-02-13 00:13:24.301 GMT [1] LOG:  listening on Unix socket "/tmp/.s.PGSQL.5432"
2023-02-13 00:13:24.328 GMT [171] LOG:  database system was shut down at 2023-02-13 00:13:24 GMT
2023-02-13 00:13:24.341 GMT [1] LOG:  database system is ready to accept connections
2023-02-13 00:13:41.037 GMT [214] ERROR:  relation "postgres_partitioned_tables" does not exist at character 85
2023-02-13 00:13:41.037 GMT [214] STATEMENT:  /*application:web,db_config_name:main*/ SELECT "postgres_partitioned_tables".* FROM "postgres_partitioned_tables" WHERE (identifier = concat(current_schema(), '.', 'audit_events')) LIMIT 1
2023-02-13 00:13:41.039 GMT [214] ERROR:  relation "postgres_partitioned_tables" does not exist at character 85
2023-02-13 00:13:41.039 GMT [214] STATEMENT:  /*application:web,db_config_name:main*/ SELECT "postgres_partitioned_tables".* FROM "postgres_partitioned_tables" WHERE (identifier = concat(current_schema(), '.', 'web_hook_logs')) LIMIT 1
2023-02-13 00:13:41.042 GMT [214] ERROR:  relation "postgres_partitioned_tables" does not exist at character 85
2023-02-13 00:13:41.042 GMT [214] STATEMENT:  /*application:web,db_config_name:main*/ SELECT "postgres_partitioned_tables".* FROM "postgres_partitioned_tables" WHERE (identifier = concat(current_schema(), '.', 'loose_foreign_keys_deleted_records')) LIMIT 1
2023-02-13 00:13:41.044 GMT [214] ERROR:  relation "postgres_partitioned_tables" does not exist at character 85
2023-02-13 00:13:41.044 GMT [214] STATEMENT:  /*application:web,db_config_name:main*/ SELECT "postgres_partitioned_tables".* FROM "postgres_partitioned_tables" WHERE (identifier = concat(current_schema(), '.', 'batched_background_migration_job_transition_logs')) LIMIT 1
2023-02-13 00:13:41.046 GMT [214] ERROR:  relation "postgres_partitioned_tables" does not exist at character 85
2023-02-13 00:13:41.046 GMT [214] STATEMENT:  /*application:web,db_config_name:main*/ SELECT "postgres_partitioned_tables".* FROM "postgres_partitioned_tables" WHERE (identifier = concat(current_schema(), '.', 'incident_management_pending_alert_escalations')) LIMIT 1
2023-02-13 00:13:41.048 GMT [214] ERROR:  relation "postgres_partitioned_tables" does not exist at character 85
2023-02-13 00:13:41.048 GMT [214] STATEMENT:  /*application:web,db_config_name:main*/ SELECT "postgres_partitioned_tables".* FROM "postgres_partitioned_tables" WHERE (identifier = concat(current_schema(), '.', 'incident_management_pending_issue_escalations')) LIMIT 1
2023-02-13 00:13:41.050 GMT [214] ERROR:  relation "postgres_partitioned_tables" does not exist at character 85
2023-02-13 00:13:41.050 GMT [214] STATEMENT:  /*application:web,db_config_name:main*/ SELECT "postgres_partitioned_tables".* FROM "postgres_partitioned_tables" WHERE (identifier = concat(current_schema(), '.', 'security_findings')) LIMIT 1
2023-02-13 00:13:41.051 GMT [214] ERROR:  relation "postgres_partitioned_tables" does not exist at character 85
2023-02-13 00:13:41.051 GMT [214] STATEMENT:  /*application:web,db_config_name:main*/ SELECT "postgres_partitioned_tables".* FROM "postgres_partitioned_tables" WHERE (identifier = concat(current_schema(), '.', 'verification_codes')) LIMIT 1
2023-02-13 00:14:14.950 GMT [284] ERROR:  duplicate key value violates unique constraint "index_shards_on_name"
2023-02-13 00:14:14.950 GMT [284] DETAIL:  Key (name)=(default) already exists.
2023-02-13 00:14:14.950 GMT [284] STATEMENT:  /*application:web,db_config_name:main*/ INSERT INTO "shards" ("name") VALUES ('default') RETURNING "id"