IAM User 생성 및 권한 부여

1. EKS Admin policy 생성

• IAM 접속 - Policies - Create policy - JSON

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "VisualEditor0",
      "Effect": "Allow",
      "Action": [
          "ecr:*",
          "ec2:*",
          "eks:*",
          "iam:*",
          "s3:*",
          "elasticfilesystem:*",
          "autoscaling:*",
          "cloudformation:*"
      ],
      "Resource": "*"
    }
  ]
}

매우 강력한 권한이니 워크샵이 끝난 이후 삭제 바랍니다.

• Review policy
• Name: EKS-admin

2. User 생성

• User name: k8s-ml
• Access type: Programmatic access
• Next Permissions
• Attach existing policies directly: EKS-admin 검색
• Next Tags - Next Review - Create user
• Access key, Secret key 저장

Setup

가장 먼저 EKS 마스터에 명령을 전달할 EC2 서버 하나를 생성합니다. 본인의 PC에서 직접 작업을 진행하셔도 무방합니다. 본 워크샵은 Ubuntu 18.04 위에서 정상적으로 동작하도록 구성이 되어 있습니다. 먼저 kubectl 명령 서버를 하나 만들겠습니다. http://console.aws.amazon.com

# git clone
git clone https://github.com/hongkunyoo/how-to-scale-your-ml-job-with-k8s.git && cd how-to-scale-your-ml-job-with-k8s

# install jq
sudo apt-get update && \
    sudo apt-get install -y jq apt-transport-https \
    ca-certificates \
    curl \
    gnupg-agent \
    software-properties-common 


# install awscli
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
    /bin/bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda3 && \
    rm Miniconda3-latest-Linux-x86_64.sh && \
    echo 'export PATH=$HOME/miniconda3/bin:$PATH' >> $HOME/.bashrc && \
    $HOME/miniconda3/bin/pip install awscli && \
    source $HOME/.bashrc

# AWS user configure
aws configure
# access key: XXX
# secret key: XXX
# region: ap-northeast-2

# 클러스터 이름과 리전을 설정합니다.
CLUSTER_NAME=k8s-ml

# installing eksctl
curl --location "https://github.com/weaveworks/eksctl/releases/download/latest_release/eksctl_$(uname -s)_amd64.tar.gz" | \
    tar xz -C /tmp && \
    sudo mv /tmp/eksctl /usr/local/bin

# installing heptio-authenticator
curl -o aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.11.5/2018-12-06/bin/linux/amd64/aws-iam-authenticator && \
    chmod +x ./aws-iam-authenticator && \
    sudo mv aws-iam-authenticator /usr/local/bin

# installing kubectl
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - && \
    echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list && \
    sudo apt-get update && \
    sudo apt-get install -y kubectl

# install docker
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - && \
    sudo add-apt-repository \
       "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
       $(lsb_release -cs) \
       stable" && \
    sudo apt-get update && \
    sudo apt-get install -y docker-ce docker-ce-cli containerd.io && \
    sudo usermod -aG docker $USER


# Create k8s cluster
eksctl create cluster --name $CLUSTER_NAME --without-nodegroup

# default worker node 구성
eksctl create nodegroup --cluster $CLUSTER_NAME --name default --nodes-min 1 --nodes-max 1 --nodes 1 --node-labels "role=default" --node-type m5.xlarge --asg-access

# CPU worker node 구성
eksctl create nodegroup --cluster $CLUSTER_NAME --name train-cpu --nodes-min 1 --nodes-max 4 --nodes 2 --node-labels "role=train-cpu" --node-type c5.xlarge --asg-access

# MEM worker node 구성
eksctl create nodegroup --cluster $CLUSTER_NAME --name train-mem --nodes-min 1 --nodes-max 2 --nodes 1 --node-labels "role=train-mem" --node-type r5.xlarge  --asg-access

# 클러스터 확인
kubectl get node -L role

# Create EFS filesystem
FS_ID=$(aws efs create-file-system --creation-token $CLUSTER_NAME | jq -r .FileSystemId)

# Manage file system access
# AWS console

# installing helm client
curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash

# RBAC setting
cat <<EOF | kubectl create -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: default-admin
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: default
  namespace: kube-system
EOF

# installing helm
helm init --service-account default

# before installing helm chart, change values
vi charts/nfs-client-provisioner/values.yaml

# line 14
nfs:
  server: !(FS_ID).efs.ap-northeast-2.amazonaws.com
  path: /

vi charts/minio/values.yaml

# line 45
nfsServer: !(FS_ID).efs.ap-northeast-2.amazonaws.com
nfsServerPath: /

# install helm charts
helm install charts/nfs-client-provisioner --namespace kube-system
helm install charts/minio --namespace kube-system
helm install charts/metrics-server --namespace kube-system

# check all chart is running
kubectl get pod -n kube-system

echo "This is your ECR repository: "$(aws sts get-caller-identity | jq -r .Account).dkr.ecr.ap-northeast-2.amazonaws.com/\$IMAGE_NAME
aws ecr create-repository --repository-name k8s-ml
$(aws ecr get-login --no-include-email)

git clone https://github.com/hongkunyoo/how-to-scale-your-ml-job-with-k8s.git && cd how-to-scale-your-ml-job-with-k8s

gcloud components update

gcloud config set compute/zone us-central1-a

CLUSTER_NAME=k8s-ml

gcloud container clusters create $CLUSTER_NAME \
    --num-nodes=1 \
    --node-labels=role=default \
    --machine-type=n1-standard-4 \
    --node-locations=us-central1-a


gcloud container node-pools create train-cpu \
    --cluster $CLUSTER_NAME \
    --node-labels=role=train-cpu \
    --enable-autoscaling \
    --min-nodes=1 \
    --num-nodes=2 \
    --max-nodes=4 \
    --machine-type=n1-highcpu-8


gcloud container node-pools create train-mem \
    --cluster $CLUSTER_NAME \
    --node-labels=role=train-mem \
    --enable-autoscaling \
    --min-nodes=1 \
    --num-nodes=1 \
    --max-nodes=2 \
    --machine-type=n1-highmem-8

# 클러스터 확인
kubectl get node -L role

# RBAC setting
cat <<EOF | kubectl create -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: default-admin
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: default
  namespace: kube-system
EOF

# installing helm
helm init --service-account default

# Create Cloud FileStore
gcloud filestore instances create nfs-storage \
    --project=$DEVSHELL_PROJECT_ID \
    --file-share=name="vol",capacity=1TB \
    --zone=us-central1-a \
    --network=name="default",reserved-ip-range="10.0.0.0/29"

vi charts/nfs-client-provisioner/values.yaml

# line 14
nfs:
  server: 10.0.0.2
  path: /vol

vi charts/minio/values.yaml

# line 45
nfsServer: 10.0.0.2
nfsServerPath: /vol

# install helm charts
helm install charts/nfs-client-provisioner --namespace kube-system
helm install charts/minio --namespace kube-system

kubectl get pod -n kube-system

echo "This is your GCR repository: "gcr.io/$(gcloud config get-value project)/IMAGE