slim
slim copied to clipboard
slimtoolkit
Slim stripping dependencies eventhough added that entire dirs in --preserve-path
Cloud build
steps:
# Step 1: Generate env.yaml file for Cloud Run deployment
- name: "gcr.io/cloud-builders/gcloud"
entrypoint: bash
args:
- -c
- |
cat <<EOF > env.yaml
ENV_TYPE: "${_ENV_TYPE}"
project_number: "${_PROJECT_NO}"
ENV_LINK: "${_ENV_LINK}"
PROJECT_ID: "${_PROJECT_ID}"
BUCKET_ORGNAME_SPEND_DATA: "${_BUCKET_ORGNAME_SPEND_DATA}"
BUCKET_ORGNAME_CONTRACT_DATA: "${_BUCKET_ORGNAME_CONTRACT_DATA}"
BUCKET_DIFFBOT_JSON_RESPONSE: "${_BUCKET_DIFFBOT_JSON_RESPONSE}"
DOC_AI_PROJECT_ID: "${_DOC_AI_PROJECT_ID}"
USER_AUTH_GCP_PROJECT_ID: "${_USER_AUTH_GCP_PROJECT_ID}"
URL_SEND_MAIL_APPSPOT: "${_SEND_MAIL_SERVICE_URL}"
FIRESTORE_DB: "${_FIRESTORE_DB}"
BQ_TABLE_NAMES: "${_BQ_TABLE_NAMES}"
EOF
id: "Generate env.yaml"
# Step 2: Generate .env file for SlimToolkit probing
- name: "gcr.io/cloud-builders/gcloud"
entrypoint: bash
args:
- -c
- |
cat <<EOF > .env
GOOGLE_CLOUD_PROJECT=${_PROJECT_ID}
ENV_TYPE=${_ENV_TYPE}
project_number=${_PROJECT_NO}
ENV_LINK=${_ENV_LINK}
BUCKET_ORGNAME_SPEND_DATA=${_BUCKET_ORGNAME_SPEND_DATA}
BUCKET_ORGNAME_CONTRACT_DATA=${_BUCKET_ORGNAME_CONTRACT_DATA}
BUCKET_DIFFBOT_JSON_RESPONSE=${_BUCKET_DIFFBOT_JSON_RESPONSE}
DOC_AI_PROJECT_ID=${_DOC_AI_PROJECT_ID}
USER_AUTH_GCP_PROJECT_ID=${_USER_AUTH_GCP_PROJECT_ID}
URL_SEND_MAIL_APPSPOT=${_SEND_MAIL_SERVICE_URL}
FIRESTORE_DB=${_FIRESTORE_DB}
BQ_TABLE_NAMES=${_BQ_TABLE_NAMES}
EOF
id: "Generate .env for SlimToolkit"
# Step 3: Get service account key from Secret Manager
- name: gcr.io/cloud-builders/gcloud
id: "Get service account key"
entrypoint: bash
args:
- -c
- |
gcloud secrets versions access latest --secret=google_application_credentials-dev > /workspace/service-account-key.json
echo "Service account key saved to /workspace/service-account-key.json"
# Step 4: Build fat Docker image
- name: gcr.io/cloud-builders/docker
args:
- build
- "-t"
- "${_REGION}-docker.pkg.dev/${_DEVOPS_PROJECT_ID}/${_REPO_NAME}/${_SERVICE_NAME}-${_ENV_TYPE}:latest"
- "-t"
- "${_REGION}-docker.pkg.dev/${_DEVOPS_PROJECT_ID}/${_REPO_NAME}/${_SERVICE_NAME}-${_ENV_TYPE}:$SHORT_SHA"
- "."
# Step 5: Minify Docker image with SlimToolkit (FIXED + secrets mounted)
- name: gcr.io/cloud-builders/docker
id: "Minify Docker image"
entrypoint: bash
args:
- -c
- |
set -e
echo "Downloading SlimToolkit..."
curl -L -o ds.tar.gz https://github.com/slimtoolkit/slim/releases/download/1.40.11/dist_linux.tar.gz
tar -xvf ds.tar.gz
mv dist_linux/mint /usr/local/bin/slim
mv dist_linux/mint-sensor /usr/local/bin/
chmod +x /usr/local/bin/slim /usr/local/bin/mint-sensor
echo "Verifying SlimToolkit..."
slim --version
echo "Building slim image with IPC and environment variables from .env..."
slim build \
--target ${_REGION}-docker.pkg.dev/${_DEVOPS_PROJECT_ID}/${_REPO_NAME}/${_SERVICE_NAME}-${_ENV_TYPE}:$SHORT_SHA \
--sensor-ipc-mode proxy \
--sensor-ipc-endpoint $(docker network inspect bridge -f '{{range .IPAM.Config}}{{.Gateway}}{{end}}' | cut -f1) \
--http-probe-ports 8080 \
--http-probe-cmd "/" \
--show-clogs \
--show-blogs \
--preserve-path /app \
--preserve-path /opt/venv \
--preserve-path-file requirements.txt \
--env-file .env \
--mount /workspace/service-account-key.json:/app/service-account-key.json:ro \
--env GOOGLE_APPLICATION_CREDENTIALS=/app/service-account-key.json \
--include-oslibs-net \
--include-cert-all \
--include-cert-dirs \
--include-new \
--keep-perms \
--include-shell \
--tag ${_REGION}-docker.pkg.dev/${_DEVOPS_PROJECT_ID}/${_REPO_NAME}/${_SERVICE_NAME}-${_ENV_TYPE}:$SHORT_SHA \
${_REGION}-docker.pkg.dev/${_DEVOPS_PROJECT_ID}/${_REPO_NAME}/${_SERVICE_NAME}-${_ENV_TYPE}:latest
echo "Tagging slim image as latest..."
docker tag ${_REGION}-docker.pkg.dev/${_DEVOPS_PROJECT_ID}/${_REPO_NAME}/${_SERVICE_NAME}-${_ENV_TYPE}:$SHORT_SHA \
${_REGION}-docker.pkg.dev/${_DEVOPS_PROJECT_ID}/${_REPO_NAME}/${_SERVICE_NAME}-${_ENV_TYPE}:latest
# Copy creport.json to workspace so next step can upload
cp /tmp/mint-state/.mint-state/images/*/artifacts/creport.json ./ || echo "creport.json not found"
# Upload Slim artifacts to Cloud Build logs bucket
- name: gcr.io/cloud-builders/gsutil
entrypoint: bash
args:
- -c
- |
echo "Uploading SlimToolkit artifacts to gs://${_LOGS_BUCKET}/slim_reports/${_SERVICE_NAME}-${_ENV_TYPE}/${SHORT_SHA}/"
# Upload slim.report.json
if [ -f "slim.report.json" ]; then
echo "Uploading slim.report.json..."
gsutil cp slim.report.json \
"gs://${_LOGS_BUCKET}/slim_reports/${_SERVICE_NAME}-${_ENV_TYPE}/${SHORT_SHA}/" \
|| echo "Failed to upload slim.report.json"
else
echo "slim.report.json not found"
fi
# Upload creport.json
if [ -f "creport.json" ]; then
echo "Uploading creport.json..."
gsutil cp creport.json \
"gs://${_LOGS_BUCKET}/slim_reports/${_SERVICE_NAME}-${_ENV_TYPE}/${SHORT_SHA}/" \
|| echo "Failed to upload creport.json"
else
echo "creport.json not found"
fi
echo "✅ Upload completed to gs://${_LOGS_BUCKET}/slim_reports/${_SERVICE_NAME}-${_ENV_TYPE}/${SHORT_SHA}/"
id: Upload Slim artifacts
# Step 6: Push Docker image - latest (now slim)
- name: gcr.io/cloud-builders/docker
args:
- push
- "${_REGION}-docker.pkg.dev/${_DEVOPS_PROJECT_ID}/${_REPO_NAME}/${_SERVICE_NAME}-${_ENV_TYPE}:latest"
id: "Push Docker image - latest"
# Step 7: Push Docker image - commit SHA (now slim)
- name: gcr.io/cloud-builders/docker
args:
- push
- "${_REGION}-docker.pkg.dev/${_DEVOPS_PROJECT_ID}/${_REPO_NAME}/${_SERVICE_NAME}-${_ENV_TYPE}:$SHORT_SHA"
id: "Push Docker image - $SHORT_SHA"
# Step 8: Deploy to Cloud Run
- name: gcr.io/google.com/cloudsdktool/cloud-sdk
args:
- run
- deploy
- "${_SERVICE_NAME}-${_ENV_TYPE}"
- "--image"
- "${_REGION}-docker.pkg.dev/${_DEVOPS_PROJECT_ID}/${_REPO_NAME}/${_SERVICE_NAME}-${_ENV_TYPE}:$SHORT_SHA"
- "--allow-unauthenticated"
- "--service-account"
- "${_SERVICE_ACCOUNT}"
- "--region"
- "${_REGION}"
- "--timeout"
- "${_TIMEOUT}"
- "--memory"
- "${_MEMORY_CONF}"
- "--cpu"
- "${_CPU_CONF}"
- "--min-instances"
- "${_MIN_INSTANCE_CONF}"
- "--max-instances"
- "${_MAX_INSTANCE_CONF}"
- "--project"
- "${_PROJECT_ID}"
- "--env-vars-file"
- "env.yaml"
id: "Deploy to Cloud Run"
entrypoint: gcloud
logsBucket: "${_LOGS_BUCKET}"
serviceAccount: "${_CLOUD_BUILD_SA}"
even though i have added to don't touch /app dir and venv still it is removing and some depencies like gcsfs , webbrowser etc....
@RahulRaval28 thank you for a detailed report! It's very rare to see that level of details. I really appreciate it. Where do you have gcsfs and webbrowser located? What are their paths and what kind of apps are they? Are they compiled binaries? The --preserve-* flags are meant to make sure changed/deleted files (during container execution) in the container image are still retained. You probably don't want to use those preserve flags unless you are explicitly looking for this behavior. Instead, try using the include flags like --include-path, etc. Speaking of including files... What do you expect from this part of the slim config: --preserve-path-file requirements.txt? Can you also share a bit more about your application in the container image? What does it do? Does it have an http API and if so can you share an example calling one of the endpoints in it?
