Merge pull request 'Update CI to work with new network architecture' (#3) from update-CI-to-new-network-architecture into main

Reviewed-on: #3

.gitea/workflows/update.yml

@@ -5,54 +5,100 @@ on:
     # Runs every day at 3:00 AM UTC
     - cron: '0 3 * * *'
   workflow_dispatch: # Allows manual triggering
+    inputs:
+      skip_image_check:
+        description: 'Skip image check and force build'
+        required: false
+        default: 'false'
+        type: choice
+        options:
+        - 'true'
+        - 'false'
 
 jobs:
-  check-images:
+  get-mlflow-full-base-image-digest:
+    if: github.event.inputs.skip_image_check != 'true'
     runs-on: ubuntu-latest
     outputs:
-      base_image_digest: ${{ steps.mlflow-base.outputs.base_image_digest }}
+      base_image_digest: ${{ steps.mlflow-full.outputs.base_image_digest }}
-      should_build: ${{ steps.decision.outputs.should_build }}
+      success: ${{ steps.mlflow-full.outputs.success }}
     steps:
-    - name: Log in to Gitea Container Registry
+    - name: Get MLflow-full base image digest
-      run: |
-        mkdir -p ~/.docker
-        echo '{"insecure-registries": ["10.0.0.2:3000"]}' > ~/.docker/config.json
-        echo "${{ secrets.CI_RUNNER_TOKEN }}" | docker login 10.0.0.2:3000 --username "${{ gitea.actor }}" --password-stdin
-    - name: Check MLflow-full image
       id: mlflow-full
+      env:
+        CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
       run: |
-        MANIFEST=$(docker manifest inspect "gitea.gt-proj.com/brian/mlflow-full:latest" 2>/dev/null || echo "{}")
+        # Configure Docker for insecure registry
-        if [ "$MANIFEST" != "{}" ]; then
+        mkdir -p ~/.docker
-          DIGEST=$(echo "$MANIFEST" | jq -r '.manifests[]? | select(.platform.architecture == "amd64" and .platform.os == "linux") | .digest // empty')
+        echo '{
-          echo "digest=$DIGEST" >> $GITHUB_OUTPUT
+          "auths": {},
-          echo "manifest_found=true" >> $GITHUB_OUTPUT
+          "insecure-registries": ["${{ secrets.DOCKER_REGISTRY }}"]
-          echo "Manifest found for MLflow-full image with digest: $DIGEST"
+        }' > ~/.docker/config.json
+        
+        # Login to internal registry
+        echo "${{ secrets.CI_RUNNER_TOKEN }}" | docker login ${{ secrets.DOCKER_REGISTRY }} --username "${{ gitea.actor }}" --password-stdin
+
+        # Pull and inspect the image
+        if docker pull ${{ secrets.DOCKER_REGISTRY }}/brian/mlflow-full:latest 2>/dev/null; then
+          BASE_IMAGE_DIGEST=$(docker inspect ${{ secrets.DOCKER_REGISTRY }}/brian/mlflow-full:latest | jq -r '.[0].Config.Labels["org.opencontainers.image.base.digest"] // empty')
+          
+          if [ -n "$BASE_IMAGE_DIGEST" ] && [ "$BASE_IMAGE_DIGEST" != "null" ]; then
+            echo "base_image_digest=$BASE_IMAGE_DIGEST" >> $GITHUB_OUTPUT
+            echo "success=true" >> $GITHUB_OUTPUT
+            echo "Base image digest from MLflow-full image: $BASE_IMAGE_DIGEST"
+          else
+            echo "success=false" >> $GITHUB_OUTPUT
+            echo "No base image digest label found"
+          fi
         else
-          echo "manifest_found=false" >> $GITHUB_OUTPUT
+          echo "success=false" >> $GITHUB_OUTPUT
-          echo "MLflow-full image not found"
+          echo "Failed to pull MLflow-full image - this might be the first run"
         fi
-    - name: Check MLflow base image
+  get-mlflow-digest:
+    runs-on: ubuntu-latest
+    outputs:
+      digest: ${{ steps.mlflow-base.outputs.digest }}
+      success: ${{ steps.mlflow-base.outputs.success }}
+    steps:
+    - name: Get MLflow base image digest
       id: mlflow-base
-      if: steps.mlflow-full.outputs.manifest_found == 'true'
       run: |
         MANIFEST=$(docker manifest inspect ghcr.io/mlflow/mlflow:latest 2>/dev/null || echo "{}")
         if [ "$MANIFEST" != "{}" ]; then
           DIGEST=$(echo "$MANIFEST" | jq -r '.manifests[]? | select(.platform.architecture == "amd64" and .platform.os == "linux") | .digest // empty')
           echo "digest=$DIGEST" >> $GITHUB_OUTPUT
-          echo "manifest_found=true" >> $GITHUB_OUTPUT
+          echo "success=true" >> $GITHUB_OUTPUT
-          echo "Manifest found for MLflow base image with digest: $DIGEST"
+          echo "Digest found for MLflow base image: $DIGEST"
         else
-          echo "manifest_found=false" >> $GITHUB_OUTPUT
+          echo "success=false" >> $GITHUB_OUTPUT
           echo "MLflow base image not found"
         fi
+  decide-to-build:
+    if: always()  # Run regardless of dependency status
+    needs:
+      - get-mlflow-full-base-image-digest
+      - get-mlflow-digest
+    runs-on: ubuntu-latest
+    outputs:
+      should_build: ${{ steps.decision.outputs.should_build }}
+    steps:
     - name: Build decision
       id: decision
       run: |
-        if [ "${{ steps.mlflow-base.outputs.manifest_found }}" == "true" ] && \
+        if [ "${{ github.event.inputs.skip_image_check }}" == "true" ]; then
-           [ "${{ steps.mlflow-full.outputs.manifest_found }}" == "true" ]; then
+          echo "Image check skipped, forcing build"
-          echo "✅ Both MLflow base and MLflow-full images found"
           echo "should_build=true" >> $GITHUB_OUTPUT
-        elif [ "${{ steps.mlflow-base.outputs.manifest_found }}" != "true" ]; then
+        elif [ "${{ needs.get-mlflow-digest.outputs.success }}" == "true" ] && \
+           [ "${{ needs.get-mlflow-full-base-image-digest.outputs.success }}" == "true" ]; then
+          echo "✅ Both MLflow base and MLflow-full images found"
+          if [ "${{ needs.get-mlflow-digest.outputs.digest }}" != "${{ needs.get-mlflow-full-base-image-digest.outputs.base_image_digest }}" ]; then
+            echo "Base image digest has changed"
+            echo "should_build=true" >> $GITHUB_OUTPUT
+          else
+            echo "Base image digest is unchanged"
+            echo "should_build=false" >> $GITHUB_OUTPUT
+          fi
+        elif [ "${{ needs.get-mlflow-base.outputs.success }}" != "true" ]; then
           echo "❌ MLflow base image not found"
           echo "should_build=false" >> $GITHUB_OUTPUT
         else
@@ -60,8 +106,10 @@ jobs:
           echo "should_build=false" >> $GITHUB_OUTPUT
         fi
   build-and-publish:
-    needs: check-images
+    if: needs.decide-to-build.outputs.should_build == 'true'
-    if: needs.check-images.outputs.should_build == 'true'
+    needs: 
+      - get-mlflow-digest
+      - decide-to-build
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repository
@@ -70,7 +118,7 @@ jobs:
       uses: docker/setup-buildx-action@v3
       with:
         config-inline: |
-          [registry."10.0.0.2:3000"]
+          [registry."${{ secrets.DOCKER_REGISTRY }}"]
             http = true
             insecure = true
     - name: Log in to Gitea Container Registry  
@@ -79,21 +127,28 @@ jobs:
         mkdir -p ~/.docker
         echo '{
           "auths": {},
-          "insecure-registries": ["10.0.0.2:3000"]
+          "insecure-registries": ["${{ secrets.DOCKER_REGISTRY }}"]
         }' > ~/.docker/config.json
         
         # Login using direct docker command
-        echo "${{ secrets.CI_RUNNER_TOKEN }}" | docker login 10.0.0.2:3000 --username "${{ gitea.actor }}" --password-stdin
+        echo "${{ secrets.CI_RUNNER_TOKEN }}" | docker login ${{ secrets.DOCKER_REGISTRY }} --username "${{ gitea.actor }}" --password-stdin
     - name: Extract metadata
       id: meta
       uses: docker/metadata-action@v5
       with:
-        images: 10.0.0.2:3000/${{ gitea.repository_owner }}/${{ gitea.event.repository.name }}
+        images: ${{ secrets.DOCKER_REGISTRY }}/${{ gitea.repository_owner }}/${{ gitea.event.repository.name }}
         tags: |
           type=schedule,pattern={{date 'YYYYMMDD'}}
           type=raw,value=latest
         labels: |
-          base-image.digest=${{ needs.check-images.outputs.base_image_digest }}
+          org.opencontainers.image.title=mlflow-full
+          org.opencontainers.image.description=MLflow with PostgreSQL support
+          org.opencontainers.image.authors=Brian Jensen
+          org.opencontainers.image.vendor=Brian Jensen
+          org.opencontainers.image.licenses=Apache-2.0
+          org.opencontainers.image.base.name=ghcr.io/mlflow/mlflow:latest
+          org.opencontainers.image.base.digest=${{ needs.get-mlflow-digest.outputs.digest }}
+          maintainer=Brian Jensen
     - name: Build and push Docker image
       id: build-publish
       uses: docker/build-push-action@v5
@@ -104,10 +159,10 @@ jobs:
         labels: ${{ steps.meta.outputs.labels }}
         cache-from: type=gha
         cache-to: type=gha,mode=max
-        build-args: |
-          BASE_IMAGE_DIGEST=${{ needs.check-images.outputs.base_image_digest }}
     - name: Update deployment info
       run: |
         echo "Docker image built and pushed successfully at $(date)"
         echo "Image tags:"
         echo "${{ steps.meta.outputs.tags }}"
+        echo "Image labels:"
+        echo "${{ steps.meta.outputs.labels }}"

Dockerfile

@@ -1,9 +1,5 @@
 FROM ghcr.io/mlflow/mlflow:latest
 
-# Set a labels for traceability
-LABEL build-date="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
-    maintainer="Brian Jensen"
-
 # Use --no-cache-dir to avoid storing pip cache in the image
 RUN pip install --no-cache-dir \
     psycopg2-binary

README.md

@@ -1,3 +1,155 @@
 # mlflow-full
 
-MLFlow docker container with support for Postgres. Implements extra environment variables for convenience.
+MLFlow docker container with support for Postgres. Implements extra environment variables for convenience.
+
+## Running the Container
+
+### Basic Usage
+
+```bash
+# Run with default settings
+docker run -p 5000:5000 gitea.gt-proj.com/brian/mlflow-full:latest
+```
+
+### With PostgreSQL Backend
+
+```bash
+# Using environment variables
+docker run -p 5000:5000 \
+  -e MLFLOW_BACKEND_STORE_URI="postgresql://user:password@host:5432/mlflow" \
+  -e MLFLOW_DEFAULT_ARTIFACT_ROOT="s3://my-bucket/mlflow-artifacts" \
+  gitea.gt-proj.com/brian/mlflow-full:latest
+```
+
+### Complete Example with Docker Compose
+
+```yaml
+version: '3.8'
+
+services:
+  postgres:
+    image: postgres:15
+    environment:
+      POSTGRES_DB: mlflow
+      POSTGRES_USER: mlflow
+      POSTGRES_PASSWORD: mlflow_password
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+
+  mlflow:
+    image: gitea.gt-proj.com/brian/mlflow-full:latest
+    ports:
+      - "5000:5000"
+    environment:
+      MLFLOW_BACKEND_STORE_URI: postgresql://mlflow:mlflow_password@postgres:5432/mlflow
+      MLFLOW_DEFAULT_ARTIFACT_ROOT: /mlflow/artifacts
+      MLFLOW_HOST: 0.0.0.0
+      MLFLOW_PORT: 5000
+    volumes:
+      - mlflow_artifacts:/mlflow/artifacts
+    depends_on:
+      - postgres
+
+volumes:
+  postgres_data:
+  mlflow_artifacts:
+```
+
+### Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `MLFLOW_BACKEND_STORE_URI` | Database connection string | `sqlite:///mlflow.db` |
+| `MLFLOW_DEFAULT_ARTIFACT_ROOT` | Artifact storage location | `./mlruns` |
+| `MLFLOW_HOST` | Host to bind the server | `0.0.0.0` |
+| `MLFLOW_PORT` | Port to bind the server | `5000` |
+
+### Advanced Examples
+
+#### With S3 Artifact Storage
+
+```bash
+docker run -p 5000:5000 \
+  -e MLFLOW_BACKEND_STORE_URI="postgresql://user:password@host:5432/mlflow" \
+  -e MLFLOW_DEFAULT_ARTIFACT_ROOT="s3://my-mlflow-bucket/artifacts" \
+  -e AWS_ACCESS_KEY_ID="your-access-key" \
+  -e AWS_SECRET_ACCESS_KEY="your-secret-key" \
+  -e AWS_DEFAULT_REGION="us-west-2" \
+  gitea.gt-proj.com/brian/mlflow-full:latest
+```
+
+#### With Custom Server Arguments
+
+```bash
+docker run -p 5000:5000 \
+  -e MLFLOW_BACKEND_STORE_URI="postgresql://user:password@host:5432/mlflow" \
+  gitea.gt-proj.com/brian/mlflow-full:latest \
+  --host 0.0.0.0 \
+  --port 5000 \
+  --workers 4 \
+  --expose-prometheus /metrics
+```
+
+### Kubernetes Deployment
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mlflow-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mlflow-server
+  template:
+    metadata:
+      labels:
+        app: mlflow-server
+    spec:
+      containers:
+      - name: mlflow-server
+        image: gitea.gt-proj.com/brian/mlflow-full:latest
+        ports:
+        - containerPort: 5000
+        env:
+        - name: MLFLOW_BACKEND_STORE_URI
+          value: "postgresql://mlflow:password@postgres-service:5432/mlflow"
+        - name: MLFLOW_DEFAULT_ARTIFACT_ROOT
+          value: "s3://mlflow-artifacts"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mlflow-service
+spec:
+  selector:
+    app: mlflow-server
+  ports:
+  - port: 5000
+    targetPort: 5000
+  type: LoadBalancer
+```
+
+## Features
+
+- 🐘 **PostgreSQL Support**: Pre-installed `psycopg2-binary` for PostgreSQL connectivity
+- 🏷️ **OCI Labels**: Fully compliant with OCI image specification
+- 🔄 **Auto-updates**: Daily builds when the base MLflow image is updated
+- 📦 **Lightweight**: Based on official MLflow container
+
+## Building Locally
+
+```bash
+# Clone the repository
+git clone https://gitea.gt-proj.com/brian/mlflow-full.git
+cd mlflow-full
+
+# Build the image
+docker build -t mlflow-full:local .
+
+# Run your local build
+docker run -p 5000:5000 mlflow-full:local
+```