Merge pull request 'Update CI to work with new network architecture' (#3) from update-CI-to-new-network-architecture into main

Reviewed-on: #3

.gitea/workflows/update.yml

@@ -5,37 +5,152 @@ on:
     # Runs every day at 3:00 AM UTC
     - cron: '0 3 * * *'
   workflow_dispatch: # Allows manual triggering
+    inputs:
+      skip_image_check:
+        description: 'Skip image check and force build'
+        required: false
+        default: 'false'
+        type: choice
+        options:
+        - 'true'
+        - 'false'
 
 jobs:
+  get-mlflow-full-base-image-digest:
+    if: github.event.inputs.skip_image_check != 'true'
+    runs-on: ubuntu-latest
+    outputs:
+      base_image_digest: ${{ steps.mlflow-full.outputs.base_image_digest }}
+      success: ${{ steps.mlflow-full.outputs.success }}
+    steps:
+    - name: Get MLflow-full base image digest
+      id: mlflow-full
+      env:
+        CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
+      run: |
+        # Configure Docker for insecure registry
+        mkdir -p ~/.docker
+        echo '{
+          "auths": {},
+          "insecure-registries": ["${{ secrets.DOCKER_REGISTRY }}"]
+        }' > ~/.docker/config.json
+        
+        # Login to internal registry
+        echo "${{ secrets.CI_RUNNER_TOKEN }}" | docker login ${{ secrets.DOCKER_REGISTRY }} --username "${{ gitea.actor }}" --password-stdin
+
+        # Pull and inspect the image
+        if docker pull ${{ secrets.DOCKER_REGISTRY }}/brian/mlflow-full:latest 2>/dev/null; then
+          BASE_IMAGE_DIGEST=$(docker inspect ${{ secrets.DOCKER_REGISTRY }}/brian/mlflow-full:latest | jq -r '.[0].Config.Labels["org.opencontainers.image.base.digest"] // empty')
+          
+          if [ -n "$BASE_IMAGE_DIGEST" ] && [ "$BASE_IMAGE_DIGEST" != "null" ]; then
+            echo "base_image_digest=$BASE_IMAGE_DIGEST" >> $GITHUB_OUTPUT
+            echo "success=true" >> $GITHUB_OUTPUT
+            echo "Base image digest from MLflow-full image: $BASE_IMAGE_DIGEST"
+          else
+            echo "success=false" >> $GITHUB_OUTPUT
+            echo "No base image digest label found"
+          fi
+        else
+          echo "success=false" >> $GITHUB_OUTPUT
+          echo "Failed to pull MLflow-full image - this might be the first run"
+        fi
+  get-mlflow-digest:
+    runs-on: ubuntu-latest
+    outputs:
+      digest: ${{ steps.mlflow-base.outputs.digest }}
+      success: ${{ steps.mlflow-base.outputs.success }}
+    steps:
+    - name: Get MLflow base image digest
+      id: mlflow-base
+      run: |
+        MANIFEST=$(docker manifest inspect ghcr.io/mlflow/mlflow:latest 2>/dev/null || echo "{}")
+        if [ "$MANIFEST" != "{}" ]; then
+          DIGEST=$(echo "$MANIFEST" | jq -r '.manifests[]? | select(.platform.architecture == "amd64" and .platform.os == "linux") | .digest // empty')
+          echo "digest=$DIGEST" >> $GITHUB_OUTPUT
+          echo "success=true" >> $GITHUB_OUTPUT
+          echo "Digest found for MLflow base image: $DIGEST"
+        else
+          echo "success=false" >> $GITHUB_OUTPUT
+          echo "MLflow base image not found"
+        fi
+  decide-to-build:
+    if: always()  # Run regardless of dependency status
+    needs:
+      - get-mlflow-full-base-image-digest
+      - get-mlflow-digest
+    runs-on: ubuntu-latest
+    outputs:
+      should_build: ${{ steps.decision.outputs.should_build }}
+    steps:
+    - name: Build decision
+      id: decision
+      run: |
+        if [ "${{ github.event.inputs.skip_image_check }}" == "true" ]; then
+          echo "Image check skipped, forcing build"
+          echo "should_build=true" >> $GITHUB_OUTPUT
+        elif [ "${{ needs.get-mlflow-digest.outputs.success }}" == "true" ] && \
+           [ "${{ needs.get-mlflow-full-base-image-digest.outputs.success }}" == "true" ]; then
+          echo "✅ Both MLflow base and MLflow-full images found"
+          if [ "${{ needs.get-mlflow-digest.outputs.digest }}" != "${{ needs.get-mlflow-full-base-image-digest.outputs.base_image_digest }}" ]; then
+            echo "Base image digest has changed"
+            echo "should_build=true" >> $GITHUB_OUTPUT
+          else
+            echo "Base image digest is unchanged"
+            echo "should_build=false" >> $GITHUB_OUTPUT
+          fi
+        elif [ "${{ needs.get-mlflow-base.outputs.success }}" != "true" ]; then
+          echo "❌ MLflow base image not found"
+          echo "should_build=false" >> $GITHUB_OUTPUT
+        else
+          echo "❌ MLflow-full image not found"
+          echo "should_build=false" >> $GITHUB_OUTPUT
+        fi
   build-and-publish:
+    if: needs.decide-to-build.outputs.should_build == 'true'
+    needs: 
+      - get-mlflow-digest
+      - decide-to-build
     runs-on: ubuntu-latest
-    
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
-      
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
-      
-    - name: Log in to Gitea Container Registry
-      uses: docker/login-action@v3
       with:
-        registry: gitea.gt-proj.com
+        config-inline: |
-        username: ${{ gitea.actor }}
+          [registry."${{ secrets.DOCKER_REGISTRY }}"]
-        password: ${{ secrets.CI_RUNNER_TOKEN }}
+            http = true
+            insecure = true
+    - name: Log in to Gitea Container Registry  
+      run: |
+        # Configure Docker to allow insecure registry for this session
+        mkdir -p ~/.docker
+        echo '{
+          "auths": {},
+          "insecure-registries": ["${{ secrets.DOCKER_REGISTRY }}"]
+        }' > ~/.docker/config.json
         
+        # Login using direct docker command
+        echo "${{ secrets.CI_RUNNER_TOKEN }}" | docker login ${{ secrets.DOCKER_REGISTRY }} --username "${{ gitea.actor }}" --password-stdin
     - name: Extract metadata
       id: meta
       uses: docker/metadata-action@v5
       with:
-        images: gitea.gt-proj.com/${{ gitea.repository_owner }}/${{ gitea.event.repository.name }}
+        images: ${{ secrets.DOCKER_REGISTRY }}/${{ gitea.repository_owner }}/${{ gitea.event.repository.name }}
         tags: |
-          type=ref,event=branch
-          type=ref,event=pr
           type=schedule,pattern={{date 'YYYYMMDD'}}
-          type=raw,value=latest,enable={{is_default_branch}}
+          type=raw,value=latest
-          
+        labels: |
+          org.opencontainers.image.title=mlflow-full
+          org.opencontainers.image.description=MLflow with PostgreSQL support
+          org.opencontainers.image.authors=Brian Jensen
+          org.opencontainers.image.vendor=Brian Jensen
+          org.opencontainers.image.licenses=Apache-2.0
+          org.opencontainers.image.base.name=ghcr.io/mlflow/mlflow:latest
+          org.opencontainers.image.base.digest=${{ needs.get-mlflow-digest.outputs.digest }}
+          maintainer=Brian Jensen
     - name: Build and push Docker image
+      id: build-publish
       uses: docker/build-push-action@v5
       with:
         context: .
@@ -44,8 +159,10 @@ jobs:
         labels: ${{ steps.meta.outputs.labels }}
         cache-from: type=gha
         cache-to: type=gha,mode=max
-        
     - name: Update deployment info
       run: |
         echo "Docker image built and pushed successfully at $(date)"
-        echo "Image tags: ${{ steps.meta.outputs.tags }}"
+        echo "Image tags:"
+        echo "${{ steps.meta.outputs.tags }}"
+        echo "Image labels:"
+        echo "${{ steps.meta.outputs.labels }}"

Dockerfile

@@ -1,5 +1,7 @@
 FROM ghcr.io/mlflow/mlflow:latest
 
-RUN pip install psycopg2-binary
+# Use --no-cache-dir to avoid storing pip cache in the image
+RUN pip install --no-cache-dir \
+    psycopg2-binary
 
 ENTRYPOINT [ "mlflow", "server" ]

README.md

@@ -1,3 +1,155 @@
 # mlflow-full
 
-MLFlow docker container with support for Postgres. Implements extra environment variables for convenience.
+MLFlow docker container with support for Postgres. Implements extra environment variables for convenience.
+
+## Running the Container
+
+### Basic Usage
+
+```bash
+# Run with default settings
+docker run -p 5000:5000 gitea.gt-proj.com/brian/mlflow-full:latest
+```
+
+### With PostgreSQL Backend
+
+```bash
+# Using environment variables
+docker run -p 5000:5000 \
+  -e MLFLOW_BACKEND_STORE_URI="postgresql://user:password@host:5432/mlflow" \
+  -e MLFLOW_DEFAULT_ARTIFACT_ROOT="s3://my-bucket/mlflow-artifacts" \
+  gitea.gt-proj.com/brian/mlflow-full:latest
+```
+
+### Complete Example with Docker Compose
+
+```yaml
+version: '3.8'
+
+services:
+  postgres:
+    image: postgres:15
+    environment:
+      POSTGRES_DB: mlflow
+      POSTGRES_USER: mlflow
+      POSTGRES_PASSWORD: mlflow_password
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+
+  mlflow:
+    image: gitea.gt-proj.com/brian/mlflow-full:latest
+    ports:
+      - "5000:5000"
+    environment:
+      MLFLOW_BACKEND_STORE_URI: postgresql://mlflow:mlflow_password@postgres:5432/mlflow
+      MLFLOW_DEFAULT_ARTIFACT_ROOT: /mlflow/artifacts
+      MLFLOW_HOST: 0.0.0.0
+      MLFLOW_PORT: 5000
+    volumes:
+      - mlflow_artifacts:/mlflow/artifacts
+    depends_on:
+      - postgres
+
+volumes:
+  postgres_data:
+  mlflow_artifacts:
+```
+
+### Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `MLFLOW_BACKEND_STORE_URI` | Database connection string | `sqlite:///mlflow.db` |
+| `MLFLOW_DEFAULT_ARTIFACT_ROOT` | Artifact storage location | `./mlruns` |
+| `MLFLOW_HOST` | Host to bind the server | `0.0.0.0` |
+| `MLFLOW_PORT` | Port to bind the server | `5000` |
+
+### Advanced Examples
+
+#### With S3 Artifact Storage
+
+```bash
+docker run -p 5000:5000 \
+  -e MLFLOW_BACKEND_STORE_URI="postgresql://user:password@host:5432/mlflow" \
+  -e MLFLOW_DEFAULT_ARTIFACT_ROOT="s3://my-mlflow-bucket/artifacts" \
+  -e AWS_ACCESS_KEY_ID="your-access-key" \
+  -e AWS_SECRET_ACCESS_KEY="your-secret-key" \
+  -e AWS_DEFAULT_REGION="us-west-2" \
+  gitea.gt-proj.com/brian/mlflow-full:latest
+```
+
+#### With Custom Server Arguments
+
+```bash
+docker run -p 5000:5000 \
+  -e MLFLOW_BACKEND_STORE_URI="postgresql://user:password@host:5432/mlflow" \
+  gitea.gt-proj.com/brian/mlflow-full:latest \
+  --host 0.0.0.0 \
+  --port 5000 \
+  --workers 4 \
+  --expose-prometheus /metrics
+```
+
+### Kubernetes Deployment
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mlflow-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mlflow-server
+  template:
+    metadata:
+      labels:
+        app: mlflow-server
+    spec:
+      containers:
+      - name: mlflow-server
+        image: gitea.gt-proj.com/brian/mlflow-full:latest
+        ports:
+        - containerPort: 5000
+        env:
+        - name: MLFLOW_BACKEND_STORE_URI
+          value: "postgresql://mlflow:password@postgres-service:5432/mlflow"
+        - name: MLFLOW_DEFAULT_ARTIFACT_ROOT
+          value: "s3://mlflow-artifacts"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mlflow-service
+spec:
+  selector:
+    app: mlflow-server
+  ports:
+  - port: 5000
+    targetPort: 5000
+  type: LoadBalancer
+```
+
+## Features
+
+- 🐘 **PostgreSQL Support**: Pre-installed `psycopg2-binary` for PostgreSQL connectivity
+- 🏷️ **OCI Labels**: Fully compliant with OCI image specification
+- 🔄 **Auto-updates**: Daily builds when the base MLflow image is updated
+- 📦 **Lightweight**: Based on official MLflow container
+
+## Building Locally
+
+```bash
+# Clone the repository
+git clone https://gitea.gt-proj.com/brian/mlflow-full.git
+cd mlflow-full
+
+# Build the image
+docker build -t mlflow-full:local .
+
+# Run your local build
+docker run -p 5000:5000 mlflow-full:local
+```