Compare commits
5
Commits
a8548c6dd9
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
 brian
|
09d1e52298 | ||
 Brian Bjarke Jensen
|
a1c5e7c3a3 | ||
|
Brian Bjarke Jensen
|
07b5c11f43 | ||
|
Brian Bjarke Jensen
|
244692118b | ||
|
Brian Bjarke Jensen
|
b235d05df3 |
+32
-65
@@ -28,65 +28,32 @@ jobs:
|
||||
env:
|
||||
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||
run: |
|
||||
# Get a bearer token for authentication
|
||||
REGISTRY_TOKEN=$(curl -s -X GET \
|
||||
-u "${{ gitea.actor }}:${{ secrets.CI_RUNNER_TOKEN }}" \
|
||||
"https://gitea.gt-proj.com/v2/token?service=container_registry&scope=repository:brian/mlflow-full:pull" \
|
||||
| jq -r '.token // empty')
|
||||
if [ -z "$REGISTRY_TOKEN" ]; then
|
||||
# Configure Docker for insecure registry
|
||||
mkdir -p ~/.docker
|
||||
echo '{
|
||||
"auths": {},
|
||||
"insecure-registries": ["${{ secrets.DOCKER_REGISTRY }}"]
|
||||
}' > ~/.docker/config.json
|
||||
|
||||
# Login to internal registry
|
||||
echo "${{ secrets.CI_RUNNER_TOKEN }}" | docker login ${{ secrets.DOCKER_REGISTRY }} --username "${{ gitea.actor }}" --password-stdin
|
||||
|
||||
# Pull and inspect the image
|
||||
if docker pull ${{ secrets.DOCKER_REGISTRY }}/brian/mlflow-full:latest 2>/dev/null; then
|
||||
BASE_IMAGE_DIGEST=$(docker inspect ${{ secrets.DOCKER_REGISTRY }}/brian/mlflow-full:latest | jq -r '.[0].Config.Labels["org.opencontainers.image.base.digest"] // empty')
|
||||
|
||||
if [ -n "$BASE_IMAGE_DIGEST" ] && [ "$BASE_IMAGE_DIGEST" != "null" ]; then
|
||||
echo "base_image_digest=$BASE_IMAGE_DIGEST" >> $GITHUB_OUTPUT
|
||||
echo "success=true" >> $GITHUB_OUTPUT
|
||||
echo "Base image digest from MLflow-full image: $BASE_IMAGE_DIGEST"
|
||||
else
|
||||
echo "success=false" >> $GITHUB_OUTPUT
|
||||
echo "No base image digest label found"
|
||||
fi
|
||||
else
|
||||
echo "success=false" >> $GITHUB_OUTPUT
|
||||
echo "Failed to obtain registry token"
|
||||
exit 1
|
||||
echo "Failed to pull MLflow-full image - this might be the first run"
|
||||
fi
|
||||
# Inspect the manifest of the MLflow-full image
|
||||
MANIFEST=$(docker manifest inspect "gitea.gt-proj.com/brian/mlflow-full:latest" 2>/dev/null || echo "{}")
|
||||
if [ "$MANIFEST" == "{}" ]; then
|
||||
echo "success=false" >> $GITHUB_OUTPUT
|
||||
echo "MLflow-full manifest not found"
|
||||
exit 1
|
||||
fi
|
||||
# Extract digest for linux/amd64 platform
|
||||
DIGEST=$(echo "$MANIFEST" | jq -r '.manifests[]? | select(.platform.architecture == "amd64" and .platform.os == "linux") | .digest // empty')
|
||||
if [ -z "$DIGEST" ]; then
|
||||
echo "success=false" >> $GITHUB_OUTPUT
|
||||
echo "No suitable manifest found for linux/amd64"
|
||||
exit 1
|
||||
fi
|
||||
# Get the manifest for the specific platform
|
||||
PLATFORM_MANIFEST=$(curl -s -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
|
||||
-H "Authorization: Bearer $REGISTRY_TOKEN" \
|
||||
"https://gitea.gt-proj.com/v2/brian/mlflow-full/manifests/$DIGEST" 2>/dev/null || echo "{}")
|
||||
if [ "$PLATFORM_MANIFEST" == "{}" ]; then
|
||||
echo "success=false" >> $GITHUB_OUTPUT
|
||||
echo "Failed to fetch platform-specific manifest"
|
||||
exit 1
|
||||
fi
|
||||
# Extract config blob digest
|
||||
CONFIG_DIGEST=$(echo "$PLATFORM_MANIFEST" | jq -r '.config.digest // empty')
|
||||
if [ -z "$CONFIG_DIGEST" ]; then
|
||||
echo "success=false" >> $GITHUB_OUTPUT
|
||||
echo "No config digest found in platform manifest"
|
||||
exit 1
|
||||
fi
|
||||
# Fetch the configuration blob to get labels
|
||||
CONFIG_BLOB=$(curl -s \
|
||||
-H "Authorization: Bearer $REGISTRY_TOKEN" \
|
||||
"https://gitea.gt-proj.com/v2/brian/mlflow-full/blobs/$CONFIG_DIGEST" 2>/dev/null || echo "{}")
|
||||
if [ "$CONFIG_BLOB" == "{}" ]; then
|
||||
echo "success=false" >> $GITHUB_OUTPUT
|
||||
echo "Failed to fetch config blob"
|
||||
exit 1
|
||||
fi
|
||||
# Extract base image digest label from config
|
||||
BASE_IMAGE_DIGEST=$(echo "$CONFIG_BLOB" | jq -r '.config.Labels["base-image.digest"] // empty')
|
||||
if [ -z "$BASE_IMAGE_DIGEST" ]; then
|
||||
echo "success=false" >> $GITHUB_OUTPUT
|
||||
echo "No base-image.digest label found in MLflow-full image"
|
||||
exit 1
|
||||
fi
|
||||
echo "base_image_digest=$BASE_IMAGE_DIGEST" >> $GITHUB_OUTPUT
|
||||
echo "success=true" >> $GITHUB_OUTPUT
|
||||
echo "Base image digest from MLflow-full image: $BASE_IMAGE_DIGEST"
|
||||
get-mlflow-digest:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
@@ -121,17 +88,17 @@ jobs:
|
||||
if [ "${{ github.event.inputs.skip_image_check }}" == "true" ]; then
|
||||
echo "Image check skipped, forcing build"
|
||||
echo "should_build=true" >> $GITHUB_OUTPUT
|
||||
elif [ "${{ steps.mlflow-base.outputs.success }}" == "true" ] && \
|
||||
[ "${{ steps.mlflow-full.outputs.success }}" == "true" ]; then
|
||||
elif [ "${{ needs.get-mlflow-digest.outputs.success }}" == "true" ] && \
|
||||
[ "${{ needs.get-mlflow-full-base-image-digest.outputs.success }}" == "true" ]; then
|
||||
echo "✅ Both MLflow base and MLflow-full images found"
|
||||
if [ "${{ steps.mlflow-base.outputs.digest }}" != "${{ steps.mlflow-full.outputs.base_image_digest }}" ]; then
|
||||
if [ "${{ needs.get-mlflow-digest.outputs.digest }}" != "${{ needs.get-mlflow-full-base-image-digest.outputs.base_image_digest }}" ]; then
|
||||
echo "Base image digest has changed"
|
||||
echo "should_build=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "Base image digest is unchanged"
|
||||
echo "should_build=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
elif [ "${{ steps.mlflow-base.outputs.success }}" != "true" ]; then
|
||||
elif [ "${{ needs.get-mlflow-base.outputs.success }}" != "true" ]; then
|
||||
echo "❌ MLflow base image not found"
|
||||
echo "should_build=false" >> $GITHUB_OUTPUT
|
||||
else
|
||||
@@ -151,7 +118,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
config-inline: |
|
||||
[registry."10.0.0.2:3000"]
|
||||
[registry."${{ secrets.DOCKER_REGISTRY }}"]
|
||||
http = true
|
||||
insecure = true
|
||||
- name: Log in to Gitea Container Registry
|
||||
@@ -160,16 +127,16 @@ jobs:
|
||||
mkdir -p ~/.docker
|
||||
echo '{
|
||||
"auths": {},
|
||||
"insecure-registries": ["10.0.0.2:3000"]
|
||||
"insecure-registries": ["${{ secrets.DOCKER_REGISTRY }}"]
|
||||
}' > ~/.docker/config.json
|
||||
|
||||
# Login using direct docker command
|
||||
echo "${{ secrets.CI_RUNNER_TOKEN }}" | docker login 10.0.0.2:3000 --username "${{ gitea.actor }}" --password-stdin
|
||||
echo "${{ secrets.CI_RUNNER_TOKEN }}" | docker login ${{ secrets.DOCKER_REGISTRY }} --username "${{ gitea.actor }}" --password-stdin
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: 10.0.0.2:3000/${{ gitea.repository_owner }}/${{ gitea.event.repository.name }}
|
||||
images: ${{ secrets.DOCKER_REGISTRY }}/${{ gitea.repository_owner }}/${{ gitea.event.repository.name }}
|
||||
tags: |
|
||||
type=schedule,pattern={{date 'YYYYMMDD'}}
|
||||
type=raw,value=latest
|
||||
|
||||
@@ -1,3 +1,155 @@
|
||||
# mlflow-full
|
||||
|
||||
MLFlow docker container with support for Postgres. Implements extra environment variables for convenience.
|
||||
MLFlow docker container with support for Postgres. Implements extra environment variables for convenience.
|
||||
|
||||
## Running the Container
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```bash
|
||||
# Run with default settings
|
||||
docker run -p 5000:5000 gitea.gt-proj.com/brian/mlflow-full:latest
|
||||
```
|
||||
|
||||
### With PostgreSQL Backend
|
||||
|
||||
```bash
|
||||
# Using environment variables
|
||||
docker run -p 5000:5000 \
|
||||
-e MLFLOW_BACKEND_STORE_URI="postgresql://user:password@host:5432/mlflow" \
|
||||
-e MLFLOW_DEFAULT_ARTIFACT_ROOT="s3://my-bucket/mlflow-artifacts" \
|
||||
gitea.gt-proj.com/brian/mlflow-full:latest
|
||||
```
|
||||
|
||||
### Complete Example with Docker Compose
|
||||
|
||||
```yaml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:15
|
||||
environment:
|
||||
POSTGRES_DB: mlflow
|
||||
POSTGRES_USER: mlflow
|
||||
POSTGRES_PASSWORD: mlflow_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
ports:
|
||||
- "5432:5432"
|
||||
|
||||
mlflow:
|
||||
image: gitea.gt-proj.com/brian/mlflow-full:latest
|
||||
ports:
|
||||
- "5000:5000"
|
||||
environment:
|
||||
MLFLOW_BACKEND_STORE_URI: postgresql://mlflow:mlflow_password@postgres:5432/mlflow
|
||||
MLFLOW_DEFAULT_ARTIFACT_ROOT: /mlflow/artifacts
|
||||
MLFLOW_HOST: 0.0.0.0
|
||||
MLFLOW_PORT: 5000
|
||||
volumes:
|
||||
- mlflow_artifacts:/mlflow/artifacts
|
||||
depends_on:
|
||||
- postgres
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
mlflow_artifacts:
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `MLFLOW_BACKEND_STORE_URI` | Database connection string | `sqlite:///mlflow.db` |
|
||||
| `MLFLOW_DEFAULT_ARTIFACT_ROOT` | Artifact storage location | `./mlruns` |
|
||||
| `MLFLOW_HOST` | Host to bind the server | `0.0.0.0` |
|
||||
| `MLFLOW_PORT` | Port to bind the server | `5000` |
|
||||
|
||||
### Advanced Examples
|
||||
|
||||
#### With S3 Artifact Storage
|
||||
|
||||
```bash
|
||||
docker run -p 5000:5000 \
|
||||
-e MLFLOW_BACKEND_STORE_URI="postgresql://user:password@host:5432/mlflow" \
|
||||
-e MLFLOW_DEFAULT_ARTIFACT_ROOT="s3://my-mlflow-bucket/artifacts" \
|
||||
-e AWS_ACCESS_KEY_ID="your-access-key" \
|
||||
-e AWS_SECRET_ACCESS_KEY="your-secret-key" \
|
||||
-e AWS_DEFAULT_REGION="us-west-2" \
|
||||
gitea.gt-proj.com/brian/mlflow-full:latest
|
||||
```
|
||||
|
||||
#### With Custom Server Arguments
|
||||
|
||||
```bash
|
||||
docker run -p 5000:5000 \
|
||||
-e MLFLOW_BACKEND_STORE_URI="postgresql://user:password@host:5432/mlflow" \
|
||||
gitea.gt-proj.com/brian/mlflow-full:latest \
|
||||
--host 0.0.0.0 \
|
||||
--port 5000 \
|
||||
--workers 4 \
|
||||
--expose-prometheus /metrics
|
||||
```
|
||||
|
||||
### Kubernetes Deployment
|
||||
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: mlflow-server
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: mlflow-server
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: mlflow-server
|
||||
spec:
|
||||
containers:
|
||||
- name: mlflow-server
|
||||
image: gitea.gt-proj.com/brian/mlflow-full:latest
|
||||
ports:
|
||||
- containerPort: 5000
|
||||
env:
|
||||
- name: MLFLOW_BACKEND_STORE_URI
|
||||
value: "postgresql://mlflow:password@postgres-service:5432/mlflow"
|
||||
- name: MLFLOW_DEFAULT_ARTIFACT_ROOT
|
||||
value: "s3://mlflow-artifacts"
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: mlflow-service
|
||||
spec:
|
||||
selector:
|
||||
app: mlflow-server
|
||||
ports:
|
||||
- port: 5000
|
||||
targetPort: 5000
|
||||
type: LoadBalancer
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
- 🐘 **PostgreSQL Support**: Pre-installed `psycopg2-binary` for PostgreSQL connectivity
|
||||
- 🏷️ **OCI Labels**: Fully compliant with OCI image specification
|
||||
- 🔄 **Auto-updates**: Daily builds when the base MLflow image is updated
|
||||
- 📦 **Lightweight**: Based on official MLflow container
|
||||
|
||||
## Building Locally
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone https://gitea.gt-proj.com/brian/mlflow-full.git
|
||||
cd mlflow-full
|
||||
|
||||
# Build the image
|
||||
docker build -t mlflow-full:local .
|
||||
|
||||
# Run your local build
|
||||
docker run -p 5000:5000 mlflow-full:local
|
||||
```
|
||||
|
||||
Reference in New Issue
Block a user