diff --git a/.env.example b/.env.example index 24e63d3..58a769a 100644 --- a/.env.example +++ b/.env.example @@ -27,6 +27,14 @@ MAX_UPLOAD_SIZE_MB=500 # Scale horizontally with: docker compose up --scale worker=N CELERY_WORKER_CONCURRENCY=8 +# MinIO (S3-compatible object storage for shared uploads) +# MinIO is started automatically via docker-compose.yml +MINIO_URL=http://minio:9000 +MINIO_USER=minioadmin +MINIO_PASSWORD=minioadmin +MINIO_BUCKET=uploads +# MinIO console UI: http://localhost:9001 (user: minioadmin, password: minioadmin) + # Blender (render-worker) # Blender >= 5.0.1 must be installed on the host at /opt/blender # The render-worker container mounts it read-only via volumes: - /opt/blender:/opt/blender:ro diff --git a/.gitignore b/.gitignore index ef978ea..a72732a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,8 @@ node_modules/ .env.local .DS_Store *.log -core +# core dump files (not directories named 'core') +/core /blender-renderer/core # Python cache diff --git a/backend/app/core/__init__.py b/backend/app/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/core/storage.py b/backend/app/core/storage.py new file mode 100644 index 0000000..f739b9a --- /dev/null +++ b/backend/app/core/storage.py @@ -0,0 +1,171 @@ +"""Storage abstraction — MinIO (production) or LocalStorage (dev fallback). + +Usage: + from app.core.storage import get_storage + storage = get_storage() + key = storage.upload(local_path, "uploads/step_files/my.step") + local_path = storage.download("uploads/step_files/my.step", tmp_dir / "my.step") + +Environment variables (set in docker-compose.yml): + MINIO_URL — S3 endpoint URL (e.g. http://minio:9000) + MINIO_USER — Access key (default: minioadmin) + MINIO_PASSWORD — Secret key (default: minioadmin) + MINIO_BUCKET — Bucket name (default: uploads) + +If MINIO_URL is not set, falls back to LocalStorage (reads/writes to UPLOAD_DIR). +""" +import logging +import os +from pathlib import Path + +logger = logging.getLogger(__name__) + + +class MinIOStorage: + """S3-compatible object storage via boto3 (MinIO backend).""" + + def __init__( + self, + url: str, + user: str, + password: str, + bucket: str = "uploads", + ): + import boto3 + from botocore.config import Config + + self._bucket = bucket + self._client = boto3.client( + "s3", + endpoint_url=url, + aws_access_key_id=user, + aws_secret_access_key=password, + config=Config(signature_version="s3v4"), + ) + self._ensure_bucket() + + def _ensure_bucket(self): + """Create the bucket if it does not exist.""" + try: + self._client.head_bucket(Bucket=self._bucket) + except Exception: + try: + self._client.create_bucket(Bucket=self._bucket) + logger.info("Created MinIO bucket: %s", self._bucket) + except Exception as exc: + logger.warning("Could not create MinIO bucket %s: %s", self._bucket, exc) + + def upload(self, local_path: Path, object_key: str) -> str: + """Upload a local file to MinIO. Returns the object_key.""" + self._client.upload_file(str(local_path), self._bucket, object_key) + logger.debug("Uploaded %s → minio://%s/%s", local_path.name, self._bucket, object_key) + return object_key + + def download(self, object_key: str, local_path: Path) -> Path: + """Download object from MinIO to local_path. Returns local_path.""" + local_path.parent.mkdir(parents=True, exist_ok=True) + self._client.download_file(self._bucket, object_key, str(local_path)) + logger.debug("Downloaded minio://%s/%s → %s", self._bucket, object_key, local_path.name) + return local_path + + def exists(self, object_key: str) -> bool: + """Return True if the object exists in MinIO.""" + try: + self._client.head_object(Bucket=self._bucket, Key=object_key) + return True + except Exception: + return False + + def delete(self, object_key: str) -> None: + """Delete an object from MinIO (best-effort).""" + try: + self._client.delete_object(Bucket=self._bucket, Key=object_key) + except Exception as exc: + logger.warning("MinIO delete failed for %s: %s", object_key, exc) + + def get_url(self, object_key: str, expires_in: int = 3600) -> str: + """Generate a presigned URL for direct download (valid for expires_in seconds).""" + return self._client.generate_presigned_url( + "get_object", + Params={"Bucket": self._bucket, "Key": object_key}, + ExpiresIn=expires_in, + ) + + @property + def backend(self) -> str: + return "minio" + + +class LocalStorage: + """Fallback: reads/writes to the local UPLOAD_DIR filesystem. + + Object keys are treated as relative paths under UPLOAD_DIR. + This preserves backward compatibility with the existing uploads/ volume. + """ + + def __init__(self, upload_dir: str): + self._root = Path(upload_dir) + + def _resolve(self, object_key: str) -> Path: + return self._root / object_key + + def upload(self, local_path: Path, object_key: str) -> str: + """Copy local_path to the local storage path for object_key.""" + import shutil + dest = self._resolve(object_key) + dest.parent.mkdir(parents=True, exist_ok=True) + if str(local_path) != str(dest): + shutil.copy2(str(local_path), str(dest)) + return object_key + + def download(self, object_key: str, local_path: Path) -> Path: + """'Download' by returning the existing local path.""" + src = self._resolve(object_key) + if src == local_path: + return local_path + import shutil + local_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(str(src), str(local_path)) + return local_path + + def exists(self, object_key: str) -> bool: + return self._resolve(object_key).exists() + + def delete(self, object_key: str) -> None: + path = self._resolve(object_key) + path.unlink(missing_ok=True) + + def get_url(self, object_key: str, expires_in: int = 3600) -> str: + return f"/api/files/{object_key}" + + @property + def backend(self) -> str: + return "local" + + +_storage_instance: MinIOStorage | LocalStorage | None = None + + +def get_storage() -> MinIOStorage | LocalStorage: + """Return the configured storage backend (singleton).""" + global _storage_instance + if _storage_instance is not None: + return _storage_instance + + minio_url = os.environ.get("MINIO_URL", "") + if minio_url: + user = os.environ.get("MINIO_USER", "minioadmin") + password = os.environ.get("MINIO_PASSWORD", "minioadmin") + bucket = os.environ.get("MINIO_BUCKET", "uploads") + try: + _storage_instance = MinIOStorage(url=minio_url, user=user, password=password, bucket=bucket) + logger.info("Storage backend: MinIO (%s, bucket=%s)", minio_url, bucket) + except Exception as exc: + logger.warning("MinIO init failed (%s) — falling back to LocalStorage: %s", minio_url, exc) + _storage_instance = LocalStorage(os.environ.get("UPLOAD_DIR", "/app/uploads")) + else: + upload_dir = os.environ.get("UPLOAD_DIR", "/app/uploads") + _storage_instance = LocalStorage(upload_dir) + logger.info("Storage backend: local filesystem (%s)", upload_dir) + + return _storage_instance diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 2609a2f..19e3890 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "httpx>=0.27.0", "python-dotenv>=1.0.1", "aiofiles>=23.2.1", - "docker>=6.1.0", + "boto3>=1.34.0", ] [project.optional-dependencies] diff --git a/docker-compose.worker.yml b/docker-compose.worker.yml new file mode 100644 index 0000000..d5723cc --- /dev/null +++ b/docker-compose.worker.yml @@ -0,0 +1,49 @@ +# External render-worker configuration. +# +# Use this compose file on remote machines (GPU nodes, NAS, cloud VMs) that +# only need Redis + MinIO access — no database, no backend, no frontend. +# +# Usage on remote machine: +# export REDIS_URL=redis://main-server:6379/0 +# export MINIO_URL=http://main-server:9000 +# export MINIO_USER=minioadmin +# export MINIO_PASSWORD=minioadmin +# docker compose -f docker-compose.worker.yml up -d +# +# Requirements on remote machine: +# - Blender >= 5.0.1 installed at /opt/blender (or set BLENDER_BIN) +# - Docker + GPU drivers (NVIDIA recommended) + +services: + render-worker: + image: schaefflerautomat-render-worker:latest + # Or build locally: build: { context: ./render-worker, dockerfile: Dockerfile } + environment: + - REDIS_URL=${REDIS_URL:?Set REDIS_URL to the main server Redis URL} + - POSTGRES_HOST=${POSTGRES_HOST:?Set POSTGRES_HOST to the main server DB host} + - POSTGRES_DB=${POSTGRES_DB:-schaeffler} + - POSTGRES_USER=${POSTGRES_USER:-schaeffler} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + - POSTGRES_PORT=${POSTGRES_PORT:-5432} + - MINIO_URL=${MINIO_URL:?Set MINIO_URL to the main server MinIO URL} + - MINIO_USER=${MINIO_USER:-minioadmin} + - MINIO_PASSWORD=${MINIO_PASSWORD:-minioadmin} + - MINIO_BUCKET=${MINIO_BUCKET:-uploads} + - UPLOAD_DIR=/tmp/render-cache + - BLENDER_BIN=${BLENDER_BIN:-/opt/blender/blender} + - RENDER_SCRIPTS_DIR=/render-scripts + - JWT_SECRET_KEY=${JWT_SECRET_KEY:?Set JWT_SECRET_KEY} + volumes: + - /opt/blender:/opt/blender:ro + - render-cache:/tmp/render-cache + deploy: + replicas: ${RENDER_WORKER_REPLICAS:-1} + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu, compute, utility, graphics] + +volumes: + render-cache: diff --git a/docker-compose.yml b/docker-compose.yml index 0419272..f1fa241 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,23 @@ services: timeout: 5s retries: 5 + minio: + image: minio/minio:latest + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${MINIO_USER:-minioadmin} + MINIO_ROOT_PASSWORD: ${MINIO_PASSWORD:-minioadmin} + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio-data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 5s + retries: 5 + backend: build: context: ./backend @@ -46,6 +63,10 @@ services: - AZURE_OPENAI_API_VERSION=${AZURE_OPENAI_API_VERSION:-2024-02-01} - UPLOAD_DIR=/app/uploads - MAX_UPLOAD_SIZE_MB=${MAX_UPLOAD_SIZE_MB:-500} + - MINIO_URL=${MINIO_URL:-http://minio:9000} + - MINIO_USER=${MINIO_USER:-minioadmin} + - MINIO_PASSWORD=${MINIO_PASSWORD:-minioadmin} + - MINIO_BUCKET=${MINIO_BUCKET:-uploads} volumes: - ./backend:/app - uploads:/app/uploads @@ -56,6 +77,8 @@ services: condition: service_healthy redis: condition: service_healthy + minio: + condition: service_healthy worker: build: @@ -75,6 +98,10 @@ services: - AZURE_OPENAI_DEPLOYMENT=${AZURE_OPENAI_DEPLOYMENT:-gpt-4o} - AZURE_OPENAI_API_VERSION=${AZURE_OPENAI_API_VERSION:-2024-02-01} - UPLOAD_DIR=/app/uploads + - MINIO_URL=${MINIO_URL:-http://minio:9000} + - MINIO_USER=${MINIO_USER:-minioadmin} + - MINIO_PASSWORD=${MINIO_PASSWORD:-minioadmin} + - MINIO_BUCKET=${MINIO_BUCKET:-uploads} - CELERY_WORKER_CONCURRENCY=${CELERY_WORKER_CONCURRENCY:-8} volumes: - ./backend:/app @@ -102,6 +129,10 @@ services: - UPLOAD_DIR=/app/uploads - BLENDER_BIN=/opt/blender/blender - RENDER_SCRIPTS_DIR=/render-scripts + - MINIO_URL=${MINIO_URL:-http://minio:9000} + - MINIO_USER=${MINIO_USER:-minioadmin} + - MINIO_PASSWORD=${MINIO_PASSWORD:-minioadmin} + - MINIO_BUCKET=${MINIO_BUCKET:-uploads} volumes: - ./backend:/app - uploads:/app/uploads @@ -163,3 +194,4 @@ services: volumes: pgdata: uploads: + minio-data: