Dockerise our ModelScan application

.github/workflows/modelscan.yml

@@ -0,0 +1,33 @@
+name: Modelscan Python REST API Unit Test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.12']
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+
+          cd lib/modelscan_api
+          python -m pip install -r requirements-dev.txt
+
+      # Pytest
+      - name: Run basic testing
+        run: |
+          cd lib/modelscan_api
+          python -m pytest

docker-compose-prod.yml

@@ -33,6 +33,19 @@ services:
       timeout: 10s
       retries: 5
 
+  modelscan:
+    image: bailo_modelscan:latest
+    build:
+      context: ./lib/modelscan_api
+      dockerfile: ./Dockerfile
+    ports:
+      - 3311:80
+    healthcheck:
+      test: ['CMD-SHELL', 'curl --fail http://localhost:80/info || exit 1']
+      interval: 30s
+      timeout: 10s
+      retries: 5
+
   mailcrab:
     image: marlonb/mailcrab:v1.2.0
     ports:
@@ -94,6 +107,8 @@ services:
     depends_on:
       clamd:
         condition: service_healthy
+      modelscan:
+        condition: service_healthy
       minio:
         condition: service_started
       mongo:

docker-compose.yml

@@ -35,6 +35,20 @@ services:
       timeout: 10s
       retries: 5
 
+  modelscan:
+    build:
+      context: ./lib/modelscan_api
+      dockerfile: ./Dockerfile.dev
+    volumes:
+      - ./lib/modelscan_api/bailo_modelscan_api:/app/bailo_modelscan_api
+    ports:
+      - 3311:80
+    healthcheck:
+      test: ['CMD-SHELL', 'curl --fail http://localhost:80/info || exit 1']
+      interval: 30s
+      timeout: 10s
+      retries: 5
+
   mailcrab:
     image: marlonb/mailcrab:v1.2.0
     ports:
@@ -100,6 +114,8 @@ services:
     depends_on:
       clamd:
         condition: service_healthy
+      modelscan:
+        condition: service_healthy
       minio:
         condition: service_started
       mongo:

lib/modelscan_api/.dockerignore

@@ -0,0 +1,26 @@
+# Git
+.git
+.gitignore
+
+# Docker
+Dockerfile
+.dockerignore
+
+# Byte-compiled / optimized / DLL files
+**__pycache__/
+**.py[cod]
+**$py.class
+
+# Environments
+.env
+*env/
+*ENV/
+*env.bak/
+
+# Test files
+.pytest_cache/
+
+# Misc
+README.md
+.pre-commit-config.yaml
+requirements-dev.txt

lib/modelscan_api/.gitignore

@@ -8,3 +8,6 @@ __pycache__/
 *env/
 *ENV/
 *env.bak/
+
+# Test files
+.pytest_cache/

lib/modelscan_api/.pre-commit-config.yaml

@@ -2,6 +2,8 @@ ci:
   autoupdate_commit_msg: 'chore: update pre-commit hooks'
   autofix_commit_msg: 'style: pre-commit fixes'
 
+files: ^lib/modelscan_api/
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0

lib/modelscan_api/Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.12-slim-bullseye
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+# Keeps Python from buffering stdout and stderr to avoid situations where the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+# curl is used by the health check
+RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
+
+COPY ./requirements.txt /app/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
+
+COPY ./bailo_modelscan_api /app/bailo_modelscan_api
+
+EXPOSE 80
+
+CMD ["fastapi", "run", "bailo_modelscan_api/main.py", "--port", "80"]

lib/modelscan_api/Dockerfile.dev

@@ -0,0 +1,19 @@
+FROM python:3.12-slim-bullseye
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+# Keeps Python from buffering stdout and stderr to avoid situations where the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+# curl is used by the health check
+RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
+
+COPY ./requirements.txt /app/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
+
+EXPOSE 80
+
+CMD ["fastapi", "dev", "bailo_modelscan_api/main.py", "--port", "80", "--host", "0.0.0.0"]

lib/modelscan_api/README.md

@@ -11,6 +11,8 @@ This directory provides all of the necessary functionality to interact with
 
 ## Setup
 
+Note that **Python 3.9 to 3.12** is required.
+
 Create and activate a virtual environment
 
 ```bash

lib/modelscan_api/bailo_modelscan_api/config.py

@@ -4,7 +4,7 @@
 from __future__ import annotations
 
 import logging
-from typing import Any
+from typing import Any, Optional
 
 from modelscan.settings import DEFAULT_SETTINGS
 from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -25,13 +25,10 @@ class Settings(BaseSettings):
 
     You can upload files and view modelscan's result."""
     app_version: str = "1.0.0"
-    download_dir: str = "."
+    # download_dir is used if it evaluates, otherwise a temporary directory is used.
+    download_dir: Optional[str] = None
     modelscan_settings: dict[str, Any] = DEFAULT_SETTINGS
     block_size: int = 1024
 
     # Load in a dotenv file to set/overwrite any properties with potentially sensitive values
     model_config = SettingsConfigDict(env_file=".env")
-
-
-logger.info("Instantiating settings.")
-settings = Settings()

lib/modelscan_api/bailo_modelscan_api/dependencies.py

@@ -15,7 +15,32 @@ def parse_path(path: str | Path | None) -> Path:
     :param path: System path to parse. Defaults to the file's current working directory if unspecified.
     :return: An absolute Path representation of the path parameter.
     """
-    logger.info("Parsing path.")
+    logger.debug("Parsing path %s", path)
     if path is None:
         path = "."
     return Path().cwd() if path == "." else Path(path).absolute()
+
+
+def safe_join(root_dir: str | Path | None, filename: str | Path) -> Path:
+    """Combine a trusted directory path with an untrusted filename to get a full path.
+
+    :param root_dir: Trusted path/directory.
+    :param filename: Untrusted filename to join to the trusted path. Any path components are stripped off.
+    :return: Fully joined path with filename.
+    """
+    logger.debug("Safely joining path '%s' with filename '%s'", root_dir, filename)
+
+    if not filename or not str(filename).strip():
+        raise ValueError("filename must not be empty")
+
+    stripped_filename = Path(str(filename)).name.strip()
+
+    if not stripped_filename:
+        raise ValueError("filename must not be empty")
+
+    parent_dir = parse_path(root_dir).resolve()
+    full_path = parent_dir.joinpath(stripped_filename).resolve()
+    if not full_path.is_relative_to(parent_dir):
+        raise ValueError("Could not safely join paths.")
+
+    return full_path