Modal Compute Knowledge

Modal Knowledge Skill

Comprehensive Modal.com platform knowledge covering all features, pricing, and best practices. Activate this skill when users need detailed information about Modal's serverless cloud platform.

Activation Triggers

Activate this skill when users ask about:

• Modal.com platform features and capabilities
• GPU-accelerated Python functions
• Serverless container configuration
• Modal pricing and billing
• Modal CLI commands
• Web endpoints and APIs on Modal
• Scheduled/cron jobs on Modal
• Modal volumes, secrets, and storage
• Parallel processing with Modal
• Modal deployment and CI/CD

---

Platform Overview

Modal is a serverless cloud platform for running Python code, optimized for AI/ML workloads with:

• Zero Configuration: Everything defined in Python code
• Fast GPU Startup: ~1 second container spin-up
• Automatic Scaling: Scale to zero, scale to thousands
• Per-Second Billing: Only pay for active compute
• Multi-Cloud: AWS, GCP, Oracle Cloud Infrastructure

---

Core Components Reference

Apps and Functions

import modal

app = modal.App("app-name")

@app.function()
def basic_function(arg: str) -> str:
    return f"Result: {arg}"

@app.local_entrypoint()
def main():
    result = basic_function.remote("test")
    print(result)

Function Decorator Parameters

---

GPU Reference

Available GPUs

GPU Configuration

# Single GPU
@app.function(gpu="A100")

# Specific memory variant
@app.function(gpu="A100-80GB")

# Multi-GPU
@app.function(gpu="H100:4")

# Fallbacks (tries in order)
@app.function(gpu=["H100", "A100", "any"])

# "any" = L4, A10G, or T4
@app.function(gpu="any")

---

Image Building

Base Images

# Debian slim (recommended)
modal.Image.debian_slim(python_version="3.11")

# From Dockerfile
modal.Image.from_dockerfile("./Dockerfile")

# From Docker registry
modal.Image.from_registry("nvidia/cuda:12.1.0-base-ubuntu22.04")

Package Installation

# pip (standard)
image.pip_install("torch", "transformers")

# uv (FASTER - 10-100x)
image.uv_pip_install("torch", "transformers")

# System packages
image.apt_install("ffmpeg", "libsm6")

# Shell commands
image.run_commands("apt-get update", "make install")

Adding Files

# Single file
image.add_local_file("./config.json", "/app/config.json")

# Directory
image.add_local_dir("./models", "/app/models")

# Python source
image.add_local_python_source("my_module")

# Environment variables
image.env({"VAR": "value"})

Build-Time Function

def download_model():
    from huggingface_hub import snapshot_download
    snapshot_download("model-name")

image.run_function(download_model, secrets=[...])

---

Storage

Volumes

# Create/reference volume
vol = modal.Volume.from_name("my-vol", create_if_missing=True)

# Mount in function
@app.function(volumes={"/data": vol})
def func():
    # Read/write to /data
    vol.commit()  # Persist changes

Secrets

# From dashboard (recommended)
modal.Secret.from_name("secret-name")

# From dictionary
modal.Secret.from_dict({"KEY": "value"})

# From local env
modal.Secret.from_local_environ(["KEY1", "KEY2"])

# From .env file
modal.Secret.from_dotenv()

# Usage
@app.function(secrets=[modal.Secret.from_name("api-keys")])
def func():
    import os
    key = os.environ["API_KEY"]

Dict and Queue

# Distributed dict
d = modal.Dict.from_name("cache", create_if_missing=True)
d["key"] = "value"
d.put("key", "value", ttl=3600)

# Distributed queue
q = modal.Queue.from_name("jobs", create_if_missing=True)
q.put("task")
item = q.get()

---

Web Endpoints

FastAPI Endpoint (Simple)

@app.function()
@modal.fastapi_endpoint()
def hello(name: str = "World"):
    return {"message": f"Hello, {name}!"}

ASGI App (Full FastAPI)

from fastapi import FastAPI
web_app = FastAPI()

@web_app.post("/predict")
def predict(text: str):
    return {"result": process(text)}

@app.function()
@modal.asgi_app()
def fastapi_app():
    return web_app

WSGI App (Flask)

from flask import Flask
flask_app = Flask(__name__)

@app.function()
@modal.wsgi_app()
def flask_endpoint():
    return flask_app

Custom Web Server

@app.function()
@modal.web_server(port=8000)
def custom_server():
    subprocess.run(["python", "-m", "http.server", "8000"])

Custom Domains

@modal.asgi_app(custom_domains=["api.example.com"])

---

Scheduling

Cron

# Daily at 8 AM UTC
@app.function(schedule=modal.Cron("0 8 * * *"))

# With timezone
@app.function(schedule=modal.Cron("0 6 * * *", timezone="America/New_York"))

Period

@app.function(schedule=modal.Period(hours=5))
@app.function(schedule=modal.Period(days=1))

Note: Scheduled functions only run with modal deploy, not modal run.

---

Parallel Processing

Map

# Parallel execution (up to 1000 concurrent)
results = list(func.map(items))

# Unordered (faster)
results = list(func.map(items, order_outputs=False))

Starmap

# Spread args
pairs = [(1, 2), (3, 4)]
results = list(add.starmap(pairs))

Spawn

# Async job (returns immediately)
call = func.spawn(data)
result = call.get()  # Get result later

# Spawn many
calls = [func.spawn(item) for item in items]
results = [call.get() for call in calls]

---

Container Lifecycle (Classes)

@app.cls(gpu="A100", container_idle_timeout=300)
class Server:

    @modal.enter()
    def load(self):
        self.model = load_model()

    @modal.method()
    def predict(self, text):
        return self.model(text)

    @modal.exit()
    def cleanup(self):
        del self.model

Concurrency

@modal.concurrent(max_inputs=100, target_inputs=80)
@modal.method()
def batched(self, item):
    pass

---

CLI Commands

Development

modal run app.py              # Run function
modal serve app.py            # Hot-reload dev server
modal shell app.py            # Interactive shell
modal shell app.py --gpu A100 # Shell with GPU

Deployment

modal deploy app.py           # Deploy
modal app list                # List apps
modal app logs app-name       # View logs
modal app stop app-name       # Stop app

Resources

# Volumes
modal volume create name
modal volume list
modal volume put name local remote
modal volume get name remote local

# Secrets
modal secret create name KEY=value
modal secret list

# Environments
modal environment create staging

---

Pricing (2025)

Plans

Compute

• CPU: $0.0000131/core/sec
• Memory: $0.00000222/GiB/sec
• GPUs: See GPU table above

Special Programs

• Startups: Up to $25k credits
• Researchers: Up to $10k credits

---

Best Practices

1. Use `@modal.enter()` for model loading 2. Use `uv_pip_install` for faster builds 3. Use GPU fallbacks for availability 4. Set appropriate timeouts and retries 5. Use environments (dev/staging/prod) 6. Download models during build, not runtime 7. Use `order_outputs=False` when order doesn't matter 8. Set `container_idle_timeout` to balance cost/latency 9. Monitor costs in Modal dashboard 10. Test with `modal run` before modal deploy

---

Common Patterns

LLM Inference

@app.cls(gpu="A100", container_idle_timeout=300)
class LLM:
    @modal.enter()
    def load(self):
        from vllm import LLM
        self.llm = LLM(model="...")

    @modal.method()
    def generate(self, prompt):
        return self.llm.generate([prompt])

Batch Processing

@app.function(volumes={"/data": vol})
def process(file):
    # Process file
    vol.commit()

# Parallel
results = list(process.map(files))

Scheduled ETL

@app.function(
    schedule=modal.Cron("0 6 * * *"),
    secrets=[modal.Secret.from_name("db")]
)
def daily_etl():
    extract()
    transform()
    load()

---

Quick Reference

Modal @modal.batched for Dynamic Batching

Overview

The @modal.batched decorator enables automatic request batching for ML inference, combining multiple inputs into efficient batch operations.

Basic Usage

import modal

app = modal.App("batched-inference")

@app.cls(gpu="A100")
class BatchedModel:
    @modal.enter()
    def load(self):
        import torch
        self.model = load_model()
        self.model.eval()

    @modal.batched(max_batch_size=32, wait_ms=100)
    @modal.method()
    def predict(self, inputs: list[str]) -> list[str]:
        """
        Receives a batch of inputs automatically collected by Modal.
        Returns a list of outputs (one per input).
        """
        # Batch inference
        results = self.model.batch_predict(inputs)
        return results

Parameters

How It Works

1. Client calls predict(single_item) multiple times 2. Modal collects requests up to max_batch_size or wait_ms 3. Your function receives a batch (list) of inputs 4. Your function returns a list of outputs 5. Modal distributes outputs back to individual callers

Client 1: predict("a") ──┐
Client 2: predict("b") ──┼──► batched predict(["a","b","c"]) ──► ["result_a","result_b","result_c"]
Client 3: predict("c") ──┘                                            │
                                                                      │
Client 1: receives "result_a" ◄───────────────────────────────────────┘
Client 2: receives "result_b" ◄───────────────────────────────────────┘
Client 3: receives "result_c" ◄───────────────────────────────────────┘

Complete Example: LLM Inference

import modal

app = modal.App("llm-batched")

image = (
    modal.Image.debian_slim(python_version="3.11")
    .pip_install("torch", "transformers", "accelerate")
)

@app.cls(
    gpu="A100",
    image=image,
    container_idle_timeout=300,
)
class LLMServer:

    @modal.enter()
    def load_model(self):
        from transformers import AutoModelForCausalLM, AutoTokenizer
        import torch

        model_id = "mistralai/Mistral-7B-Instruct-v0.2"

        self.tokenizer = AutoTokenizer.from_pretrained(model_id)
        self.tokenizer.pad_token = self.tokenizer.eos_token

        self.model = AutoModelForCausalLM.from_pretrained(
            model_id,
            torch_dtype=torch.float16,
            device_map="auto",
        )

    @modal.batched(max_batch_size=16, wait_ms=50)
    @modal.method()
    def generate(self, prompts: list[str]) -> list[str]:
        """Batch generate responses for multiple prompts"""
        import torch

        # Tokenize batch
        inputs = self.tokenizer(
            prompts,
            return_tensors="pt",
            padding=True,
            truncation=True,
            max_length=512,
        ).to("cuda")

        # Generate
        with torch.no_grad():
            outputs = self.model.generate(
                **inputs,
                max_new_tokens=256,
                do_sample=True,
                temperature=0.7,
            )

        # Decode
        responses = self.tokenizer.batch_decode(
            outputs, skip_special_tokens=True
        )

        return responses


# Usage (clients call with single items)
@app.local_entrypoint()
def main():
    model = LLMServer()

    # These will be automatically batched
    prompts = [
        "What is Python?",
        "Explain machine learning",
        "How does a GPU work?",
    ]

    # Parallel calls - batched automatically
    results = list(model.generate.map(prompts))

    for prompt, result in zip(prompts, results):
        print(f"Q: {prompt}")
        print(f"A: {result}\n")

Batched Image Processing

@app.cls(gpu="T4")
class ImageProcessor:

    @modal.enter()
    def load(self):
        from transformers import CLIPProcessor, CLIPModel

        self.processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
        self.model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").cuda()

    @modal.batched(max_batch_size=64, wait_ms=100)
    @modal.method()
    def embed_images(self, image_bytes_list: list[bytes]) -> list[list[float]]:
        """Batch image embedding"""
        from PIL import Image
        import io
        import torch

        # Decode images
        images = [Image.open(io.BytesIO(b)) for b in image_bytes_list]

        # Process batch
        inputs = self.processor(images=images, return_tensors="pt").to("cuda")

        with torch.no_grad():
            embeddings = self.model.get_image_features(**inputs)

        return embeddings.cpu().numpy().tolist()

Tuning Batch Parameters

High Throughput (Maximize GPU Utilization)

@modal.batched(max_batch_size=64, wait_ms=200)
@modal.method()
def high_throughput(self, inputs: list):
    # Larger batches, longer wait
    pass

Low Latency (Minimize Response Time)

@modal.batched(max_batch_size=8, wait_ms=10)
@modal.method()
def low_latency(self, inputs: list):
    # Smaller batches, shorter wait
    pass

Balanced

@modal.batched(max_batch_size=32, wait_ms=50)
@modal.method()
def balanced(self, inputs: list):
    pass

Combining with @modal.concurrent

For handling many requests while batching:

@app.cls(gpu="A100")
class Server:

    @modal.concurrent(max_inputs=100)
    @modal.batched(max_batch_size=32)
    @modal.method()
    def process(self, items: list):
        # Container handles 100 concurrent callers
        # Requests batched into groups of 32
        return batch_process(items)

Best Practices

1. Match batch size to GPU memory - Larger batches use more VRAM 2. Set `wait_ms` based on latency requirements - Lower = faster response, smaller batches 3. Input/output must be lists - Function receives list, returns list 4. Same length requirement - Output list must match input list length 5. Use with GPU workloads - Batching shines for GPU inference 6. Monitor throughput - Adjust parameters based on real traffic

When to Use Batching

Modal Scaling and Autoscaler Configuration

Autoscaler Settings (Modal 1.0 SDK)

Modal's autoscaler manages container provisioning based on workload. Configure via @app.function() or @app.cls():

@app.function(
    min_containers=0,          # Minimum warm containers (default: 0)
    max_containers=100,        # Maximum concurrent containers
    buffer_containers=0,       # Pre-warm buffer containers
    scaledown_window=300,      # Seconds before scaling down (default: 300)
)
def scalable_func():
    pass

Parameters Explained

Scaling Patterns

High-Traffic API (Keep Warm)

@app.cls(
    gpu="A10G",
    min_containers=2,        # Always warm
    buffer_containers=1,     # Extra headroom
    container_idle_timeout=600,
)
class ProductionAPI:
    @modal.enter()
    def load(self):
        self.model = load_model()

    @modal.method()
    def predict(self, data):
        return self.model(data)

Burst Processing (Scale Quickly)

@app.function(
    max_containers=500,      # Allow massive scale-out
    scaledown_window=60,     # Scale down quickly when done
)
def burst_processor(item):
    return process(item)

# Process 10,000 items in parallel
results = list(burst_processor.map(items))

Cost-Optimized (Scale to Zero)

@app.function(
    min_containers=0,        # Scale to zero
    max_containers=10,       # Limit costs
    scaledown_window=120,    # Quick scale down
)
def cost_optimized(data):
    return process(data)

@modal.concurrent Decorator

Replaces the old allow_concurrent_inputs parameter. Allows one container to handle multiple requests:

@app.cls(gpu="A100")
class ConcurrentServer:
    @modal.enter()
    def load(self):
        self.model = load_model()

    @modal.concurrent(max_inputs=100, target_inputs=80)
    @modal.method()
    def predict(self, data):
        # Container handles up to 100 concurrent requests
        # Autoscaler adds containers when hitting ~80 requests
        return self.model(data)

Concurrent Parameters

When to Use Concurrency

Scaling Limits

Parallel Processing Methods

.map() - Synchronized Batch

# Process in parallel, wait for all
results = list(func.map(items))

# Unordered for better performance
results = list(func.map(items, order_outputs=False))

# With return exceptions
results = list(func.map(items, return_exceptions=True))

.starmap() - Multiple Arguments

# Each item is unpacked as positional args
pairs = [(1, 2), (3, 4), (5, 6)]
results = list(add.starmap(pairs))

.spawn() - Fire and Forget

# Submit without waiting
call = func.spawn(data)

# Get result later
result = call.get()

# Poll status
if call.status() == "completed":
    result = call.get()

# Cancel if needed
call.cancel()

.for_each() - Side Effects

# Execute for side effects, no return values
func.for_each(items)

Performance Tuning

Reduce Cold Starts

@app.function(
    min_containers=1,           # Keep 1 warm
    container_idle_timeout=600, # Stay warm 10 min
)
def low_latency():
    pass

Maximize Throughput

@app.cls(
    gpu="A100",
    max_containers=100,
)
class HighThroughput:
    @modal.concurrent(max_inputs=50)
    @modal.method()
    def process(self, data):
        return self.batch_inference(data)

Balance Cost and Latency

@app.function(
    min_containers=0,           # Scale to zero when idle
    buffer_containers=1,        # 1 extra for bursts
    scaledown_window=180,       # 3 min before scale down
    container_idle_timeout=300, # 5 min idle timeout
)
def balanced():
    pass

Monitoring Scaling

# View function metrics
modal app show my-app

# Watch container scaling
modal app logs my-app --follow

# Dashboard provides:
# - Active containers
# - Queue depth
# - Latency percentiles
# - Error rates