Function Calling

Use function calling across providers with automatic format translation

Overview

Function calling (also called tool use) allows LLMs to call external functions and APIs. LiteLLM provides a unified interface that automatically translates between different provider formats.

Quick Start

from litellm import completion

tools = [
    {
        "type": "function",
        "function": {
            "name": "get_weather",
            "description": "Get the current weather in a location",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {
                        "type": "string",
                        "description": "City and state, e.g. San Francisco, CA"
                    },
                    "unit": {
                        "type": "string",
                        "enum": ["celsius", "fahrenheit"]
                    }
                },
                "required": ["location"]
            }
        }
    }
]

response = completion(
    model="gpt-4o",
    messages=[{"role": "user", "content": "What's the weather in NYC?"}],
    tools=tools
)

if response.choices[0].message.tool_calls:
    for tool_call in response.choices[0].message.tool_calls:
        print(f"Function: {tool_call.function.name}")
        print(f"Arguments: {tool_call.function.arguments}")

Defining Functions

OpenAI Format

Standard format used by LiteLLM.

tools = [
    {
        "type": "function",
        "function": {
            "name": "search_database",
            "description": "Search the product database",
            "parameters": {
                "type": "object",
                "properties": {
                    "query": {
                        "type": "string",
                        "description": "Search query"
                    },
                    "category": {
                        "type": "string",
                        "enum": ["electronics", "clothing", "books"]
                    },
                    "max_results": {
                        "type": "integer",
                        "description": "Max number of results",
                        "default": 10
                    }
                },
                "required": ["query"]
            }
        }
    }
]

Multiple Functions

Define multiple tools for the model to choose from.

tools = [
    {
        "type": "function",
        "function": {
            "name": "get_weather",
            "description": "Get weather information",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {"type": "string"}
                },
                "required": ["location"]
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "search_web",
            "description": "Search the web",
            "parameters": {
                "type": "object",
                "properties": {
                    "query": {"type": "string"}
                },
                "required": ["query"]
            }
        }
    }
]

Function Calling Flow

from litellm import completion
import json

# Step 1: Define tools
tools = [
    {
        "type": "function",
        "function": {
            "name": "get_stock_price",
            "description": "Get current stock price",
            "parameters": {
                "type": "object",
                "properties": {
                    "symbol": {"type": "string", "description": "Stock symbol"}
                },
                "required": ["symbol"]
            }
        }
    }
]

# Step 2: Initial request
messages = [{"role": "user", "content": "What's AAPL stock price?"}]
response = completion(
    model="gpt-4o",
    messages=messages,
    tools=tools
)

# Step 3: Check for tool calls
if response.choices[0].message.tool_calls:
    # Step 4: Execute functions
    for tool_call in response.choices[0].message.tool_calls:
        function_name = tool_call.function.name
        arguments = json.loads(tool_call.function.arguments)
        
        if function_name == "get_stock_price":
            # Execute your function
            result = {"symbol": arguments["symbol"], "price": 150.25}
            
            # Step 5: Add tool response to messages
            messages.append(response.choices[0].message)
            messages.append({
                "role": "tool",
                "tool_call_id": tool_call.id,
                "content": json.dumps(result)
            })
    
    # Step 6: Get final response
    final_response = completion(
        model="gpt-4o",
        messages=messages,
        tools=tools
    )
    print(final_response.choices[0].message.content)

Tool Choice Control

Auto (Default)

Let the model decide whether to call functions.

response = completion(
    model="gpt-4o",
    messages=[{"role": "user", "content": "Hello!"}],
    tools=tools
    # tool_choice="auto" is implicit
)

Force Function Call

Require the model to call a specific function.

response = completion(
    model="gpt-4o",
    messages=[{"role": "user", "content": "Get weather"}],
    tools=tools,
    tool_choice={
        "type": "function",
        "function": {"name": "get_weather"}
    }
)

Require Any Function

Force a function call but let model choose which one.

response = completion(
    model="gpt-4o",
    messages=[{"role": "user", "content": "Help me"}],
    tools=tools,
    tool_choice="required"  # or "any"
)

No Function Calls

Disable function calling for this request.

response = completion(
    model="gpt-4o",
    messages=[{"role": "user", "content": "Just chat"}],
    tools=tools,
    tool_choice="none"
)

Provider Support

OpenAI

Native support with full features.

from litellm import completion

response = completion(
    model="gpt-4o",
    messages=[{"role": "user", "content": "What's the weather?"}],
    tools=tools
)

Anthropic

Automatic translation to Claude’s tool format.

response = completion(
    model="anthropic/claude-3.5-sonnet",
    messages=[{"role": "user", "content": "What's the weather?"}],
    tools=tools
)
# LiteLLM translates OpenAI format → Anthropic format

Cohere

Automatic translation to Cohere’s tool format.

response = completion(
    model="cohere/command-r-plus",
    messages=[{"role": "user", "content": "What's the weather?"}],
    tools=tools
)
# LiteLLM translates to Cohere's parameter_definitions format

Groq

Native OpenAI-compatible support.

response = completion(
    model="groq/llama-3.3-70b-versatile",
    messages=[{"role": "user", "content": "What's the weather?"}],
    tools=tools
)

Ollama

Requires Ollama 0.4+ for native support.

response = completion(
    model="ollama/llama3.3",
    messages=[{"role": "user", "content": "What's the weather?"}],
    tools=tools,
    api_base="http://localhost:11434"
)

Streaming with Function Calls

from litellm import completion

response = completion(
    model="gpt-4o",
    messages=[{"role": "user", "content": "What's the weather in NYC?"}],
    tools=tools,
    stream=True
)

tool_calls_buffer = []
content_buffer = ""

for chunk in response:
    delta = chunk.choices[0].delta
    
    # Handle tool call deltas
    if delta.tool_calls:
        for tool_call_delta in delta.tool_calls:
            # Accumulate tool call information
            if tool_call_delta.function.name:
                print(f"\nCalling: {tool_call_delta.function.name}")
            if tool_call_delta.function.arguments:
                print(tool_call_delta.function.arguments, end="")
    
    # Handle content
    elif delta.content:
        content_buffer += delta.content
        print(delta.content, end="")

Parallel Function Calls

Some providers support calling multiple functions simultaneously.

from litellm import completion
import json

tools = [
    {"type": "function", "function": {"name": "get_weather", ...}},
    {"type": "function", "function": {"name": "get_time", ...}}
]

response = completion(
    model="gpt-4o",
    messages=[{"role": "user", "content": "What's the weather and time in NYC?"}],
    tools=tools,
    parallel_tool_calls=True  # Enable parallel calls
)

# Handle multiple tool calls
if response.choices[0].message.tool_calls:
    for tool_call in response.choices[0].message.tool_calls:
        print(f"Call {tool_call.id}: {tool_call.function.name}")
        # Execute functions in parallel

Complex Parameter Types

tools = [
    {
        "type": "function",
        "function": {
            "name": "book_flight",
            "description": "Book a flight",
            "parameters": {
                "type": "object",
                "properties": {
                    "departure": {
                        "type": "object",
                        "properties": {
                            "city": {"type": "string"},
                            "airport": {"type": "string"},
                            "date": {"type": "string", "format": "date"}
                        },
                        "required": ["city", "date"]
                    },
                    "arrival": {
                        "type": "object",
                        "properties": {
                            "city": {"type": "string"},
                            "airport": {"type": "string"}
                        },
                        "required": ["city"]
                    },
                    "passengers": {
                        "type": "array",
                        "items": {
                            "type": "object",
                            "properties": {
                                "name": {"type": "string"},
                                "age": {"type": "integer"}
                            }
                        }
                    },
                    "class": {
                        "type": "string",
                        "enum": ["economy", "business", "first"]
                    }
                },
                "required": ["departure", "arrival"]
            }
        }
    }
]

Error Handling

from litellm import completion
import json

try:
    response = completion(
        model="gpt-4o",
        messages=[{"role": "user", "content": "Get weather"}],
        tools=tools
    )
    
    if response.choices[0].message.tool_calls:
        for tool_call in response.choices[0].message.tool_calls:
            try:
                args = json.loads(tool_call.function.arguments)
                # Execute function
                result = execute_function(tool_call.function.name, args)
            except json.JSONDecodeError:
                print("Invalid JSON in arguments")
            except Exception as e:
                print(f"Function execution error: {e}")
                # Return error to model
                messages.append({
                    "role": "tool",
                    "tool_call_id": tool_call.id,
                    "content": json.dumps({"error": str(e)})
                })
                
except Exception as e:
    print(f"Completion error: {e}")

Best Practices

Function Descriptions

• Write clear, detailed descriptions
• Explain when the function should be used
• Include example values in parameter descriptions
• Be specific about parameter formats

Parameter Design

• Use descriptive parameter names
• Mark only truly required parameters as required
• Provide enums for fixed choices
• Use appropriate JSON schema types

Response Handling

• Always validate function arguments
• Handle JSON parsing errors gracefully
• Return structured error messages to the model
• Include relevant context in function results

Conversation Flow

• Keep full conversation history in messages
• Include both assistant and tool messages
• Allow multiple tool call rounds if needed
• Provide clear context in tool results

Advanced Patterns

Retry Logic

def call_with_retry(messages, tools, max_retries=3):
    for attempt in range(max_retries):
        response = completion(
            model="gpt-4o",
            messages=messages,
            tools=tools
        )
        
        if response.choices[0].message.tool_calls:
            try:
                # Execute and validate
                result = execute_tools(response.choices[0].message.tool_calls)
                return result
            except Exception as e:
                # Add error feedback
                messages.append(response.choices[0].message)
                messages.append({
                    "role": "tool",
                    "content": f"Error: {e}. Please try again."
                })
        else:
            return response
    
    raise Exception("Max retries exceeded")

Function Registry

function_registry = {}

def register_function(schema):
    def decorator(func):
        function_registry[schema["function"]["name"]] = {
            "function": func,
            "schema": schema
        }
        return func
    return decorator

@register_function({
    "type": "function",
    "function": {
        "name": "get_weather",
        "description": "Get weather",
        "parameters": {...}
    }
})
def get_weather(location):
    return {"temperature": 72, "condition": "sunny"}

# Use registered functions
tools = [entry["schema"] for entry in function_registry.values()]

Debugging

import litellm
from litellm import completion

# Enable debug logging
litellm.set_verbose = True

response = completion(
    model="gpt-4o",
    messages=[{"role": "user", "content": "Get weather"}],
    tools=tools
)

# Inspect tool calls
if response.choices[0].message.tool_calls:
    for i, tool_call in enumerate(response.choices[0].message.tool_calls):
        print(f"\nTool Call {i}:")
        print(f"  ID: {tool_call.id}")
        print(f"  Type: {tool_call.type}")
        print(f"  Function: {tool_call.function.name}")
        print(f"  Arguments: {tool_call.function.arguments}")