Prompt Optimization with Evidently¶

# If you haven't installed the required packages yet:
# !pip install evidently openai pandas

import pandas as pd

from evidently import Dataset, DataDefinition, LLMClassification
from evidently.llm.templates import BinaryClassificationPromptTemplate
from evidently.descriptors import LLMEval
from evidently.llm.optimization import PromptOptimizer
from evidently.descriptors import HuggingFace, HuggingFaceToxicity

# === Datasets Path ===
from pathlib import Path
DATASETS_DIR = Path("./datasets")
print(f"Datasets: {[f.name for f in DATASETS_DIR.glob('*.csv')]}")

import os
from typing import Dict, Any, List, Optional
from evidently.llm.utils.wrapper import OpenAIOptions, OpenAIWrapper, LLMMessage, LLMResult

# === Ollama Configuration via OpenAI-Compatible API ===
OLLAMA_HOST = os.getenv("OLLAMA_HOST", "http://ollama:11434")

# === Model Configuration ===
HF_LLM_MODEL = "NousResearch/Nous-Hermes-2-Mistral-7B-DPO-GGUF"
OLLAMA_LLM_MODEL = f"hf.co/{HF_LLM_MODEL}:Q4_K_M"

OLLAMA_OPTIONS = OpenAIOptions(
    api_key="ollama",
    api_url=f"{OLLAMA_HOST}/v1"
)

# === Patch OpenAIWrapper for smart JSON mode detection ===
# Evidently's OpenAI wrapper doesn't enable JSON mode by default.
# This patch detects when JSON output is expected and enables it.
_original_openai_complete = OpenAIWrapper.complete

async def _json_aware_complete(self, messages: List[LLMMessage], seed: Optional[int] = None) -> LLMResult[str]:
    import openai
    from openai.types.chat.chat_completion import ChatCompletion
    
    message_text = " ".join(m.content for m in messages if m.content)
    needs_json = "json" in message_text.lower() or '"category"' in message_text
    needs_xml = "<new_prompt>" in message_text
    
    formatted_messages = [{"role": msg.role, "content": msg.content} for msg in messages]
    
    try:
        kwargs = {"model": self.model, "messages": formatted_messages, "seed": seed}
        if needs_json and not needs_xml:
            kwargs["response_format"] = {"type": "json_object"}
        
        response: ChatCompletion = await self.client.chat.completions.create(**kwargs)
    except openai.RateLimitError as e:
        from evidently.llm.utils.wrapper import LLMRateLimitError
        raise LLMRateLimitError(e.message) from e
    except openai.APIError as e:
        from evidently.llm.utils.wrapper import LLMRequestError
        raise LLMRequestError(f"Failed to call OpenAI complete API: {e.message}", original_error=e) from e

    content = response.choices[0].message.content
    assert content is not None
    if response.usage is None:
        return LLMResult(content, 0, 0)
    return LLMResult(content, response.usage.prompt_tokens, response.usage.completion_tokens)

OpenAIWrapper.complete = _json_aware_complete

print(f"Ollama host: {OLLAMA_HOST}")
print(f"Model: {OLLAMA_LLM_MODEL}")
print(f"Using OpenAI-compatible API with smart JSON mode detection")

# Load your dataset
review_dataset = pd.read_csv(DATASETS_DIR / "code_review_dataset.csv")
review_dataset.head()

# Define how Evidently should interpret your dataset
dd = DataDefinition(
    text_columns=["Generated review", "Expert comment"],
    categorical_columns=["Expert label"],
    llm=LLMClassification(input="Generated review", target="Expert label", reasoning="Expert comment")
)

# Show the best-performing prompt template found by the optimizer
print(optimizer.best_prompt())

import pandas as pd

from evidently import Dataset, DataDefinition, LLMClassification
from evidently.descriptors import LLMEval
from evidently.llm.templates import MulticlassClassificationPromptTemplate
from evidently.llm.optimization import PromptOptimizer

data = pd.read_csv(DATASETS_DIR / "booking_queries_dataset.csv")
data.head()

dd = DataDefinition(
    text_columns=["query"],
    categorical_columns=["label"],
    llm=LLMClassification(input="query", target="label")
)

dataset = Dataset.from_pandas(data, data_definition=dd)

base_prompt = "Classify inqueries by categories"

t = MulticlassClassificationPromptTemplate(
    pre_messages=[("system", "You are classifying user queries.")],
    criteria=base_prompt,
    category_criteria={
        "Booking": "bookings",
        "Technical": "technical questions",
        "Policy": "questions about policies",
        "Payment": "payment questions",
        "Escalation": "escalation requests"
    },
    uncertainty="unknown",
    include_reasoning=True,
)

judge = LLMEval(
    alias="bookings",
    provider="openai",  # Use OpenAI provider with Ollama's OpenAI-compatible API
    model=OLLAMA_LLM_MODEL,
    column_name="query",
    template=t
)

optimizer = PromptOptimizer(
    "bookings_example", 
    strategy="feedback",
    provider="openai",  # Use OpenAI provider with Ollama's OpenAI-compatible API
    model=OLLAMA_LLM_MODEL
)
optimizer.set_input_dataset(dataset)
await optimizer.arun(judge, "accuracy", options=OLLAMA_OPTIONS)

print(optimizer.best_prompt())

print(optimizer.best_prompt())