将智能体追踪转换为评估数据集

from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient

credential = DefaultAzureCredential()
project_client = AIProjectClient(
    endpoint="https://<your-resource>.services.ai.azure.com/api/projects/<your-project>",
    credential=credential,
)

npm install "@azure/ai-projects@^2.8.0" @azure/identity

import { DefaultAzureCredential } from "@azure/identity";
import { AIProjectClient } from "@azure/ai-projects";

const projectEndpoint =
  "https://<your-resource>.services.ai.azure.com/api/projects/<your-project>";
const projectClient = new AIProjectClient(
  projectEndpoint,
  new DefaultAzureCredential(),
);

import time
from datetime import datetime, timedelta, timezone

from azure.ai.projects.models import (
    DataGenerationJobOutputWriteMode,
    DatasetDataGenerationJobOutput,
    EvaluationDataGenerationJobInputs,
    EvaluationDataGenerationJobOutputConfiguration,
    TracesDataGenerationJobConfiguration,
    TracesDataGenerationJobSource,
)

AGENT_NAME = "retail-agent"
poll_interval_seconds = 10

# 1. Record the window around your traffic.
end_time = datetime.now(tz=timezone.utc)
start_time = end_time - timedelta(days=7)

# 2. Define an evaluation job. The class sets scenario="evaluation".
job = EvaluationDataGenerationJobInputs(
    name="retail-agent-eval-set",
    sources=[
        TracesDataGenerationJobSource(
            description=(
                "Application Insights conversation traces for the Foundry "
                "agent."
            ),
            agent_name=AGENT_NAME,
            start_time=start_time,
            end_time=end_time,
            # agent_version="3",  # Pin to a specific version.
            # trace_ids=["trace-id-1", "trace-id-2"],  # Select exact traces.
        ),
    ],
    generation_configuration=TracesDataGenerationJobConfiguration(
        # Omit max_samples to turn off intelligent sampling.
        max_samples=100,
        # Private content is redacted by default.
        redact_private_content=True,
    ),
    output_configuration=EvaluationDataGenerationJobOutputConfiguration(
        name="retail-agent-eval-set",
        description="Representative production traces for agent evaluation.",
        tags={"source": "production-traces"},
        write_mode=DataGenerationJobOutputWriteMode.OVERWRITE,
    ),
)

# 3. Submit and wait for completion.
poller = project_client.datasets.begin_create_generation_job(job=job)
while not poller.done():
    print(f"\tstatus=`{poller.status()}`")
    time.sleep(poll_interval_seconds)
result = poller.result()

# 4. Resolve the generated dataset.
output_name = ""
output_version = ""
for output in (result.outputs if result is not None else None) or []:
    if isinstance(output, DatasetDataGenerationJobOutput):
        output_name = output.name or ""
        output_version = output.version or ""
        break

if not output_name or not output_version:
    raise RuntimeError("The data generation job didn't return a dataset output.")

dataset = project_client.datasets.get(name=output_name, version=output_version)
print(f"Generated dataset: {dataset.name} v{dataset.version} (id: {dataset.id})")
if result is not None and result.generated_samples is not None:
    print(f"Generated samples: {result.generated_samples}")

const agentName = "retail-agent";
const jobName = "retail-agent-eval-set";

// 1. Record the window around your traffic.
const endTime = new Date();
endTime.setMilliseconds(0);
const startTime = new Date(endTime.getTime() - 7 * 24 * 60 * 60 * 1000);

// 2. Define and submit the evaluation job.
const generationPoller = projectClient.datasets.createGenerationJob({
  name: jobName,
  scenario: "evaluation",
  sources: [
    {
      type: "traces",
      description:
        "Application Insights conversation traces for the Foundry agent.",
      agent_name: agentName,
      start_time: startTime,
      end_time: endTime,
      // agent_version: "3", // Pin to a specific version.
      // trace_ids: ["trace-id-1", "trace-id-2"], // Select exact traces.
    },
  ],
  generation_configuration: {
    type: "traces",
    // Omit max_samples to turn off intelligent sampling.
    max_samples: 100,
    // Private content is redacted by default.
    redact_private_content: true,
  },
  output_configuration: {
    name: jobName,
    description: "Representative production traces for agent evaluation.",
    tags: { source: "production-traces" },
    write_mode: "overwrite",
  },
});

await generationPoller.submitted();
const result = await generationPoller.pollUntilDone();

// 3. Resolve the generated dataset.
const datasetOutput = result.outputs?.find((output) => output.type === "dataset");
if (
  !datasetOutput ||
  !("name" in datasetOutput) ||
  !("version" in datasetOutput) ||
  !datasetOutput.name ||
  !datasetOutput.version
) {
  throw new Error("The data generation job didn't return a dataset output.");
}

const dataset = await projectClient.datasets.get(
  datasetOutput.name,
  datasetOutput.version,
);
console.log(
  `Generated dataset: ${dataset.name} v${dataset.version} (id: ${dataset.id})`,
);
if (result.generated_samples !== undefined) {
  console.log(`Generated samples: ${result.generated_samples}`);
}

# List recent evaluation jobs.
for job in project_client.datasets.list_generation_jobs(
    limit=20,
    order="desc",
):
    if job.scenario == "evaluation":
        print(f"{job.id}  {job.status:<12}  {job.name}")

# Get a job.
job = project_client.datasets.get_generation_job(job_id="job_...")

# Cancel a running job.
project_client.datasets.cancel_generation_job(job_id="job_...")

# Delete a job record.
project_client.datasets.delete_generation_job(job_id="job_...")

// List recent evaluation jobs.
for await (const job of projectClient.datasets.listGenerationJobs({
  limit: 20,
})) {
  if (job.scenario === "evaluation") {
    console.log(`${job.id}  ${job.status}  ${job.scenario}  ${job.name}`);
  }
}

// Get a job.
const job = await projectClient.datasets.getGenerationJob("job_...");

// Cancel a running job.
await projectClient.datasets.cancelGenerationJob("job_...");

// Delete a job record.
await projectClient.datasets.deleteGenerationJob("job_...");