Create Evaluation
curl --request POST \
--url https://api.nugen.in/api/v3/evaluations/create \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"model_id": "model_01k4x9m2p7q3r9d1",
"benchmark_id": "benchmark_01k4x9m2p7q3r6a1",
"evaluation_name": "Clause Review v2 Eval",
"custom_metrics": {
"defaults": [
"binary_correctness",
"response_relevance"
],
"custom": [
{
"name": "hallucination_check",
"prompt": "Does the generated answer contain factually incorrect information? Rate from 0-1.",
"parse_json": true,
"score_min": 0,
"score_max": 1
}
]
},
"baseline_model_id": "model_01k4x9m2p7q3r9d2"
}
'import requests
url = "https://api.nugen.in/api/v3/evaluations/create"
payload = {
"model_id": "model_01k4x9m2p7q3r9d1",
"benchmark_id": "benchmark_01k4x9m2p7q3r6a1",
"evaluation_name": "Clause Review v2 Eval",
"custom_metrics": {
"defaults": ["binary_correctness", "response_relevance"],
"custom": [
{
"name": "hallucination_check",
"prompt": "Does the generated answer contain factually incorrect information? Rate from 0-1.",
"parse_json": True,
"score_min": 0,
"score_max": 1
}
]
},
"baseline_model_id": "model_01k4x9m2p7q3r9d2"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
model_id: 'model_01k4x9m2p7q3r9d1',
benchmark_id: 'benchmark_01k4x9m2p7q3r6a1',
evaluation_name: 'Clause Review v2 Eval',
custom_metrics: {
defaults: ['binary_correctness', 'response_relevance'],
custom: [
{
name: 'hallucination_check',
prompt: 'Does the generated answer contain factually incorrect information? Rate from 0-1.',
parse_json: true,
score_min: 0,
score_max: 1
}
]
},
baseline_model_id: 'model_01k4x9m2p7q3r9d2'
})
};
fetch('https://api.nugen.in/api/v3/evaluations/create', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.nugen.in/api/v3/evaluations/create",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'model_id' => 'model_01k4x9m2p7q3r9d1',
'benchmark_id' => 'benchmark_01k4x9m2p7q3r6a1',
'evaluation_name' => 'Clause Review v2 Eval',
'custom_metrics' => [
'defaults' => [
'binary_correctness',
'response_relevance'
],
'custom' => [
[
'name' => 'hallucination_check',
'prompt' => 'Does the generated answer contain factually incorrect information? Rate from 0-1.',
'parse_json' => true,
'score_min' => 0,
'score_max' => 1
]
]
],
'baseline_model_id' => 'model_01k4x9m2p7q3r9d2'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.nugen.in/api/v3/evaluations/create"
payload := strings.NewReader("{\n \"model_id\": \"model_01k4x9m2p7q3r9d1\",\n \"benchmark_id\": \"benchmark_01k4x9m2p7q3r6a1\",\n \"evaluation_name\": \"Clause Review v2 Eval\",\n \"custom_metrics\": {\n \"defaults\": [\n \"binary_correctness\",\n \"response_relevance\"\n ],\n \"custom\": [\n {\n \"name\": \"hallucination_check\",\n \"prompt\": \"Does the generated answer contain factually incorrect information? Rate from 0-1.\",\n \"parse_json\": true,\n \"score_min\": 0,\n \"score_max\": 1\n }\n ]\n },\n \"baseline_model_id\": \"model_01k4x9m2p7q3r9d2\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.nugen.in/api/v3/evaluations/create")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"model_id\": \"model_01k4x9m2p7q3r9d1\",\n \"benchmark_id\": \"benchmark_01k4x9m2p7q3r6a1\",\n \"evaluation_name\": \"Clause Review v2 Eval\",\n \"custom_metrics\": {\n \"defaults\": [\n \"binary_correctness\",\n \"response_relevance\"\n ],\n \"custom\": [\n {\n \"name\": \"hallucination_check\",\n \"prompt\": \"Does the generated answer contain factually incorrect information? Rate from 0-1.\",\n \"parse_json\": true,\n \"score_min\": 0,\n \"score_max\": 1\n }\n ]\n },\n \"baseline_model_id\": \"model_01k4x9m2p7q3r9d2\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.nugen.in/api/v3/evaluations/create")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"model_id\": \"model_01k4x9m2p7q3r9d1\",\n \"benchmark_id\": \"benchmark_01k4x9m2p7q3r6a1\",\n \"evaluation_name\": \"Clause Review v2 Eval\",\n \"custom_metrics\": {\n \"defaults\": [\n \"binary_correctness\",\n \"response_relevance\"\n ],\n \"custom\": [\n {\n \"name\": \"hallucination_check\",\n \"prompt\": \"Does the generated answer contain factually incorrect information? Rate from 0-1.\",\n \"parse_json\": true,\n \"score_min\": 0,\n \"score_max\": 1\n }\n ]\n },\n \"baseline_model_id\": \"model_01k4x9m2p7q3r9d2\"\n}"
response = http.request(request)
puts response.read_body{
"evaluation_id": "evaluation_01k4x9m2p7q3r7b1",
"evaluation_name": "Clause Review v2 Eval",
"model_id": "model_01k4x9m2p7q3r9d1",
"status": "PROCESSING",
"benchmark_id": "benchmark_01k4x9m2p7q3r6a1",
"created_at": "2024-02-24T10:00:00Z",
"message": "Evaluation created successfully"
}{
"detail": "Model not found"
}{
"detail": "Model not found"
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}Evaluations
Create Evaluation
Create a new model evaluation using an existing benchmark.
This endpoint initiates a model evaluation using a benchmark from your library. Supports both single model evaluation and comparison mode where two models are evaluated side-by-side.
Request Body:
model_id: Primary model to evaluate (required). It need not be deployed. A deployed model answers within seconds. An undeployed one is loaded by the GPU on demand and its answers wait for the next batch cycle, about 5 minutes.benchmark_id: ID of the benchmark to use for evaluation (required)baseline_model_id(optional): Baseline model ID for comparison modecustom_metrics(optional):defaults, the built-in metrics to run (binary_correctness,response_relevance; both run when omitted), andcustom, up to 5 metrics of your own, each withname,prompt,parse_json,score_minandscore_max. A customnamecannot reuse a built-in name. Each custom metric adds one judge call per question, so it adds time and cost. An invalid configuration is refused with 422evaluation_name(optional): Display name. Defaults to the evaluation id if omitted
Returns:
evaluation_id: Unique identifier for tracking the evaluationmodel_id: The primary model being evaluatedstatus: Initial status (alwaysPROCESSING)benchmark_id: The benchmark being usedcreated_at: Timestamp when evaluation was createdmessage: Confirmation message
Example Request (Single Model):
POST /api/v3/evaluations/create
Headers: {"Authorization": "Bearer <api_key>"}
{
"model_id": "model_01kmqm4nrn9fw6r",
"benchmark_id": "benchmark_01k4x9m2p7q3r6a1",
"custom_metrics": {
"defaults": ["binary_correctness", "response_relevance"],
"custom": [
{
"name": "hallucination_check",
"prompt": "Does the generated answer contain factually incorrect information? Rate from 0-1.",
"score_min": 0.0,
"score_max": 1.0
}
]
}
}
Example Request (Comparison Mode):
POST /api/v3/evaluations/create
Headers: {"Authorization": "Bearer <api_key>"}
{
"model_id": "Qwen/Qwen2.5-0.5B-Instruct",
"baseline_model_id": "model_01kmqm4nrn9fw6r",
"benchmark_id": "benchmark_01k4x9m2p7q3r6a1"
}
Example Response:
{
"evaluation_id": "evaluation_01k4x9m2p7q3r7b1",
"model_id": "model_01kmqm4nrn9fw6r",
"status": "PROCESSING",
"benchmark_id": "benchmark_01k4x9m2p7q3r6a1",
"created_at": "2024-01-15T10:30:00Z",
"message": "Evaluation created successfully and queued for execution"
}
Notes:
- Single model mode: Evaluates one model against the benchmark
- Comparison mode: Provide
baseline_model_idto compare two models side-by-side - Evaluation runs asynchronously - use the returned
evaluation_idto check status - Each metric reports
<name>_mean,_min,_max,_stdand_valid_countin the results, custom ones included. A_valid_countof 0 means the judge gave no usable score for that metric. Comparison mode adds the mean difference for custom metrics - Use
/evaluations/{evaluation_id}/statusto track progress
POST
/
api
/
v3
/
evaluations
/
create
Create Evaluation
curl --request POST \
--url https://api.nugen.in/api/v3/evaluations/create \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"model_id": "model_01k4x9m2p7q3r9d1",
"benchmark_id": "benchmark_01k4x9m2p7q3r6a1",
"evaluation_name": "Clause Review v2 Eval",
"custom_metrics": {
"defaults": [
"binary_correctness",
"response_relevance"
],
"custom": [
{
"name": "hallucination_check",
"prompt": "Does the generated answer contain factually incorrect information? Rate from 0-1.",
"parse_json": true,
"score_min": 0,
"score_max": 1
}
]
},
"baseline_model_id": "model_01k4x9m2p7q3r9d2"
}
'import requests
url = "https://api.nugen.in/api/v3/evaluations/create"
payload = {
"model_id": "model_01k4x9m2p7q3r9d1",
"benchmark_id": "benchmark_01k4x9m2p7q3r6a1",
"evaluation_name": "Clause Review v2 Eval",
"custom_metrics": {
"defaults": ["binary_correctness", "response_relevance"],
"custom": [
{
"name": "hallucination_check",
"prompt": "Does the generated answer contain factually incorrect information? Rate from 0-1.",
"parse_json": True,
"score_min": 0,
"score_max": 1
}
]
},
"baseline_model_id": "model_01k4x9m2p7q3r9d2"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
model_id: 'model_01k4x9m2p7q3r9d1',
benchmark_id: 'benchmark_01k4x9m2p7q3r6a1',
evaluation_name: 'Clause Review v2 Eval',
custom_metrics: {
defaults: ['binary_correctness', 'response_relevance'],
custom: [
{
name: 'hallucination_check',
prompt: 'Does the generated answer contain factually incorrect information? Rate from 0-1.',
parse_json: true,
score_min: 0,
score_max: 1
}
]
},
baseline_model_id: 'model_01k4x9m2p7q3r9d2'
})
};
fetch('https://api.nugen.in/api/v3/evaluations/create', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.nugen.in/api/v3/evaluations/create",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'model_id' => 'model_01k4x9m2p7q3r9d1',
'benchmark_id' => 'benchmark_01k4x9m2p7q3r6a1',
'evaluation_name' => 'Clause Review v2 Eval',
'custom_metrics' => [
'defaults' => [
'binary_correctness',
'response_relevance'
],
'custom' => [
[
'name' => 'hallucination_check',
'prompt' => 'Does the generated answer contain factually incorrect information? Rate from 0-1.',
'parse_json' => true,
'score_min' => 0,
'score_max' => 1
]
]
],
'baseline_model_id' => 'model_01k4x9m2p7q3r9d2'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.nugen.in/api/v3/evaluations/create"
payload := strings.NewReader("{\n \"model_id\": \"model_01k4x9m2p7q3r9d1\",\n \"benchmark_id\": \"benchmark_01k4x9m2p7q3r6a1\",\n \"evaluation_name\": \"Clause Review v2 Eval\",\n \"custom_metrics\": {\n \"defaults\": [\n \"binary_correctness\",\n \"response_relevance\"\n ],\n \"custom\": [\n {\n \"name\": \"hallucination_check\",\n \"prompt\": \"Does the generated answer contain factually incorrect information? Rate from 0-1.\",\n \"parse_json\": true,\n \"score_min\": 0,\n \"score_max\": 1\n }\n ]\n },\n \"baseline_model_id\": \"model_01k4x9m2p7q3r9d2\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.nugen.in/api/v3/evaluations/create")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"model_id\": \"model_01k4x9m2p7q3r9d1\",\n \"benchmark_id\": \"benchmark_01k4x9m2p7q3r6a1\",\n \"evaluation_name\": \"Clause Review v2 Eval\",\n \"custom_metrics\": {\n \"defaults\": [\n \"binary_correctness\",\n \"response_relevance\"\n ],\n \"custom\": [\n {\n \"name\": \"hallucination_check\",\n \"prompt\": \"Does the generated answer contain factually incorrect information? Rate from 0-1.\",\n \"parse_json\": true,\n \"score_min\": 0,\n \"score_max\": 1\n }\n ]\n },\n \"baseline_model_id\": \"model_01k4x9m2p7q3r9d2\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.nugen.in/api/v3/evaluations/create")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"model_id\": \"model_01k4x9m2p7q3r9d1\",\n \"benchmark_id\": \"benchmark_01k4x9m2p7q3r6a1\",\n \"evaluation_name\": \"Clause Review v2 Eval\",\n \"custom_metrics\": {\n \"defaults\": [\n \"binary_correctness\",\n \"response_relevance\"\n ],\n \"custom\": [\n {\n \"name\": \"hallucination_check\",\n \"prompt\": \"Does the generated answer contain factually incorrect information? Rate from 0-1.\",\n \"parse_json\": true,\n \"score_min\": 0,\n \"score_max\": 1\n }\n ]\n },\n \"baseline_model_id\": \"model_01k4x9m2p7q3r9d2\"\n}"
response = http.request(request)
puts response.read_body{
"evaluation_id": "evaluation_01k4x9m2p7q3r7b1",
"evaluation_name": "Clause Review v2 Eval",
"model_id": "model_01k4x9m2p7q3r9d1",
"status": "PROCESSING",
"benchmark_id": "benchmark_01k4x9m2p7q3r6a1",
"created_at": "2024-02-24T10:00:00Z",
"message": "Evaluation created successfully"
}{
"detail": "Model not found"
}{
"detail": "Model not found"
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>"
}
]
}Body
application/json
Start an evaluation of a model, or a comparison of two models, on a benchmark.
Response
Returns a unique identifier for the initiated evaluation along with the initial evaluation status. This endpoint starts an asynchronous evaluation using a specified benchmark and model(s), allowing users to track progress and retrieve results once completed.
Acknowledgement of a new evaluation.
Was this page helpful?