RomanNum3ral
/
hermes-agent
mirror of https://github.com/NousResearch/hermes-agent.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(observability): preserve shared metrics compatibility 

Signed-off-by: Alex Fournier <afournier@nvidia.com>
This commit is contained in:
Alex Fournier 2026-07-29 14:09:47 -07:00
parent a0476b3605
commit dfb8c1bd4d
10 changed files with 686 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/observability/relay-shared-metrics.md
View File

@ -74,6 +74,9 @@ lowercased and structurally bounded, but they are not normalized through a
checked-in model catalog. Pricing and model-family classification belong to
the metrics backend. Prompts, responses, endpoints, errors, session IDs, task
IDs, and request IDs are not included in the metrics event or package.
New calls use `hermes.model_route.count`. The previous
`hermes.model_call.count` contract remains readable only so pending local
counters created by older builds can be exported without losing data.
Each task run is a Relay `Function` scope named `hermes.task_run`, parented to
the owning Hermes session. The start counter contains only bounded execution
@ -99,6 +102,9 @@ files are immutable delta documents that conform to a closed JSON schema and
are written with atomic replacement. Fully packaged aggregate rows and
successfully exported package rows and files are retained locally for 30 days.
Pending package rows and counters with unexported deltas are never pruned.
Package schema v1 remains unchanged for existing outbox files. New packages
use v2, which accepts both the retired model-call contract and the current
model-route contract so upgrades can drain pending counters safely.
Each package contains an `install_id` generated as a random UUID. Despite the
schema field name, its current scope is one `HERMES_HOME`, so it is more

66
hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json
View File

@ -94,21 +94,63 @@
"type": "object",
"additionalProperties": false,
"required": [
"model",
"provider"
"call_role",
"locality",
"model_family",
"outcome",
"provider_family"
],
"properties": {
"model": {
"type": "string",
"minLength": 1,
"maxLength": 256,
"pattern": "^[a-z0-9][a-z0-9._:/@+\\-]*$"
"call_role": {
"const": "primary"
},
"provider": {
"type": "string",
"minLength": 1,
"maxLength": 64,
"pattern": "^[a-z0-9][a-z0-9._:/@+\\-]*$"
"locality": {
"enum": [
"local",
"remote",
"unknown"
]
},
"model_family": {
"enum": [
"claude",
"deepseek",
"gemini",
"gemma",
"glm",
"gpt",
"grok",
"kimi",
"llama",
"minimax",
"mimo",
"mistral",
"nemotron",
"nova",
"o1",
"o3",
"o4",
"qwen",
"step",
"trinity",
"unknown"
]
},
"outcome": {
"enum": [
"cancelled",
"failed",
"success"
]
},
"provider_family": {
"enum": [
"aggregator",
"custom",
"direct",
"local",
"unknown"
]
}
}
},

385
hermes_cli/observability/schemas/hermes.shared_metrics.v2.schema.json Normal file
View File

@ -0,0 +1,385 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "urn:hermes-agent:schema:shared-metrics:v2",
"title": "Hermes Shared Metrics Package v2",
"type": "object",
"additionalProperties": false,
"required": [
"schema_version",
"package_id",
"install_id",
"period_start",
"period_end",
"generated_at",
"resource",
"metrics"
],
"properties": {
"schema_version": {
"const": "hermes.shared_metrics.v2"
},
"package_id": {
"$ref": "#/$defs/uuid"
},
"install_id": {
"description": "Random persistent identifier scoped to one HERMES_HOME; local-only in schema v2.",
"$ref": "#/$defs/uuid"
},
"period_start": {
"type": "string",
"format": "date-time"
},
"period_end": {
"type": "string",
"format": "date-time"
},
"generated_at": {
"type": "string",
"format": "date-time"
},
"resource": {
"type": "object",
"additionalProperties": false,
"required": [
"hermes_version"
],
"properties": {
"hermes_version": {
"type": "string",
"minLength": 1,
"maxLength": 64
}
}
},
"metrics": {
"type": "array",
"minItems": 1,
"items": {
"oneOf": [
{
"$ref": "#/$defs/model_call_counter"
},
{
"$ref": "#/$defs/model_route_counter"
},
{
"$ref": "#/$defs/task_started_counter"
},
{
"$ref": "#/$defs/task_finished_counter"
}
]
}
}
},
"$defs": {
"uuid": {
"type": "string",
"pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
},
"model_call_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.model_call.count"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"call_role",
"locality",
"model_family",
"outcome",
"provider_family"
],
"properties": {
"call_role": {
"const": "primary"
},
"locality": {
"enum": [
"local",
"remote",
"unknown"
]
},
"model_family": {
"enum": [
"claude",
"deepseek",
"gemini",
"gemma",
"glm",
"gpt",
"grok",
"kimi",
"llama",
"minimax",
"mimo",
"mistral",
"nemotron",
"nova",
"o1",
"o3",
"o4",
"qwen",
"step",
"trinity",
"unknown"
]
},
"outcome": {
"enum": [
"cancelled",
"failed",
"success"
]
},
"provider_family": {
"enum": [
"aggregator",
"custom",
"direct",
"local",
"unknown"
]
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"model_route_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.model_route.count"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"model",
"provider"
],
"properties": {
"model": {
"type": "string",
"minLength": 1,
"maxLength": 256,
"pattern": "^[a-z0-9][a-z0-9._:/@+\\-]*$"
},
"provider": {
"type": "string",
"minLength": 1,
"maxLength": 64,
"pattern": "^[a-z0-9][a-z0-9._:/@+\\-]*$"
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"task_started_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.task_run.started"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"entrypoint",
"execution_surface"
],
"properties": {
"entrypoint": {
"$ref": "#/$defs/task_entrypoint"
},
"execution_surface": {
"$ref": "#/$defs/execution_surface"
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"task_finished_counter": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"type",
"dimensions",
"value"
],
"properties": {
"name": {
"const": "hermes.task_run.finished"
},
"type": {
"const": "counter"
},
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": [
"duration_bucket",
"end_reason",
"entrypoint",
"execution_surface",
"model_call_count_bucket",
"outcome",
"retry_count_bucket",
"termination",
"tool_call_count_bucket"
],
"properties": {
"duration_bucket": {
"$ref": "#/$defs/duration_bucket"
},
"end_reason": {
"enum": [
"approval_denied",
"completed",
"failed",
"guardrail_blocked",
"iteration_limit",
"system_aborted",
"timed_out",
"unknown",
"user_cancelled"
]
},
"entrypoint": {
"$ref": "#/$defs/task_entrypoint"
},
"execution_surface": {
"$ref": "#/$defs/execution_surface"
},
"model_call_count_bucket": {
"$ref": "#/$defs/count_bucket"
},
"outcome": {
"enum": [
"cancelled",
"failed",
"success",
"timed_out",
"unknown"
]
},
"retry_count_bucket": {
"$ref": "#/$defs/count_bucket"
},
"termination": {
"enum": [
"none",
"system_aborted",
"timed_out",
"unknown",
"user_cancelled"
]
},
"tool_call_count_bucket": {
"$ref": "#/$defs/count_bucket"
}
}
},
"value": {
"type": "integer",
"minimum": 1
}
}
},
"execution_surface": {
"enum": [
"api",
"batch",
"cli",
"desktop",
"gateway",
"other",
"python",
"scheduled_task",
"tui",
"unknown"
]
},
"task_entrypoint": {
"enum": [
"api",
"background",
"batch",
"delegated",
"gateway_message",
"interactive",
"other",
"python",
"scheduled_task",
"unknown"
]
},
"duration_bucket": {
"enum": [
"1s_to_5s",
"2m_to_10m",
"30s_to_2m",
"5s_to_30s",
"gte_10m",
"lt_1s"
]
},
"count_bucket": {
"enum": [
"0",
"1",
"2",
"3_to_5",
"6_to_10",
"gte_11"
]
}
}
}

6
hermes_cli/observability/shared_metrics.py
View File

@ -18,12 +18,12 @@ from utils import atomic_json_write
from .shared_metrics_contract import (
COUNTER_METRICS,
MODEL_CALL_METRIC,
MODEL_ROUTE_METRIC,
counter_dimensions_are_valid,
)
_PACKAGE_SCHEMA_VERSION = "hermes.shared_metrics.v1"
_PACKAGE_SCHEMA_VERSION = "hermes.shared_metrics.v2"
_STORE_SCHEMA_VERSION = "1"
_BUSY_TIMEOUT_MS = 250
_SCHEMA_BUSY_TIMEOUT_MS = 5_000
@ -62,7 +62,7 @@ class SharedMetricsStore:
hermes_version: str,
) -> None:
"""Increment the terminal model-call counter for the current UTC day."""
self.record_counter(MODEL_CALL_METRIC, dimensions, hermes_version)
self.record_counter(MODEL_ROUTE_METRIC, dimensions, hermes_version)
def record_counter(
self,

58
hermes_cli/observability/shared_metrics_contract.py
View File

@ -7,12 +7,13 @@ from typing import Any
from agent.relay_runtime import RUNTIME_INSTANCE_KEY
SCHEMA_KEY = "hermes.metrics.schema_version"
SCHEMA_VERSION = "hermes.metrics.event.v1"
SCHEMA_VERSION = "hermes.metrics.event.v2"
MODEL_CALL_SCOPE = "hermes.model_call"
MODEL_CALL_PROFILE_MODEL = "unknown"
TASK_SCOPE = "hermes.task_run"
SUBSCRIBER_NAME = "hermes.nemo_relay.shared_metrics"
MODEL_CALL_METRIC = "hermes.model_call.count"
LEGACY_MODEL_CALL_METRIC = "hermes.model_call.count"
MODEL_ROUTE_METRIC = "hermes.model_route.count"
TASK_STARTED_METRIC = "hermes.task_run.started"
TASK_FINISHED_METRIC = "hermes.task_run.finished"
MODEL_IDENTIFIER_MAX_LENGTH = 256
@ -90,7 +91,48 @@ COUNT_BUCKETS: frozenset[str] = frozenset({
"gte_11",
})
_LEGACY_PROVIDER_FAMILIES = frozenset({
"aggregator",
"custom",
"direct",
"local",
"unknown",
})
_LEGACY_MODEL_LOCALITIES = frozenset({"local", "remote", "unknown"})
_LEGACY_MODEL_OUTCOMES = frozenset({"cancelled", "failed", "success"})
_LEGACY_MODEL_FAMILIES = frozenset({
"claude",
"deepseek",
"gemini",
"gemma",
"glm",
"gpt",
"grok",
"kimi",
"llama",
"minimax",
"mimo",
"mistral",
"nemotron",
"nova",
"o1",
"o3",
"o4",
"qwen",
"step",
"trinity",
"unknown",
})
_COUNTER_DIMENSION_VALUES: dict[str, dict[str, frozenset[str]]] = {
# Retained only so pre-v2 pending rows remain packageable.
LEGACY_MODEL_CALL_METRIC: {
"call_role": frozenset({"primary"}),
"locality": _LEGACY_MODEL_LOCALITIES,
"model_family": _LEGACY_MODEL_FAMILIES,
"outcome": _LEGACY_MODEL_OUTCOMES,
"provider_family": _LEGACY_PROVIDER_FAMILIES,
},
TASK_STARTED_METRIC: {
"entrypoint": TASK_ENTRYPOINTS,
"execution_surface": EXECUTION_SURFACES,
@ -107,9 +149,11 @@ _COUNTER_DIMENSION_VALUES: dict[str, dict[str, frozenset[str]]] = {
"tool_call_count_bucket": COUNT_BUCKETS,
},
}
COUNTER_METRICS: frozenset[str] = frozenset(
{*_COUNTER_DIMENSION_VALUES, MODEL_CALL_METRIC}
)
COUNTER_METRICS: frozenset[str] = frozenset({
MODEL_ROUTE_METRIC,
TASK_FINISHED_METRIC,
TASK_STARTED_METRIC,
})
def counter_dimensions_are_valid(
@ -117,7 +161,7 @@ def counter_dimensions_are_valid(
dimensions: dict[str, Any],
) -> bool:
"""Return whether dimensions match one closed shared-metric contract."""
if metric_name == MODEL_CALL_METRIC:
if metric_name == MODEL_ROUTE_METRIC:
return (
set(dimensions) == {"model", "provider"}
and dimensions["model"]
@ -172,7 +216,7 @@ def model_call_dimensions(event: Any) -> dict[str, str] | None:
if not isinstance(data, dict) or set(data) != expected_fields:
return None
dimensions = {field: data.get(field) for field in sorted(expected_fields)}
if not counter_dimensions_are_valid(MODEL_CALL_METRIC, dimensions):
if not counter_dimensions_are_valid(MODEL_ROUTE_METRIC, dimensions):
return None
return dimensions

4
hermes_cli/observability/shared_metrics_subscriber.py
View File

@ -9,7 +9,7 @@ from typing import Any
from agent.relay_runtime import RUNTIME_INSTANCE_KEY
from .shared_metrics import SharedMetricsStore
from .shared_metrics_contract import MODEL_CALL_METRIC, model_call_dimensions, task_counter
from .shared_metrics_contract import MODEL_ROUTE_METRIC, model_call_dimensions, task_counter
logger = logging.getLogger(__name__)
@ -44,7 +44,7 @@ class SharedMetricsSubscriber:
):
return
dimensions = model_call_dimensions(event)
metric_name = MODEL_CALL_METRIC
metric_name = MODEL_ROUTE_METRIC
if dimensions is None:
task_metric = task_counter(event)
if task_metric is None:

16
scripts/smoke_nemo_relay_shared_metrics.py
View File

@ -222,7 +222,7 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:
]
by_name = {counter["name"]: counter for counter in counters}
if set(by_name) != {
"hermes.model_call.count",
"hermes.model_route.count",
"hermes.task_run.finished",
"hermes.task_run.started",
}:
@ -230,7 +230,7 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:
f"Unexpected SQLite counters:\n{json.dumps(counters, indent=2)}"
)
expected_model = {
"name": "hermes.model_call.count",
"name": "hermes.model_route.count",
"dimensions": {
"model": MODEL_CANARY,
"provider": "custom",
@ -238,9 +238,9 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]:
"value": 1,
"packaged_value": 1,
}
if by_name["hermes.model_call.count"] != expected_model:
if by_name["hermes.model_route.count"] != expected_model:
raise AssertionError(
f"Unexpected model counter: {by_name['hermes.model_call.count']}"
f"Unexpected model counter: {by_name['hermes.model_route.count']}"
)
expected_start = {
"name": "hermes.task_run.started",
@ -299,20 +299,20 @@ def _validate_package(outbox: Path, schema_path: Path) -> tuple[Path, dict[str,
)
metrics = {metric["name"]: metric for metric in package.get("metrics", [])}
if set(metrics) != {
"hermes.model_call.count",
"hermes.model_route.count",
"hermes.task_run.finished",
"hermes.task_run.started",
}:
raise AssertionError(
f"Unexpected package metrics:\n{json.dumps(package.get('metrics'), indent=2)}"
)
model_dimensions = metrics["hermes.model_call.count"]["dimensions"]
model_dimensions = metrics["hermes.model_route.count"]["dimensions"]
if model_dimensions != {
"model": MODEL_CANARY,
"provider": "custom",
}:
raise AssertionError(
f"Unexpected model metric: {metrics['hermes.model_call.count']}"
f"Unexpected model metric: {metrics['hermes.model_route.count']}"
)
terminal = metrics["hermes.task_run.finished"]
if terminal["dimensions"] != {
@ -422,7 +422,7 @@ def main() -> int:
/ "hermes_cli"
/ "observability"
/ "schemas"
/ "hermes.shared_metrics.v1.schema.json",
/ "hermes.shared_metrics.v2.schema.json",
)
print("Hermes -> NeMo Relay shared-metrics smoke test passed")

181
tests/hermes_cli/test_relay_shared_metrics.py
View File

@ -24,9 +24,13 @@ from hermes_cli.observability.shared_metrics_contract import (
COUNT_BUCKETS,
DURATION_BUCKETS,
EXECUTION_SURFACES,
LEGACY_MODEL_CALL_METRIC,
MODEL_CALL_PROFILE_MODEL,
MODEL_IDENTIFIER_MAX_LENGTH,
MODEL_ROUTE_METRIC,
PROVIDER_IDENTIFIER_MAX_LENGTH,
SCHEMA_KEY,
SCHEMA_VERSION,
TASK_END_REASONS,
TASK_ENTRYPOINTS,
TASK_OUTCOMES,
@ -48,13 +52,14 @@ SCHEMA_PATH = (
/ "hermes_cli"
/ "observability"
/ "schemas"
/ "hermes.shared_metrics.v1.schema.json"
/ "hermes.shared_metrics.v2.schema.json"
)
LEGACY_SCHEMA_PATH = SCHEMA_PATH.with_name("hermes.shared_metrics.v1.schema.json")
def _schema_validator():
def _schema_validator(path: Path = SCHEMA_PATH):
jsonschema = pytest.importorskip("jsonschema")
schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8"))
schema = json.loads(path.read_text(encoding="utf-8"))
jsonschema.Draft202012Validator.check_schema(schema)
return jsonschema.Draft202012Validator(
schema,
@ -64,7 +69,7 @@ def _schema_validator():
def _package_dimension_schema() -> dict[str, object]:
schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8"))
return schema["$defs"]["model_call_counter"]["properties"]["dimensions"]
return schema["$defs"]["model_route_counter"]["properties"]["dimensions"]
def _task_dimension_schema(kind: str) -> dict[str, object]:
@ -79,6 +84,16 @@ def _dimensions() -> dict[str, str]:
}
def _legacy_dimensions() -> dict[str, str]:
return {
"call_role": "primary",
"locality": "remote",
"model_family": "claude",
"outcome": "success",
"provider_family": "direct",
}
def _record_model_calls_in_process(
database_path: str,
outbox_directory: str,
@ -106,11 +121,11 @@ def test_model_call_counter_survives_restart_and_exports_only_new_deltas(tmp_pat
_schema_validator().validate(first_package)
uuid.UUID(first_package["package_id"])
uuid.UUID(first_package["install_id"])
assert first_package["schema_version"] == "hermes.shared_metrics.v1"
assert first_package["schema_version"] == "hermes.shared_metrics.v2"
assert first_package["resource"] == {"hermes_version": "test-version"}
assert first_package["metrics"] == [
{
"name": "hermes.model_call.count",
"name": MODEL_ROUTE_METRIC,
"type": "counter",
"dimensions": _dimensions(),
"value": 2,
@ -135,6 +150,120 @@ def test_model_call_counter_survives_restart_and_exports_only_new_deltas(tmp_pat
assert restarted.counter_snapshot()[0]["packaged_value"] == 3
def test_v2_package_preserves_pending_v1_model_counters(tmp_path):
database_path = tmp_path / "metrics.sqlite3"
outbox_directory = tmp_path / "outbox"
store = SharedMetricsStore(database_path, outbox_directory)
period_start = shared_metrics_module._utc_now().date().isoformat()
legacy_dimensions_json = json.dumps(
_legacy_dimensions(),
sort_keys=True,
separators=(",", ":"),
)
with sqlite3.connect(database_path) as connection:
connection.execute(
"""
INSERT INTO counter_aggregates(
period_start,
metric_name,
hermes_version,
dimensions_json,
value,
packaged_value
) VALUES (?, ?, ?, ?, 3, 0)
""",
(
period_start,
LEGACY_MODEL_CALL_METRIC,
"test-version",
legacy_dimensions_json,
),
)
store.record_model_call(_dimensions(), "test-version")
[package_path] = store.create_and_export_package()
package = json.loads(package_path.read_text(encoding="utf-8"))
_schema_validator().validate(package)
assert package["schema_version"] == "hermes.shared_metrics.v2"
assert package["metrics"] == [
{
"name": LEGACY_MODEL_CALL_METRIC,
"type": "counter",
"dimensions": _legacy_dimensions(),
"value": 3,
},
{
"name": MODEL_ROUTE_METRIC,
"type": "counter",
"dimensions": _dimensions(),
"value": 1,
},
]
assert all(
row["value"] == row["packaged_value"] for row in store.counter_snapshot()
)
def test_v1_outbox_package_exports_unchanged_after_upgrade(tmp_path):
database_path = tmp_path / "metrics.sqlite3"
outbox_directory = tmp_path / "outbox"
store = SharedMetricsStore(database_path, outbox_directory)
package_id = str(uuid.uuid4())
payload = {
"schema_version": "hermes.shared_metrics.v1",
"package_id": package_id,
"install_id": str(uuid.uuid4()),
"period_start": "2026-07-28T00:00:00Z",
"period_end": "2026-07-29T00:00:00Z",
"generated_at": "2026-07-29T01:00:00Z",
"resource": {"hermes_version": "legacy-version"},
"metrics": [
{
"name": LEGACY_MODEL_CALL_METRIC,
"type": "counter",
"dimensions": _legacy_dimensions(),
"value": 2,
}
],
}
payload_json = json.dumps(payload, sort_keys=True, separators=(",", ":"))
_schema_validator(LEGACY_SCHEMA_PATH).validate(payload)
with sqlite3.connect(database_path) as connection:
connection.execute(
"""
INSERT INTO package_outbox(
package_id,
period_start,
period_end,
payload_json,
created_at
) VALUES (?, ?, ?, ?, ?)
""",
(
package_id,
payload["period_start"],
payload["period_end"],
payload_json,
payload["generated_at"],
),
)
assert store.create_and_export_package() == [
outbox_directory / f"{package_id}.json"
]
exported = json.loads(
(outbox_directory / f"{package_id}.json").read_text(encoding="utf-8")
)
assert exported == payload
with sqlite3.connect(database_path) as connection:
[persisted_payload_json] = connection.execute(
"SELECT payload_json FROM package_outbox WHERE package_id = ?",
(package_id,),
).fetchone()
assert persisted_payload_json == payload_json
def test_due_export_runs_once_per_utc_day_and_catches_up_pending_deltas(
tmp_path, monkeypatch
):
@ -152,7 +281,7 @@ def test_due_export_runs_once_per_utc_day_and_catches_up_pending_deltas(
assert len(list((tmp_path / "outbox").glob("*.json"))) == 1
assert store.counter_snapshot()[0] == {
"period_start": "2026-07-28",
"metric_name": "hermes.model_call.count",
"metric_name": MODEL_ROUTE_METRIC,
"hermes_version": "test-version",
"dimensions": _dimensions(),
"value": 2,
@ -173,8 +302,10 @@ def test_due_export_runs_once_per_utc_day_and_catches_up_pending_deltas(
def test_package_schema_matches_the_model_call_contract():
schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8"))
properties = _package_dimension_schema()["properties"]
assert schema["properties"]["schema_version"]["const"] == "hermes.shared_metrics.v2"
assert set(properties) == {"model", "provider"}
assert properties["model"]["maxLength"] == MODEL_IDENTIFIER_MAX_LENGTH
assert properties["provider"]["maxLength"] == PROVIDER_IDENTIFIER_MAX_LENGTH
@ -182,6 +313,21 @@ def test_package_schema_matches_the_model_call_contract():
assert "enum" not in properties["provider"]
def test_v1_package_schema_retains_the_legacy_model_contract():
schema = json.loads(LEGACY_SCHEMA_PATH.read_text(encoding="utf-8"))
model_counter = schema["$defs"]["model_call_counter"]
assert schema["properties"]["schema_version"]["const"] == "hermes.shared_metrics.v1"
assert model_counter["properties"]["name"]["const"] == LEGACY_MODEL_CALL_METRIC
assert set(model_counter["properties"]["dimensions"]["properties"]) == {
"call_role",
"locality",
"model_family",
"outcome",
"provider_family",
}
def test_package_schema_matches_the_task_contract():
schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8"))
start = _task_dimension_schema("task_started_counter")["properties"]
@ -377,7 +523,7 @@ def test_subscriber_contract_rejects_unknown_fields_and_dimension_values():
category_profile={"model_name": MODEL_CALL_PROFILE_MODEL},
name="hermes.model_call",
scope_category="end",
metadata={"hermes.metrics.schema_version": "hermes.metrics.event.v1"},
metadata={SCHEMA_KEY: SCHEMA_VERSION},
data=_dimensions(),
)
@ -410,7 +556,7 @@ def test_task_subscriber_contract_accepts_only_bounded_scope_events():
category_profile=None,
name="hermes.task_run",
scope_category="start",
metadata={"hermes.metrics.schema_version": "hermes.metrics.event.v1"},
metadata={SCHEMA_KEY: SCHEMA_VERSION},
data={"entrypoint": "interactive", "execution_surface": "cli"},
)
assert task_counter(start) == (
@ -534,7 +680,7 @@ def test_store_rejects_dimensions_outside_the_metric_contract(tmp_path):
with pytest.raises(ValueError, match="Unsupported dimensions"):
store.record_counter(
"hermes.model_call.count",
MODEL_ROUTE_METRIC,
{"prompt": "must-not-be-persisted"},
"test-version",
)
@ -542,6 +688,19 @@ def test_store_rejects_dimensions_outside_the_metric_contract(tmp_path):
assert store.counter_snapshot() == []
def test_store_does_not_record_the_retired_model_metric(tmp_path):
store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox")
with pytest.raises(ValueError, match="Unsupported shared metric"):
store.record_counter(
LEGACY_MODEL_CALL_METRIC,
_legacy_dimensions(),
"test-version",
)
assert store.counter_snapshot() == []
def test_package_builder_rejects_tampered_dimensions(tmp_path):
database_path = tmp_path / "metrics.sqlite3"
outbox_directory = tmp_path / "outbox"
@ -708,7 +867,7 @@ def test_package_export_does_not_chase_concurrent_updates(tmp_path, monkeypatch)
assert create_calls == 1
assert len(first_paths) == 1
[counter] = store.counter_snapshot()
assert counter["metric_name"] == "hermes.model_call.count"
assert counter["metric_name"] == MODEL_ROUTE_METRIC
assert counter["dimensions"] == _dimensions()
assert counter["value"] == 2
assert counter["packaged_value"] == 1

12
tests/hermes_cli/test_relay_shared_metrics_runtime.py
View File

@ -279,15 +279,15 @@ def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_pa
package = json.loads(packages[0].read_text(encoding="utf-8"))
metrics = {metric["name"]: metric for metric in package["metrics"]}
assert set(metrics) == {
"hermes.model_call.count",
"hermes.model_route.count",
"hermes.task_run.finished",
"hermes.task_run.started",
}
assert metrics["hermes.model_call.count"]["dimensions"] == {
assert metrics["hermes.model_route.count"]["dimensions"] == {
"model": "claude-sonnet",
"provider": "anthropic",
}
assert metrics["hermes.model_call.count"]["value"] == 1
assert metrics["hermes.model_route.count"]["value"] == 1
assert metrics["hermes.task_run.started"] == {
"name": "hermes.task_run.started",
"type": "counter",
@ -432,8 +432,8 @@ def test_real_binding_drives_lifecycle_aggregation_export_and_snapshot(
assert len(by_metric["hermes.task_run.started"]) == 1
assert by_metric["hermes.task_run.started"][0]["value"] == 3
assert len(by_metric["hermes.model_call.count"]) == 1
model_counter = by_metric["hermes.model_call.count"][0]
assert len(by_metric["hermes.model_route.count"]) == 1
model_counter = by_metric["hermes.model_route.count"][0]
assert model_counter["dimensions"] == {
"model": model_canary,
"provider": "custom",
@ -468,7 +468,7 @@ def test_real_binding_drives_lifecycle_aggregation_export_and_snapshot(
json.loads(package.read_text(encoding="utf-8")) for package in packages
]
for package in package_payloads:
assert package["schema_version"] == "hermes.shared_metrics.v1"
assert package["schema_version"] == "hermes.shared_metrics.v2"
for metric in package["metrics"]:
key = (metric["name"], tuple(sorted(metric["dimensions"].items())))
package_values[key] = package_values.get(key, 0) + metric["value"]

2
tests/plugins/test_nemo_relay_plugin.py
View File

@ -466,7 +466,7 @@ def test_shared_metrics_and_rich_plugin_share_one_core_session(
)
assert len(packages) == 1
package = json.loads(packages[0].read_text(encoding="utf-8"))
assert package["metrics"][0]["name"] == "hermes.model_call.count"
assert package["metrics"][0]["name"] == "hermes.model_route.count"
assert package["metrics"][0]["value"] == 1
assert (tmp_path / "atif" / "hermes-atif-s1.json").exists()