Skip to content

Commit 0739242

Browse files
authored
feat(cli): add telemetry client library (#930)
* feat(cli): add telemetry client library Signed-off-by: Matt Kornfield <mkornfield@nvidia.com> * fix(cli): address telemetry review feedback Signed-off-by: Matt Kornfield <mkornfield@nvidia.com> * chore(sdk): sync vendored telemetry files Signed-off-by: Matt Kornfield <mkornfield@nvidia.com> --------- Signed-off-by: Matt Kornfield <mkornfield@nvidia.com>
1 parent bd7ac5c commit 0739242

26 files changed

Lines changed: 3826 additions & 0 deletions

File tree

docs/cli/configuration.mdx

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ Environment variables override configuration file settings. This is useful for C
9393
| `NMP_OUTPUT_FORMAT` | Output format (table, json, yaml, csv, markdown) |
9494
| `NMP_TIMESTAMP_FORMAT` | Timestamp format (relative, iso8601) |
9595
| `NMP_COLOR_OUTPUT` | Enable/disable colored output (true/false) |
96+
| `NEMO_TELEMETRY_ENABLED` | Enable CLI telemetry only when set to `true`; any other explicit value disables it |
9697

9798
Example:
9899

@@ -115,6 +116,27 @@ Settings are resolved in this order (highest priority first):
115116

116117
This means you can set defaults in your config file and override them as needed with environment variables or flags.
117118

119+
## CLI Telemetry
120+
121+
The NeMo CLI sends anonymous usage telemetry by default to help improve setup, command reliability, and product workflows. The first CLI invocation with telemetry enabled prints a notice to stderr and creates a `telemetry-notice-shown` marker next to the CLI config file. Stdout is not changed, so scripts that parse command output remain stable.
122+
123+
Telemetry events include the command or workflow category, task status, duration, client version, a random session ID that rotates every 30 days, deployment type, and whether the command appears to be running in CI. Telemetry does not include prompts, model inputs or outputs, datasets, secrets, access tokens, configuration file contents, file contents, local paths, usernames, email addresses, or hostnames.
124+
125+
To disable telemetry for a single command or shell session:
126+
127+
```bash
128+
NEMO_TELEMETRY_ENABLED=false nemo models list
129+
export NEMO_TELEMETRY_ENABLED=false
130+
```
131+
132+
To disable telemetry persistently, add this top-level field to your CLI config file:
133+
134+
```yaml
135+
telemetry_enabled: false
136+
```
137+
138+
If `NEMO_TELEMETRY_ENABLED` is set, only `true` keeps the environment layer enabled. Any other explicit value disables telemetry for that process. A persisted `telemetry_enabled: false` value disables telemetry even when the environment variable is set to `true`.
139+
118140
## Shell Completion
119141

120142
The NeMo CLI supports tab completion for Bash, Zsh, and Fish shells. Enable it with:

packages/nemo_platform_ext/pyproject.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ dependencies = [
1616
"ngcsdk>=4.8.2",
1717
"nvidia-ml-py>=13.0.0",
1818
"psutil>=5.9.0",
19+
"httpx>=0.23.0,<1",
1920
]
2021
requires-python = ">=3.11,<3.15"
2122
readme = "README.md"
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
from nemo_platform_ext.cli.telemetry.handler import (
5+
QueuedEvent,
6+
TelemetryHandler,
7+
build_payload,
8+
)
9+
10+
__all__ = [
11+
"QueuedEvent",
12+
"TelemetryHandler",
13+
"build_payload",
14+
]
Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
"""Fire-and-flush emission with three opt-out layers and the first-run notice."""
4+
5+
from __future__ import annotations
6+
7+
import logging
8+
import sys
9+
from pathlib import Path
10+
11+
from nemo_platform_ext.cli.telemetry.events import PlatformTelemetryEvent
12+
from nemo_platform_ext.cli.telemetry.handler import TelemetryHandler, _telemetry_enabled
13+
from nemo_platform_ext.cli.telemetry.session import get_session_id
14+
15+
logger = logging.getLogger(__name__)
16+
17+
_invocation_opt_out = False
18+
19+
_NOTICE_TEXT = (
20+
"NeMo Platform CLI telemetry is on by default and sends anonymous usage data to improve the product. "
21+
"It does not send prompts, model inputs or outputs, datasets, secrets, file contents, or personal identifiers. "
22+
"Turn it off with NEMO_TELEMETRY_ENABLED=false or telemetry_enabled: false in the CLI config. "
23+
"Run nemo docs cli/configuration for details.\n"
24+
)
25+
26+
27+
def set_invocation_opt_out(value: bool) -> None:
28+
"""Per-invocation opt-out (e.g. a --no-telemetry flag on the current command)."""
29+
global _invocation_opt_out
30+
_invocation_opt_out = value
31+
32+
33+
def _config_opted_out() -> bool:
34+
"""True when the persisted config file sets ``telemetry_enabled: false``."""
35+
try:
36+
from nemo_platform_ext.config.config import Config
37+
38+
cfg = Config.load()
39+
return cfg.get_config_file().telemetry_enabled is False
40+
except Exception:
41+
# A privacy control must fail closed: if we cannot read the config to confirm
42+
# the user is opted in, treat them as opted out and do not send.
43+
logger.debug("Could not read telemetry opt-out config; failing closed (opted out)", exc_info=True)
44+
return True
45+
46+
47+
def telemetry_opted_in() -> bool:
48+
"""Opted in only when all three layers agree: per-invocation, env, and config."""
49+
if _invocation_opt_out:
50+
return False
51+
if not _telemetry_enabled():
52+
return False
53+
return not _config_opted_out()
54+
55+
56+
def _client_version() -> str:
57+
try:
58+
import nemo_platform
59+
60+
return nemo_platform.__version__
61+
except Exception:
62+
logger.debug("Could not resolve client version for telemetry", exc_info=True)
63+
return "undefined"
64+
65+
66+
def emit_event(event: PlatformTelemetryEvent) -> None:
67+
"""Best effort. Telemetry must never break a user command."""
68+
try:
69+
if not telemetry_opted_in():
70+
return
71+
# No retries on the CLI exit path: a synchronous send blocks the user's command,
72+
# so cap the worst case at one bounded send (SEND_TIMEOUT_SECONDS) rather than
73+
# retrying against a slow or unreachable endpoint while the user waits.
74+
handler = TelemetryHandler(source_client_version=_client_version(), session_id=get_session_id(), max_retries=0)
75+
handler.enqueue(event)
76+
handler.stop()
77+
except Exception:
78+
logger.debug("Failed to emit telemetry event", exc_info=True)
79+
80+
81+
def _notice_marker_path() -> Path:
82+
from nemo_platform_ext.config.config import Config
83+
84+
return Config.get_default_config_path().parent / "telemetry-notice-shown"
85+
86+
87+
def maybe_print_first_run_notice() -> None:
88+
"""Print the first-run notice to stderr once. Stdout stays machine-clean."""
89+
try:
90+
if not telemetry_opted_in():
91+
return
92+
marker = _notice_marker_path()
93+
if marker.exists():
94+
return
95+
marker.parent.mkdir(parents=True, exist_ok=True)
96+
marker.touch()
97+
sys.stderr.write(_NOTICE_TEXT)
98+
except Exception:
99+
logger.debug("Failed to print telemetry notice", exc_info=True)
Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
"""Platform usage-telemetry event models.
4+
5+
Field names and aliases follow the shared NeMo telemetry schema
6+
(aire/microservices/nemo-telemetry, schemas/anonymous_events.json, v1.9).
7+
"""
8+
9+
from __future__ import annotations
10+
11+
import os
12+
from enum import Enum
13+
from typing import ClassVar
14+
15+
from pydantic import BaseModel, ConfigDict, Field
16+
17+
_CI_ENV_VARS = (
18+
"CI",
19+
"GITLAB_CI",
20+
"GITHUB_ACTIONS",
21+
"BUILDKITE",
22+
"CIRCLECI",
23+
"JENKINS_URL",
24+
"TEAMCITY_VERSION",
25+
"TF_BUILD",
26+
"TRAVIS",
27+
)
28+
_FALSEY = ("", "0", "false", "no", "off")
29+
30+
31+
def is_ci_environment() -> bool:
32+
return any(os.getenv(v, "").lower() not in _FALSEY for v in _CI_ENV_VARS)
33+
34+
35+
class TaskStatusEnum(str, Enum):
36+
COMPLETED = "completed"
37+
ERROR = "error"
38+
CANCELED = "canceled"
39+
UNDEFINED = "undefined"
40+
41+
42+
class DeploymentTypeEnum(str, Enum):
43+
CLI = "cli"
44+
SDK = "sdk"
45+
NVIDIA_INTERNAL = "nvidia-internal"
46+
UNDEFINED = "undefined"
47+
48+
49+
def _deployment_type() -> DeploymentTypeEnum:
50+
raw = os.getenv("NEMO_DEPLOYMENT_TYPE", "cli").lower()
51+
try:
52+
return DeploymentTypeEnum(raw)
53+
except ValueError:
54+
return DeploymentTypeEnum.UNDEFINED
55+
56+
57+
class PlatformTelemetryEvent(BaseModel):
58+
"""Base for all platform events. extra="forbid" is a privacy guard."""
59+
60+
model_config = ConfigDict(extra="forbid", populate_by_name=True)
61+
62+
_event_name: ClassVar[str] = "undefined"
63+
_schema_version: ClassVar[str] = "1.9"
64+
65+
nemo_source: str = Field(default="platform", serialization_alias="nemoSource")
66+
task_status: TaskStatusEnum = Field(serialization_alias="taskStatus")
67+
deployment_type: DeploymentTypeEnum = Field(default_factory=_deployment_type, serialization_alias="deploymentType")
68+
is_ci: bool = Field(default_factory=is_ci_environment, serialization_alias="isCi")
69+
70+
71+
class OnboardingStepEvent(PlatformTelemetryEvent):
72+
_event_name: ClassVar[str] = "onboarding_step"
73+
74+
step: str
75+
provider_type: str = Field(default="undefined", serialization_alias="providerType")
76+
models_discovered_bucket: str = Field(default="undefined", serialization_alias="modelsDiscoveredBucket")
77+
skills_target: str = Field(default="undefined", serialization_alias="skillsTarget")
78+
agent_deployed: bool = Field(default=False, serialization_alias="agentDeployed")
79+
80+
81+
class CommandInvokedEvent(PlatformTelemetryEvent):
82+
_event_name: ClassVar[str] = "command_invoked"
83+
84+
command: str
85+
duration_sec: float = Field(serialization_alias="durationSec")
86+
agent_mode: bool = Field(default=False, serialization_alias="agentMode")
87+
88+
89+
class JobRunEvent(PlatformTelemetryEvent):
90+
_event_name: ClassVar[str] = "job_run"
91+
92+
job_type: str = Field(serialization_alias="jobType")
93+
duration_sec: float = Field(default=-1.0, serialization_alias="durationSec")
94+
plugins: list[str] = Field(default_factory=list)
95+
model: str = "undefined"
96+
input_tokens: int = Field(default=-1, serialization_alias="inputTokens")
97+
output_tokens: int = Field(default=-1, serialization_alias="outputTokens")

0 commit comments

Comments
 (0)