Skip to content

Commit 0b03104

Browse files
gary-huanggary-huang
authored
Add LLMObs configuration (#8076)
* add LLM obs configs * refactor & centralize api key checks * rename LLM_OBS consts to be more aligned with other languages * undo jmx fetch changes
1 parent 58a9ee3 commit 0b03104

File tree

7 files changed

+212
-11
lines changed

7 files changed

+212
-11
lines changed

‎dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,9 @@ public static Set<InstrumenterModule.TargetSystem> getEnabledSystems() {
311311
if (cfg.isUsmEnabled()) {
312312
enabledSystems.add(InstrumenterModule.TargetSystem.USM);
313313
}
314+
if (cfg.isLlmObsEnabled()) {
315+
enabledSystems.add(InstrumenterModule.TargetSystem.LLMOBS);
316+
}
314317
return enabledSystems;
315318
}
316319

‎dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,8 @@ public enum TargetSystem {
4949
APPSEC,
5050
IAST,
5151
CIVISIBILITY,
52-
USM
52+
USM,
53+
LLMOBS,
5354
}
5455

5556
private static final Logger log = LoggerFactory.getLogger(InstrumenterModule.class);

‎dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,9 @@ public final class ConfigDefaults {
137137

138138
static final boolean DEFAULT_IAST_STACK_TRACE_ENABLED = true;
139139

140+
static final boolean DEFAULT_LLM_OBS_ENABLED = false;
141+
static final boolean DEFAULT_LLM_OBS_AGENTLESS_ENABLED = false;
142+
140143
static final boolean DEFAULT_USM_ENABLED = false;
141144

142145
static final boolean DEFAULT_CIVISIBILITY_ENABLED = false;

‎dd-trace-api/src/main/java/datadog/trace/api/config/LlmObsConfig.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package datadog.trace.api.config;
2+
3+
/**
4+
* Constant with names of configuration options for LLM Observability. (EXPERIMENTAL AND SUBJECT TO
5+
* CHANGE)
6+
*/
7+
public final class LlmObsConfig {
8+
9+
public static final String LLMOBS_ENABLED = "llmobs.enabled";
10+
11+
public static final String LLMOBS_ML_APP = "llmobs.ml.app";
12+
13+
public static final String LLMOBS_AGENTLESS_ENABLED = "llmobs.agentless.enabled";
14+
15+
private LlmObsConfig() {}
16+
}

‎internal-api/src/main/java/datadog/trace/api/Config.java‎

Lines changed: 55 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import static datadog.trace.api.config.GeneralConfig.SERVICE_NAME;
1414
import static datadog.trace.api.config.IastConfig.*;
1515
import static datadog.trace.api.config.JmxFetchConfig.*;
16+
import static datadog.trace.api.config.LlmObsConfig.*;
1617
import static datadog.trace.api.config.ProfilingConfig.*;
1718
import static datadog.trace.api.config.RemoteConfigConfig.*;
1819
import static datadog.trace.api.config.TraceInstrumentationConfig.*;
@@ -309,6 +310,9 @@ public static String getHostName() {
309310
private final boolean iastExperimentalPropagationEnabled;
310311
private final String iastSecurityControlsConfiguration;
311312

313+
private final boolean llmObsAgentlessEnabled;
314+
private final String llmObsMlApp;
315+
312316
private final boolean ciVisibilityTraceSanitationEnabled;
313317
private final boolean ciVisibilityAgentlessEnabled;
314318
private final String ciVisibilityAgentlessUrl;
@@ -1336,6 +1340,10 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
13361340
iastSecurityControlsConfiguration =
13371341
configProvider.getString(IAST_SECURITY_CONTROLS_CONFIGURATION, null);
13381342

1343+
llmObsAgentlessEnabled =
1344+
configProvider.getBoolean(LLMOBS_AGENTLESS_ENABLED, DEFAULT_LLM_OBS_AGENTLESS_ENABLED);
1345+
llmObsMlApp = configProvider.getString(LLMOBS_ML_APP);
1346+
13391347
ciVisibilityTraceSanitationEnabled =
13401348
configProvider.getBoolean(CIVISIBILITY_TRACE_SANITATION_ENABLED, true);
13411349

@@ -1766,21 +1774,46 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
17661774
this.traceFlushIntervalSeconds =
17671775
configProvider.getFloat(
17681776
TracerConfig.TRACE_FLUSH_INTERVAL, ConfigDefaults.DEFAULT_TRACE_FLUSH_INTERVAL);
1769-
if (profilingAgentless && apiKey == null) {
1770-
log.warn(
1771-
"Agentless profiling activated but no api key provided. Profile uploading will likely fail");
1772-
}
17731777

17741778
this.tracePostProcessingTimeout =
17751779
configProvider.getLong(
17761780
TRACE_POST_PROCESSING_TIMEOUT, ConfigDefaults.DEFAULT_TRACE_POST_PROCESSING_TIMEOUT);
17771781

1778-
if (isCiVisibilityEnabled()
1779-
&& ciVisibilityAgentlessEnabled
1780-
&& (apiKey == null || apiKey.isEmpty())) {
1781-
throw new FatalAgentMisconfigurationError(
1782-
"Attempt to start in Agentless mode without API key. "
1783-
+ "Please ensure that either an API key is configured, or the tracer is set up to work with the Agent");
1782+
if (isLlmObsEnabled()) {
1783+
log.debug("Attempting to enable LLM Observability");
1784+
if (llmObsMlApp == null || llmObsMlApp.isEmpty()) {
1785+
throw new IllegalArgumentException(
1786+
"Attempt to enable LLM Observability without ML app defined."
1787+
+ "Please ensure that the name of the ML app is provided through properties or env variable");
1788+
}
1789+
1790+
log.debug(
1791+
"LLM Observability enabled for ML app {}, agentless mode {}",
1792+
llmObsMlApp,
1793+
llmObsAgentlessEnabled);
1794+
}
1795+
1796+
// if API key is not provided, check if any products are using agentless mode and require it
1797+
if (apiKey == null || apiKey.isEmpty()) {
1798+
// CI Visibility
1799+
if (isCiVisibilityEnabled() && ciVisibilityAgentlessEnabled) {
1800+
throw new FatalAgentMisconfigurationError(
1801+
"Attempt to start in CI Visibility in Agentless mode without API key. "
1802+
+ "Please ensure that either an API key is configured, or the tracer is set up to work with the Agent");
1803+
}
1804+
1805+
// Profiling
1806+
if (profilingAgentless) {
1807+
log.warn(
1808+
"Agentless profiling activated but no api key provided. Profile uploading will likely fail");
1809+
}
1810+
1811+
// LLM Observability
1812+
if (isLlmObsEnabled() && llmObsAgentlessEnabled) {
1813+
throw new FatalAgentMisconfigurationError(
1814+
"Attempt to start LLM Observability in Agentless mode without API key. "
1815+
+ "Please ensure that either an API key is configured, or the tracer is set up to work with the Agent");
1816+
}
17841817
}
17851818

17861819
this.telemetryDebugRequestsEnabled =
@@ -2640,6 +2673,18 @@ public String getIastSecurityControlsConfiguration() {
26402673
return iastSecurityControlsConfiguration;
26412674
}
26422675

2676+
public boolean isLlmObsEnabled() {
2677+
return instrumenterConfig.isLlmObsEnabled();
2678+
}
2679+
2680+
public boolean isLlmObsAgentlessEnabled() {
2681+
return llmObsAgentlessEnabled;
2682+
}
2683+
2684+
public String getLlmObsMlApp() {
2685+
return llmObsMlApp;
2686+
}
2687+
26432688
public boolean isCiVisibilityEnabled() {
26442689
return instrumenterConfig.isCiVisibilityEnabled();
26452690
}

‎internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import static datadog.trace.api.ConfigDefaults.DEFAULT_CODE_ORIGIN_FOR_SPANS_ENABLED;
66
import static datadog.trace.api.ConfigDefaults.DEFAULT_IAST_ENABLED;
77
import static datadog.trace.api.ConfigDefaults.DEFAULT_INTEGRATIONS_ENABLED;
8+
import static datadog.trace.api.ConfigDefaults.DEFAULT_LLM_OBS_ENABLED;
89
import static datadog.trace.api.ConfigDefaults.DEFAULT_MEASURE_METHODS;
910
import static datadog.trace.api.ConfigDefaults.DEFAULT_RESOLVER_RESET_INTERVAL;
1011
import static datadog.trace.api.ConfigDefaults.DEFAULT_RUNTIME_CONTEXT_FIELD_INJECTION;
@@ -26,6 +27,7 @@
2627
import static datadog.trace.api.config.GeneralConfig.TRACE_TRIAGE;
2728
import static datadog.trace.api.config.GeneralConfig.TRIAGE_REPORT_TRIGGER;
2829
import static datadog.trace.api.config.IastConfig.IAST_ENABLED;
30+
import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED;
2931
import static datadog.trace.api.config.ProfilingConfig.PROFILING_DIRECT_ALLOCATION_ENABLED;
3032
import static datadog.trace.api.config.ProfilingConfig.PROFILING_DIRECT_ALLOCATION_ENABLED_DEFAULT;
3133
import static datadog.trace.api.config.ProfilingConfig.PROFILING_ENABLED;
@@ -111,6 +113,7 @@ public class InstrumenterConfig {
111113
private final boolean iastFullyDisabled;
112114
private final boolean usmEnabled;
113115
private final boolean telemetryEnabled;
116+
private final boolean llmObsEnabled;
114117

115118
private final String traceExtensionsPath;
116119

@@ -199,6 +202,7 @@ private InstrumenterConfig() {
199202
iastFullyDisabled = iastEnabled != null && !iastEnabled;
200203
usmEnabled = configProvider.getBoolean(USM_ENABLED, DEFAULT_USM_ENABLED);
201204
telemetryEnabled = configProvider.getBoolean(TELEMETRY_ENABLED, DEFAULT_TELEMETRY_ENABLED);
205+
llmObsEnabled = configProvider.getBoolean(LLMOBS_ENABLED, DEFAULT_LLM_OBS_ENABLED);
202206
} else {
203207
// disable these features in native-image
204208
ciVisibilityEnabled = false;
@@ -207,6 +211,7 @@ private InstrumenterConfig() {
207211
iastFullyDisabled = true;
208212
telemetryEnabled = false;
209213
usmEnabled = false;
214+
llmObsEnabled = false;
210215
}
211216

212217
traceExtensionsPath = configProvider.getString(TRACE_EXTENSIONS_PATH);
@@ -355,6 +360,10 @@ public boolean isIastFullyDisabled() {
355360
return iastFullyDisabled;
356361
}
357362

363+
public boolean isLlmObsEnabled() {
364+
return llmObsEnabled;
365+
}
366+
358367
public boolean isUsmEnabled() {
359368
return usmEnabled;
360369
}

‎internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_REFRESH_BEANS_PE
6363
import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_STATSD_HOST
6464
import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_STATSD_PORT
6565
import static datadog.trace.api.config.JmxFetchConfig.JMX_TAGS
66+
import static datadog.trace.api.config.LlmObsConfig.LLMOBS_AGENTLESS_ENABLED
67+
import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ML_APP
68+
import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED
6669
import static datadog.trace.api.config.ProfilingConfig.PROFILING_AGENTLESS
6770
import static datadog.trace.api.config.ProfilingConfig.PROFILING_API_KEY_FILE_OLD
6871
import static datadog.trace.api.config.ProfilingConfig.PROFILING_API_KEY_FILE_VERY_OLD
@@ -163,6 +166,9 @@ class ConfigTest extends DDSpecification {
163166
private static final DD_PROFILING_TAGS_ENV = "DD_PROFILING_TAGS"
164167
private static final DD_PROFILING_PROXY_PASSWORD_ENV = "DD_PROFILING_PROXY_PASSWORD"
165168
private static final DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH = "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH"
169+
private static final DD_LLMOBS_ENABLED_ENV = "DD_LLMOBS_ENABLED"
170+
private static final DD_LLMOBS_ML_APP_ENV = "DD_LLMOBS_ML_APP"
171+
private static final DD_LLMOBS_AGENTLESS_ENABLED_ENV = "DD_LLMOBS_AGENTLESS_ENABLED"
166172

167173
def "specify overrides via properties"() {
168174
setup:
@@ -2208,6 +2214,124 @@ class ConfigTest extends DDSpecification {
22082214
!hostname.trim().isEmpty()
22092215
}
22102216

2217+
def "config instantiation should fail if llm obs is enabled via sys prop and ml app is not set"() {
2218+
setup:
2219+
Properties properties = new Properties()
2220+
properties.setProperty(LLMOBS_ENABLED, "true")
2221+
2222+
when:
2223+
new Config(ConfigProvider.withPropertiesOverride(properties))
2224+
2225+
then:
2226+
thrown IllegalArgumentException
2227+
}
2228+
2229+
def "config instantiation should fail if llm obs is enabled via env var and ml app is not set"() {
2230+
setup:
2231+
environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true")
2232+
2233+
when:
2234+
new Config()
2235+
2236+
then:
2237+
thrown IllegalArgumentException
2238+
}
2239+
2240+
2241+
def "config instantiation should NOT fail if llm obs is enabled (agentless disabled) via sys prop and ml app is set"() {
2242+
setup:
2243+
Properties properties = new Properties()
2244+
properties.setProperty(LLMOBS_ENABLED, "true")
2245+
properties.setProperty(LLMOBS_AGENTLESS_ENABLED, "false")
2246+
properties.setProperty(LLMOBS_ML_APP, "test-ml-app")
2247+
2248+
when:
2249+
def config = new Config(ConfigProvider.withPropertiesOverride(properties))
2250+
2251+
then:
2252+
noExceptionThrown()
2253+
config.isLlmObsEnabled()
2254+
!config.isLlmObsAgentlessEnabled()
2255+
config.llmObsMlApp == "test-ml-app"
2256+
}
2257+
2258+
def "config instantiation should NOT fail if llm obs is enabled (agentless disabled) via env var and ml app is set"() {
2259+
setup:
2260+
environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true")
2261+
environmentVariables.set(DD_LLMOBS_ML_APP_ENV, "test-ml-app")
2262+
2263+
when:
2264+
def config = new Config()
2265+
2266+
then:
2267+
noExceptionThrown()
2268+
config.isLlmObsEnabled()
2269+
!config.isLlmObsAgentlessEnabled()
2270+
config.llmObsMlApp == "test-ml-app"
2271+
}
2272+
2273+
def "config instantiation should fail if llm obs is in agentless mode via sys prop and API key is not set"() {
2274+
setup:
2275+
Properties properties = new Properties()
2276+
properties.setProperty(LLMOBS_ENABLED, "true")
2277+
properties.setProperty(LLMOBS_AGENTLESS_ENABLED, "true")
2278+
properties.setProperty(LLMOBS_ML_APP, "test-ml-app")
2279+
2280+
when:
2281+
new Config(ConfigProvider.withPropertiesOverride(properties))
2282+
2283+
then:
2284+
thrown FatalAgentMisconfigurationError
2285+
}
2286+
2287+
def "config instantiation should fail if llm obs is in agentless mode via env var and API key is not set"() {
2288+
setup:
2289+
environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true")
2290+
environmentVariables.set(DD_LLMOBS_ML_APP_ENV, "a")
2291+
environmentVariables.set(DD_LLMOBS_AGENTLESS_ENABLED_ENV, "true")
2292+
2293+
when:
2294+
new Config()
2295+
2296+
then:
2297+
thrown FatalAgentMisconfigurationError
2298+
}
2299+
2300+
def "config instantiation should NOT fail if llm obs is enabled (agentless enabled) and API key & ml app are set via sys prop"() {
2301+
setup:
2302+
Properties properties = new Properties()
2303+
properties.setProperty(LLMOBS_ENABLED, "true")
2304+
properties.setProperty(LLMOBS_AGENTLESS_ENABLED, "true")
2305+
properties.setProperty(LLMOBS_ML_APP, "test-ml-app")
2306+
properties.setProperty(API_KEY, "123456789")
2307+
2308+
when:
2309+
def config = new Config(ConfigProvider.withPropertiesOverride(properties))
2310+
2311+
then:
2312+
noExceptionThrown()
2313+
config.isLlmObsEnabled()
2314+
config.isLlmObsAgentlessEnabled()
2315+
config.llmObsMlApp == "test-ml-app"
2316+
}
2317+
2318+
def "config instantiation should NOT fail if llm obs is enabled (agentless enabled) and API key & ml app are set via env var"() {
2319+
setup:
2320+
environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true")
2321+
environmentVariables.set(DD_LLMOBS_ML_APP_ENV, "a")
2322+
environmentVariables.set(DD_LLMOBS_AGENTLESS_ENABLED_ENV, "true")
2323+
environmentVariables.set(DD_API_KEY_ENV, "8663294466")
2324+
2325+
when:
2326+
def config = new Config()
2327+
2328+
then:
2329+
noExceptionThrown()
2330+
config.isLlmObsEnabled()
2331+
config.isLlmObsAgentlessEnabled()
2332+
config.llmObsMlApp == "a"
2333+
}
2334+
22112335
def "config instantiation should fail if CI visibility agentless mode is enabled and API key is not set"() {
22122336
setup:
22132337
Properties properties = new Properties()

0 commit comments

Comments
 (0)