Skip to content

Commit a394d0b

Browse files
luhenryjbachorik
luhenry
and
jbachorik
authored
Add agent-crashtracking to upload crash (DataDog#3594)
Co-authored-by: Jaroslav Bachorik <[email protected]>
1 parent ddca12d commit a394d0b

21 files changed

Lines changed: 689 additions & 47 deletions

File tree

‎communication/communication.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ dependencies {
99
implementation project(':utils:version-utils')
1010

1111
api deps.okhttp
12-
api group: 'com.squareup.moshi', name: 'moshi', version: '1.9.2'
12+
api group: 'com.squareup.moshi', name: 'moshi', version: versions.moshi
1313
implementation group: 'com.datadoghq', name: 'java-dogstatsd-client', version: "${versions.dogstatsd}"
1414

1515
testImplementation project(':utils:test-utils')

‎communication/src/main/java/datadog/communication/http/OkHttpUtils.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ public final class OkHttpUtils {
4545
"Datadog-Meta-Lang-Interpreter-Vendor";
4646
private static final String DATADOG_CONTAINER_ID = "Datadog-Container-ID";
4747

48+
private static final String DD_API_KEY = "DD-API-KEY";
49+
4850
private static final String JAVA_VERSION = System.getProperty("java.version", "unknown");
4951
private static final String JAVA_VM_NAME = System.getProperty("java.vm.name", "unknown");
5052
private static final String JAVA_VM_VENDOR = System.getProperty("java.vm.vendor", "unknown");
@@ -76,8 +78,8 @@ public static OkHttpClient buildHttpClient(
7678
final Config config,
7779
final Dispatcher dispatcher,
7880
final HttpUrl url,
79-
final boolean retryOnConnectionFailure,
80-
final int maxRunningRequests,
81+
final Boolean retryOnConnectionFailure,
82+
final Integer maxRunningRequests,
8183
final String proxyHost,
8284
final Integer proxyPort,
8385
final String proxyUsername,
@@ -193,6 +195,23 @@ public static Request.Builder prepareRequest(final HttpUrl url, Map<String, Stri
193195
return builder;
194196
}
195197

198+
public static Request.Builder prepareRequest(
199+
final HttpUrl url,
200+
final Map<String, String> headers,
201+
final Config config,
202+
final boolean agentless) {
203+
Request.Builder builder = prepareRequest(url, headers);
204+
205+
final String apiKey = config.getApiKey();
206+
if (agentless && apiKey != null) {
207+
// we only add the api key header if we know we're doing agentless. No point in adding it to
208+
// other agent-based requests since we know the datadog-agent isn't going to make use of it.
209+
builder.addHeader(DD_API_KEY, apiKey);
210+
}
211+
212+
return builder;
213+
}
214+
196215
public static RequestBody msgpackRequestBodyOf(List<ByteBuffer> buffers) {
197216
return new ByteBufferRequestBody(buffers);
198217
}

‎dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,23 @@ public static synchronized Class<?> installAgentCLI(final URL bootstrapURL) thro
281281
return AGENT_CLASSLOADER.loadClass("datadog.trace.agent.tooling.AgentCLI");
282282
}
283283

284+
public static synchronized Object installAgentClassLoader(final URL bootstrapURL)
285+
throws Exception {
286+
createSharedClassloader(bootstrapURL);
287+
if (null == AGENT_CLASSLOADER) {
288+
// in CLI mode we skip installation of instrumentation because we're not running as an agent
289+
// we still create the agent classloader so we can install the tracer and query integrations
290+
AGENT_CLASSLOADER = createDelegateClassLoader("inst", bootstrapURL, SHARED_CLASSLOADER);
291+
}
292+
ClassLoader current = Thread.currentThread().getContextClassLoader();
293+
Thread.currentThread().setContextClassLoader(AGENT_CLASSLOADER);
294+
return current;
295+
}
296+
297+
public static synchronized void uninstallAgentClassLoader(final Object cookie) throws Exception {
298+
Thread.currentThread().setContextClassLoader((ClassLoader) cookie);
299+
}
300+
284301
private static void registerLogManagerCallback(final ClassLoadCallBack callback) {
285302
try {
286303
final Class<?> agentInstallerClass = AGENT_CLASSLOADER.loadClass(AGENT_INSTALLER_CLASS_NAME);

‎dd-java-agent/agent-crashtracking/agent-crashtracking.gradle‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
plugins {
2+
id "com.github.johnrengelman.shadow"
3+
}
4+
5+
// Set properties before any plugins get loaded
6+
ext {
7+
enableJunitPlatform = true
8+
minJavaVersionForTests = JavaVersion.VERSION_1_8
9+
}
10+
11+
apply from: "$rootDir/gradle/java.gradle"
12+
apply from: "$rootDir/gradle/version.gradle"
13+
14+
// FIXME: Improve test coverage.
15+
minimumBranchCoverage = 0.6
16+
minimumInstructionCoverage = 0.9
17+
18+
dependencies {
19+
implementation deps.slf4j
20+
implementation project(':communication')
21+
implementation project(':internal-api')
22+
implementation project(':utils:container-utils')
23+
implementation project(':utils:process-utils')
24+
implementation project(':utils:socket-utils')
25+
implementation project(':utils:version-utils')
26+
27+
implementation deps.okhttp
28+
implementation group: 'com.squareup.moshi', name: 'moshi', version: versions.moshi
29+
30+
testImplementation deps.junit5
31+
testImplementation project(':dd-java-agent:agent-profiling:profiling-testing')
32+
testImplementation deps.mockito
33+
testImplementation group: 'com.squareup.okhttp3', name: 'mockwebserver', version: versions.okhttp
34+
testImplementation(group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.10')
35+
}
36+
37+
sourceCompatibility = JavaVersion.VERSION_1_8
38+
targetCompatibility = JavaVersion.VERSION_1_8
39+
40+
configurations {
41+
// exclude bootstrap dependencies from shadowJar
42+
runtime.exclude module: deps.slf4j
43+
runtime.exclude group: 'org.slf4j'
44+
}
45+
46+
shadowJar {
47+
dependencies deps.excludeShared
48+
exclude {
49+
if (it.path.startsWith('org/jctools/')) {
50+
def ret = true
51+
if (it.path.equals('org/jctools/util') || it.path.equals('org/jctools/maps')) {
52+
ret = false
53+
} else {
54+
ret = !(it.path.contains('/util') || it.path.contains('AbstractEntry') || it.path.contains('ConcurrentAutoTable') || it.path.contains('NonBlockingHashMap'))
55+
}
56+
return ret
57+
}
58+
return false
59+
}
60+
}
61+
62+
jar {
63+
classifier = 'unbundled'
64+
}
65+
66+
sourceCompatibility = JavaVersion.VERSION_1_8
67+
targetCompatibility = JavaVersion.VERSION_1_8

‎dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashUploader.java‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
package com.datadog.crashtracking;
2+
3+
import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_PROXY_HOST;
4+
import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_PROXY_PASSWORD;
5+
import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_PROXY_PORT;
6+
import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_PROXY_USERNAME;
7+
import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_UPLOAD_TIMEOUT;
8+
import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_UPLOAD_TIMEOUT_DEFAULT;
9+
10+
import com.squareup.moshi.JsonWriter;
11+
import datadog.common.container.ContainerInfo;
12+
import datadog.common.process.PidHelper;
13+
import datadog.common.version.VersionInfo;
14+
import datadog.communication.http.OkHttpUtils;
15+
import datadog.trace.api.Config;
16+
import datadog.trace.bootstrap.config.provider.ConfigProvider;
17+
import java.io.IOException;
18+
import java.io.InputStream;
19+
import java.io.InputStreamReader;
20+
import java.nio.charset.StandardCharsets;
21+
import java.time.Instant;
22+
import java.util.HashMap;
23+
import java.util.List;
24+
import java.util.Map;
25+
import java.util.concurrent.TimeUnit;
26+
import java.util.stream.Collectors;
27+
import javax.annotation.Nonnull;
28+
import okhttp3.Call;
29+
import okhttp3.HttpUrl;
30+
import okhttp3.MediaType;
31+
import okhttp3.OkHttpClient;
32+
import okhttp3.RequestBody;
33+
import okhttp3.Response;
34+
import okio.Buffer;
35+
import org.slf4j.Logger;
36+
import org.slf4j.LoggerFactory;
37+
38+
/** Crash Reporter implementation */
39+
public class CrashUploader {
40+
41+
private static final Logger log = LoggerFactory.getLogger(CrashUploader.class);
42+
43+
// Header names and values
44+
static final String JAVA_LANG = "java";
45+
static final String HEADER_DD_EVP_ORIGIN = "DD-EVP-ORIGIN";
46+
static final String JAVA_TRACING_LIBRARY = "dd-trace-java";
47+
static final String HEADER_DD_EVP_ORIGIN_VERSION = "DD-EVP-ORIGIN-VERSION";
48+
static final String HEADER_DD_TELEMETRY_API_VERSION = "DD-Telemetry-API-Version";
49+
static final String API_VERSION = "v1";
50+
static final String HEADER_DD_TELEMETRY_REQUEST_TYPE = "DD-Telemetry-Request-Type";
51+
static final String REQUEST_TYPE = "logs";
52+
53+
private static final MediaType APPLICATION_JSON =
54+
MediaType.get("application/json; charset=utf-8");
55+
private static final MediaType APPLICATION_OCTET_STREAM =
56+
MediaType.parse("application/octet-stream");
57+
58+
private final Config config;
59+
private final ConfigProvider configProvider;
60+
61+
private final OkHttpClient client;
62+
private final boolean agentless;
63+
private final HttpUrl url;
64+
private final String tags;
65+
66+
public CrashUploader() {
67+
this(Config.get(), ConfigProvider.getInstance());
68+
}
69+
70+
CrashUploader(final Config config, final ConfigProvider configProvider) {
71+
this.config = config;
72+
this.configProvider = configProvider;
73+
74+
url = HttpUrl.get(config.getFinalCrashTrackingUrl());
75+
agentless = config.isCrashTrackingAgentless();
76+
77+
final Map<String, String> tagsMap = new HashMap<>(config.getMergedCrashTrackingTags());
78+
tagsMap.put(VersionInfo.LIBRARY_VERSION_TAG, VersionInfo.VERSION);
79+
// PID can be null if we cannot find it out from the system
80+
if (PidHelper.PID != null) {
81+
tagsMap.put(PidHelper.PID_TAG, PidHelper.PID.toString());
82+
}
83+
// Comma separated tags string for V2.4 format
84+
tags = tagsToString(tagsMap);
85+
86+
client =
87+
OkHttpUtils.buildHttpClient(
88+
config,
89+
null, /* dispatcher */
90+
url,
91+
true, /* retryOnConnectionFailure */
92+
null, /* maxRunningRequests */
93+
configProvider.getString(CRASH_TRACKING_PROXY_HOST),
94+
configProvider.getInteger(CRASH_TRACKING_PROXY_PORT),
95+
configProvider.getString(CRASH_TRACKING_PROXY_USERNAME),
96+
configProvider.getString(CRASH_TRACKING_PROXY_PASSWORD),
97+
TimeUnit.SECONDS.toMillis(
98+
configProvider.getInteger(
99+
CRASH_TRACKING_UPLOAD_TIMEOUT, CRASH_TRACKING_UPLOAD_TIMEOUT_DEFAULT)));
100+
}
101+
102+
private String tagsToString(final Map<String, String> tags) {
103+
return tags.entrySet().stream()
104+
.filter(e -> e.getValue() != null && !e.getValue().isEmpty())
105+
.map(e -> e.getKey() + ":" + e.getValue())
106+
.collect(Collectors.joining(","));
107+
}
108+
109+
public void upload(@Nonnull List<InputStream> files) throws IOException {
110+
Call call = makeRequest(files);
111+
try {
112+
handleSuccess(call, call.execute());
113+
} catch (IOException e) {
114+
handleFailure(call, e);
115+
}
116+
}
117+
118+
private Call makeRequest(@Nonnull List<InputStream> files) throws IOException {
119+
final RequestBody requestBody = makeRequestBody(files);
120+
121+
final Map<String, String> headers = new HashMap<>();
122+
// Set chunked transfer
123+
headers.put("Content-Type", requestBody.contentType().toString());
124+
headers.put("Content-Length", Long.toString(requestBody.contentLength()));
125+
headers.put("Transfer-Encoding", "chunked");
126+
headers.put(HEADER_DD_EVP_ORIGIN, JAVA_TRACING_LIBRARY);
127+
headers.put(HEADER_DD_EVP_ORIGIN_VERSION, VersionInfo.VERSION);
128+
headers.put(HEADER_DD_TELEMETRY_API_VERSION, API_VERSION);
129+
headers.put(HEADER_DD_TELEMETRY_REQUEST_TYPE, REQUEST_TYPE);
130+
131+
return client.newCall(
132+
OkHttpUtils.prepareRequest(url, headers, config, agentless).post(requestBody).build());
133+
}
134+
135+
private RequestBody makeRequestBody(@Nonnull List<InputStream> files) throws IOException {
136+
Buffer out = new Buffer();
137+
try (JsonWriter writer = JsonWriter.of(out)) {
138+
writer.beginObject();
139+
140+
writer.name("api_version").value(API_VERSION);
141+
writer.name("request_type").value("logs");
142+
writer
143+
.name("runtime_id")
144+
// randomly generated, https://xkcd.com/221/
145+
.value("5e5b1180-2a0b-41a6-bed2-bc341d19f853");
146+
writer.name("tracer_time").value(Instant.now().getEpochSecond());
147+
writer.name("seq_id").value(1);
148+
writer.name("debug").value(true);
149+
writer.name("payload");
150+
writer.beginArray();
151+
for (InputStream file : files) {
152+
writer.beginObject();
153+
writer.name("message").value(readContent(file));
154+
writer.name("level").value("ERROR");
155+
writer.endObject();
156+
}
157+
writer.endArray();
158+
writer.name("application");
159+
writer.beginObject();
160+
writer.name("env").value(config.getEnv());
161+
writer.name("language_name").value(JAVA_LANG);
162+
writer.name("language_version").value(System.getProperty("java.version", "unknown"));
163+
writer.name("service_name").value(config.getServiceName());
164+
writer.name("service_version").value(config.getVersion());
165+
writer.name("tracer_version").value(VersionInfo.VERSION);
166+
writer.endObject();
167+
writer.name("host");
168+
writer.beginObject();
169+
if (ContainerInfo.get().getContainerId() != null) {
170+
writer.name("container_id").value(ContainerInfo.get().getContainerId());
171+
}
172+
writer.name("hostname").value(config.getHostName());
173+
writer.name("env").value(config.getEnv());
174+
writer.endObject();
175+
writer.endObject();
176+
}
177+
178+
return RequestBody.create(APPLICATION_JSON, out.readByteString());
179+
}
180+
181+
private String readContent(InputStream file) throws IOException {
182+
try (InputStreamReader reader = new InputStreamReader(file, StandardCharsets.UTF_8)) {
183+
int read;
184+
char[] buffer = new char[1 << 14];
185+
StringBuilder sb = new StringBuilder();
186+
while ((read = reader.read(buffer, 0, buffer.length)) > 0) {
187+
sb.append(buffer, 0, read);
188+
}
189+
return sb.toString();
190+
}
191+
}
192+
193+
private void handleSuccess(final Call call, final Response response) throws IOException {
194+
if (response.isSuccessful()) {
195+
log.info(
196+
"Successfully uploaded the crash files, code = {} \"{}\"",
197+
response.code(),
198+
response.message());
199+
} else {
200+
log.error(
201+
"Failed to upload crash files, code = {} \"{}\", body = \"{}\"",
202+
response.code(),
203+
response.message(),
204+
response.body() != null ? response.body().string().trim() : "<null>");
205+
}
206+
}
207+
208+
private void handleFailure(final Call call, final IOException exception) {
209+
log.error("Failed to upload crash files, got exception: {}", exception.getMessage(), exception);
210+
}
211+
}

0 commit comments

Comments
 (0)