From 007dde8d45f8f88b67749f150438c1275381fc2a Mon Sep 17 00:00:00 2001
From: Jonathan Klabunde Tomer <125505367+jkt-signal@users.noreply.github.com>
Date: Mon, 22 Sep 2025 11:09:40 -0700
Subject: [PATCH] add OTLP logging appender factory

---
 pom.xml                                       | 14 +++++
 service/pom.xml                               | 52 ++++++++++++++-----
 .../textsecuregcm/WhisperServerService.java   |  1 +
 .../textsecuregcm/metrics/MetricsUtil.java    | 37 +++++++++++++
 .../metrics/OpenTelemetryAppenderFactory.java | 50 ++++++++++++++++++
 .../workers/CommandDependencies.java          |  3 ++
 ....dropwizard.logging.common.AppenderFactory |  1 +
 7 files changed, 145 insertions(+), 13 deletions(-)
 create mode 100644 service/src/main/java/org/whispersystems/textsecuregcm/metrics/OpenTelemetryAppenderFactory.java

diff --git a/pom.xml b/pom.xml
index c05ea2db4..26659788a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -163,6 +163,20 @@
         <type>pom</type>
         <scope>import</scope>
       </dependency>
+      <dependency>
+        <groupId>io.opentelemetry</groupId>
+        <artifactId>opentelemetry-bom</artifactId>
+        <version>1.54.0</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+      <dependency>
+        <groupId>io.opentelemetry.instrumentation</groupId>
+        <artifactId>opentelemetry-instrumentation-bom</artifactId>
+        <version>2.20.0</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
       <dependency>
         <groupId>io.projectreactor</groupId>
         <artifactId>reactor-bom</artifactId>
diff --git a/service/pom.xml b/service/pom.xml
index 4f0f7889e..f9f27173c 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -35,7 +35,7 @@
       <artifactId>app-store-server-library</artifactId>
       <version>${storekit.version}</version>
       <exclusions>
-        <!-- conflicts with okio-jvm from apollo-api-jvm -->
+        <!-- conflicts with other users; resolved manually with explicit import -->
         <exclusion>
           <groupId>com.squareup.okio</groupId>
           <artifactId>okio-jvm</artifactId>
@@ -198,6 +198,28 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-exporter-otlp</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry.instrumentation</groupId>
+      <artifactId>opentelemetry-logback-appender-1.0</artifactId>
+      <!-- *all* opentelemetry-logback-appender versions are "alpha" despite the advanced version number -->
+      <version>2.19.0-alpha</version>
+      <exclusions>
+        <!-- incubator packages aren't included in the opentelemetry BOM, and we don't use them -->
+        <exclusion>
+          <groupId>io.opentelemetry.instrumentation</groupId>
+          <artifactId>opentelemetry-instrumentation-api-incubator</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
     <dependency>
       <groupId>party.iroiro.luajava</groupId>
       <artifactId>luajava</artifactId>
@@ -258,16 +280,11 @@
       </exclusions>
     </dependency>
 
-    <!-- resolve opentelemetry-semconv conflicts from lower in firebase-admin and firestore dependency trees -->
     <dependency>
       <groupId>com.google.firebase</groupId>
       <artifactId>firebase-admin</artifactId>
       <version>${firebase-admin.version}</version>
       <exclusions>
-        <exclusion>
-          <groupId>io.opentelemetry.semconv</groupId>
-          <artifactId>opentelemetry-semconv</artifactId>
-        </exclusion>
         <!-- our direct import of guava brings in a more recent version of failureaccess, so excluding it here -->
         <exclusion>
           <groupId>com.google.guava</groupId>
@@ -275,18 +292,15 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <dependency>
-      <groupId>io.opentelemetry.semconv</groupId>
-      <artifactId>opentelemetry-semconv</artifactId>
-      <version>1.37.0</version>
-    </dependency>
+
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-firestore</artifactId>
       <exclusions>
+        <!-- incubator packages aren't included in the opentelemetry BOM, and we don't use them -->
         <exclusion>
-          <groupId>io.opentelemetry.semconv</groupId>
-          <artifactId>opentelemetry-semconv</artifactId>
+          <groupId>io.opentelemetry.instrumentation</groupId>
+          <artifactId>opentelemetry-instrumentation-api-incubator</artifactId>
         </exclusion>
         <!-- our direct import of guava brings in a more recent version of failureaccess, so excluding it here -->
         <exclusion>
@@ -561,9 +575,21 @@
           <groupId>org.jetbrains</groupId>
           <artifactId>annotations</artifactId>
         </exclusion>
+        <!-- conflicts with other users; resolved manually with explicit import -->
+        <exclusion>
+          <groupId>com.squareup.okio</groupId>
+          <artifactId>okio-jvm</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
+    <!-- to resolve conflicting imports from other dependencies -->
+    <dependency>
+      <groupId>com.squareup.okio</groupId>
+      <artifactId>okio-jvm</artifactId>
+      <version>3.15.0</version>
+    </dependency>
+
   </dependencies>
 
   <profiles>
diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
index 8799d50b7..f1a36998a 100644
--- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
@@ -385,6 +385,7 @@ public class WhisperServerService extends Application<WhisperServerConfiguration
     dynamicConfigurationManager.start();
 
     MetricsUtil.configureRegistries(config, environment, dynamicConfigurationManager);
+    MetricsUtil.configureLogging(config, environment);
 
     ExperimentEnrollmentManager experimentEnrollmentManager = new ExperimentEnrollmentManager(dynamicConfigurationManager);
 
diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java
index 5d7b3ead7..c860b88d8 100644
--- a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java
@@ -8,6 +8,7 @@ package org.whispersystems.textsecuregcm.metrics;
 import com.codahale.metrics.SharedMetricRegistries;
 import com.google.common.annotations.VisibleForTesting;
 import io.dropwizard.core.setup.Environment;
+import io.dropwizard.lifecycle.Managed;
 import io.micrometer.core.instrument.Meter;
 import io.micrometer.core.instrument.MeterRegistry;
 import io.micrometer.core.instrument.Metrics;
@@ -21,6 +22,14 @@ import io.micrometer.core.instrument.config.MeterFilter;
 import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
 import io.micrometer.registry.otlp.OtlpMeterRegistry;
 import io.micrometer.statsd.StatsdMeterRegistry;
+import io.opentelemetry.exporter.otlp.http.logs.OtlpHttpLogRecordExporter;
+import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.logs.SdkLoggerProvider;
+import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor;
+import io.opentelemetry.sdk.resources.Resource;
+import io.opentelemetry.semconv.ServiceAttributes;
+
 import java.time.Duration;
 import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
 import org.whispersystems.textsecuregcm.WhisperServerVersion;
@@ -99,6 +108,34 @@ public class MetricsUtil {
         new MicrometerRegistryManager(Metrics.globalRegistry, shutdownWaitDuration));
   }
 
+  public static void configureLogging(final WhisperServerConfiguration config, final Environment environment) {
+    if (!config.getOpenTelemetryConfiguration().enabled()) {
+      return;
+    }
+    final OpenTelemetrySdk openTelemetry =
+      OpenTelemetrySdk.builder()
+        .setLoggerProvider(
+          SdkLoggerProvider.builder()
+            .setResource(
+              Resource.builder()
+                .put(ServiceAttributes.SERVICE_NAME, "chat")
+                .put(ServiceAttributes.SERVICE_VERSION, WhisperServerVersion.getServerVersion())
+                .build())
+            .addLogRecordProcessor(
+              BatchLogRecordProcessor.builder(OtlpHttpLogRecordExporter.getDefault()).build())
+            .build())
+        .build();
+
+    OpenTelemetryAppender.install(openTelemetry);
+
+    environment.lifecycle().manage(new Managed() {
+      @Override
+      public void stop() {
+        openTelemetry.shutdown();
+      }
+    });
+  }
+
   @VisibleForTesting
   static void configureMeterFilters(MeterRegistry.Config config,
       final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager) {
diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/OpenTelemetryAppenderFactory.java b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/OpenTelemetryAppenderFactory.java
new file mode 100644
index 000000000..201a4923b
--- /dev/null
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/OpenTelemetryAppenderFactory.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2025 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.textsecuregcm.metrics;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Appender;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import io.dropwizard.logging.common.AbstractAppenderFactory;
+import io.dropwizard.logging.common.async.AsyncAppenderFactory;
+import io.dropwizard.logging.common.filter.LevelFilterFactory;
+import io.dropwizard.logging.common.layout.LayoutFactory;
+import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender;
+import jakarta.validation.constraints.NotEmpty;
+
+@JsonTypeName("otlp")
+public class OpenTelemetryAppenderFactory extends AbstractAppenderFactory<ILoggingEvent> {
+
+  @JsonProperty
+  private String destination;
+
+  @Override
+  public Appender<ILoggingEvent> build(
+      final LoggerContext context,
+      final String applicationName,
+      final LayoutFactory<ILoggingEvent> layoutFactory,
+      final LevelFilterFactory<ILoggingEvent> levelFilterFactory,
+      final AsyncAppenderFactory<ILoggingEvent> asyncAppenderFactory) {
+
+    final OpenTelemetryAppender appender = new OpenTelemetryAppender();
+    appender.setCaptureCodeAttributes(true);
+    appender.setCaptureLoggerContext(true);
+
+    // The installation of an OpenTelemetry configuration happens in
+    // WhisperServerService (or CommandDependencies), in order to let us tie
+    // into Dropwizard's lifecycle management; this allows us to buffer any
+    // logs emitted between now and that happening.
+    appender.setNumLogsCapturedBeforeOtelInstall(1000);
+
+    appender.addFilter(levelFilterFactory.build(threshold));
+    getFilterFactories().forEach(f -> appender.addFilter(f.build()));
+    appender.start();
+
+    return wrapAsync(appender, asyncAppenderFactory);
+  }
+}
diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java
index b6e790183..e5386896e 100644
--- a/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java
+++ b/service/src/main/java/org/whispersystems/textsecuregcm/workers/CommandDependencies.java
@@ -45,6 +45,7 @@ import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager;
 import org.whispersystems.textsecuregcm.experiment.PushNotificationExperimentSamples;
 import org.whispersystems.textsecuregcm.grpc.net.GrpcClientConnectionManager;
 import org.whispersystems.textsecuregcm.limits.RateLimiters;
+import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
 import org.whispersystems.textsecuregcm.metrics.MicrometerAwsSdkMetricPublisher;
 import org.whispersystems.textsecuregcm.push.APNSender;
 import org.whispersystems.textsecuregcm.push.FcmSender;
@@ -127,6 +128,8 @@ record CommandDependencies(
       throws IOException, GeneralSecurityException, InvalidInputException {
     Clock clock = Clock.systemUTC();
 
+    MetricsUtil.configureLogging(configuration, environment);
+
     environment.getObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
 
     final AwsCredentialsProvider awsCredentialsProvider = configuration.getAwsCredentialsConfiguration().build();
diff --git a/service/src/main/resources/META-INF/services/io.dropwizard.logging.common.AppenderFactory b/service/src/main/resources/META-INF/services/io.dropwizard.logging.common.AppenderFactory
index ebe62c359..a4d1da70b 100644
--- a/service/src/main/resources/META-INF/services/io.dropwizard.logging.common.AppenderFactory
+++ b/service/src/main/resources/META-INF/services/io.dropwizard.logging.common.AppenderFactory
@@ -1 +1,2 @@
 org.whispersystems.textsecuregcm.metrics.LogstashTcpSocketAppenderFactory
+org.whispersystems.textsecuregcm.metrics.OpenTelemetryAppenderFactory