Break out into a multi-module project

This commit is contained in:
Moxie Marlinspike
2019-04-20 21:56:20 -07:00
parent b41dde777e
commit d0d375aeb7
318 changed files with 255 additions and 215 deletions

View File

@@ -0,0 +1,16 @@
package org.whispersystems.textsecuregcm.metrics;
import com.codahale.metrics.Gauge;
import com.sun.management.OperatingSystemMXBean;
import java.lang.management.ManagementFactory;
public class CpuUsageGauge implements Gauge<Integer> {
@Override
public Integer getValue() {
OperatingSystemMXBean mbean = (com.sun.management.OperatingSystemMXBean)
ManagementFactory.getOperatingSystemMXBean();
return (int) Math.ceil(mbean.getSystemCpuLoad() * 100);
}
}

View File

@@ -0,0 +1,19 @@
package org.whispersystems.textsecuregcm.metrics;
import com.codahale.metrics.Gauge;
import java.io.File;
public class FileDescriptorGauge implements Gauge<Integer> {
@Override
public Integer getValue() {
File file = new File("/proc/self/fd");
if (file.isDirectory() && file.exists()) {
return file.list().length;
}
return 0;
}
}

View File

@@ -0,0 +1,17 @@
package org.whispersystems.textsecuregcm.metrics;
import com.codahale.metrics.Gauge;
import com.sun.management.OperatingSystemMXBean;
import java.lang.management.ManagementFactory;
public class FreeMemoryGauge implements Gauge<Long> {
@Override
public Long getValue() {
OperatingSystemMXBean mbean = (com.sun.management.OperatingSystemMXBean)
ManagementFactory.getOperatingSystemMXBean();
return mbean.getFreePhysicalMemorySize();
}
}

View File

@@ -0,0 +1,255 @@
package org.whispersystems.textsecuregcm.metrics;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Metered;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.ScheduledReporter;
import com.codahale.metrics.Snapshot;
import com.codahale.metrics.Timer;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.Map;
import java.util.SortedMap;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
public class JsonMetricsReporter extends ScheduledReporter {
private static final Pattern SIMPLE_NAMES = Pattern.compile("[^a-zA-Z0-9_.\\-~]");
private final Logger logger = LoggerFactory.getLogger(JsonMetricsReporter.class);
private final JsonFactory factory = new JsonFactory();
private final URI uri;
private final HttpClient httpClient;
public JsonMetricsReporter(MetricRegistry registry, String token, String hostname,
MetricFilter filter, TimeUnit rateUnit, TimeUnit durationUnit)
throws UnknownHostException
{
super(registry, "json-reporter", filter, rateUnit, durationUnit);
this.httpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).build();
this.uri = URI.create(String.format("https://%s/report/metrics?t=%s&h=%s", hostname, token, InetAddress.getLocalHost().getHostName()));
}
@Override
public void report(SortedMap<String, Gauge> stringGaugeSortedMap,
SortedMap<String, Counter> stringCounterSortedMap,
SortedMap<String, Histogram> stringHistogramSortedMap,
SortedMap<String, Meter> stringMeterSortedMap,
SortedMap<String, Timer> stringTimerSortedMap)
{
try {
logger.debug("Reporting metrics...");
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
JsonGenerator json = factory.createGenerator(outputStream, JsonEncoding.UTF8);
json.writeStartObject();
for (Map.Entry<String, Gauge> gauge : stringGaugeSortedMap.entrySet()) {
reportGauge(json, gauge.getKey(), gauge.getValue());
}
for (Map.Entry<String, Counter> counter : stringCounterSortedMap.entrySet()) {
reportCounter(json, counter.getKey(), counter.getValue());
}
for (Map.Entry<String, Histogram> histogram : stringHistogramSortedMap.entrySet()) {
reportHistogram(json, histogram.getKey(), histogram.getValue());
}
for (Map.Entry<String, Meter> meter : stringMeterSortedMap.entrySet()) {
reportMeter(json, meter.getKey(), meter.getValue());
}
for (Map.Entry<String, Timer> timer : stringTimerSortedMap.entrySet()) {
reportTimer(json, timer.getKey(), timer.getValue());
}
json.writeEndObject();
json.close();
outputStream.close();
HttpRequest request = HttpRequest.newBuilder()
.uri(uri)
.POST(HttpRequest.BodyPublishers.ofByteArray(outputStream.toByteArray()))
.header("Content-Type", "application/json")
.build();
HttpResponse<Void> response = httpClient.send(request, HttpResponse.BodyHandlers.discarding());
logger.debug("Metrics server response: " + response.statusCode());
} catch (IOException e) {
logger.warn("Error sending metrics", e);
} catch (Exception e) {
logger.warn("error", e);
}
}
private void reportGauge(JsonGenerator json, String name, Gauge gauge) throws IOException {
Object gaugeValue = evaluateGauge(gauge);
if (gaugeValue instanceof Number) {
json.writeFieldName(sanitize(name));
json.writeObject(gaugeValue);
}
}
private void reportCounter(JsonGenerator json, String name, Counter counter) throws IOException {
json.writeFieldName(sanitize(name));
json.writeNumber(counter.getCount());
}
private void reportHistogram(JsonGenerator json, String name, Histogram histogram) throws IOException {
Snapshot snapshot = histogram.getSnapshot();
json.writeFieldName(sanitize(name));
json.writeStartObject();
json.writeNumberField("count", histogram.getCount());
writeSnapshot(json, snapshot);
json.writeEndObject();
}
private void reportMeter(JsonGenerator json, String name, Meter meter) throws IOException {
json.writeFieldName(sanitize(name));
json.writeStartObject();
writeMetered(json, meter);
json.writeEndObject();
}
private void reportTimer(JsonGenerator json, String name, Timer timer) throws IOException {
json.writeFieldName(sanitize(name));
json.writeStartObject();
json.writeFieldName("rate");
json.writeStartObject();
writeMetered(json, timer);
json.writeEndObject();
json.writeFieldName("duration");
json.writeStartObject();
writeTimedSnapshot(json, timer.getSnapshot());
json.writeEndObject();
json.writeEndObject();
}
private Object evaluateGauge(Gauge gauge) {
try {
return gauge.getValue();
} catch (RuntimeException e) {
logger.warn("Error reading gauge", e);
return "error reading gauge";
}
}
private void writeTimedSnapshot(JsonGenerator json, Snapshot snapshot) throws IOException {
json.writeNumberField("max", convertDuration(snapshot.getMax()));
json.writeNumberField("mean", convertDuration(snapshot.getMean()));
json.writeNumberField("min", convertDuration(snapshot.getMin()));
json.writeNumberField("stddev", convertDuration(snapshot.getStdDev()));
json.writeNumberField("median", convertDuration(snapshot.getMedian()));
json.writeNumberField("p75", convertDuration(snapshot.get75thPercentile()));
json.writeNumberField("p95", convertDuration(snapshot.get95thPercentile()));
json.writeNumberField("p98", convertDuration(snapshot.get98thPercentile()));
json.writeNumberField("p99", convertDuration(snapshot.get99thPercentile()));
json.writeNumberField("p999", convertDuration(snapshot.get999thPercentile()));
}
private void writeSnapshot(JsonGenerator json, Snapshot snapshot) throws IOException {
json.writeNumberField("max", snapshot.getMax());
json.writeNumberField("mean", snapshot.getMean());
json.writeNumberField("min", snapshot.getMin());
json.writeNumberField("stddev", snapshot.getStdDev());
json.writeNumberField("median", snapshot.getMedian());
json.writeNumberField("p75", snapshot.get75thPercentile());
json.writeNumberField("p95", snapshot.get95thPercentile());
json.writeNumberField("p98", snapshot.get98thPercentile());
json.writeNumberField("p99", snapshot.get99thPercentile());
json.writeNumberField("p999", snapshot.get999thPercentile());
}
private void writeMetered(JsonGenerator json, Metered meter) throws IOException {
json.writeNumberField("count", convertRate(meter.getCount()));
json.writeNumberField("mean", convertRate(meter.getMeanRate()));
json.writeNumberField("m1", convertRate(meter.getOneMinuteRate()));
json.writeNumberField("m5", convertRate(meter.getFiveMinuteRate()));
json.writeNumberField("m15", convertRate(meter.getFifteenMinuteRate()));
}
private String sanitize(String metricName) {
return SIMPLE_NAMES.matcher(metricName).replaceAll("_");
}
public static Builder forRegistry(MetricRegistry registry) {
return new Builder(registry);
}
public static class Builder {
private final MetricRegistry registry;
private MetricFilter filter = MetricFilter.ALL;
private TimeUnit rateUnit = TimeUnit.SECONDS;
private TimeUnit durationUnit = TimeUnit.MILLISECONDS;
private String token;
private String hostname;
private Builder(MetricRegistry registry) {
this.registry = registry;
this.rateUnit = TimeUnit.SECONDS;
this.durationUnit = TimeUnit.MILLISECONDS;
this.filter = MetricFilter.ALL;
}
public Builder convertRatesTo(TimeUnit rateUnit) {
this.rateUnit = rateUnit;
return this;
}
public Builder convertDurationsTo(TimeUnit durationUnit) {
this.durationUnit = durationUnit;
return this;
}
public Builder filter(MetricFilter filter) {
this.filter = filter;
return this;
}
public Builder withToken(String token) {
this.token = token;
return this;
}
public Builder withHostname(String hostname) {
this.hostname = hostname;
return this;
}
public JsonMetricsReporter build() throws UnknownHostException {
if (hostname == null) {
throw new IllegalArgumentException("No hostname specified!");
}
if (token == null) {
throw new IllegalArgumentException("No token specified!");
}
return new JsonMetricsReporter(registry, token, hostname, filter, rateUnit, durationUnit);
}
}
}

View File

@@ -0,0 +1,38 @@
package org.whispersystems.textsecuregcm.metrics;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.ScheduledReporter;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import javax.validation.constraints.NotNull;
import java.net.UnknownHostException;
import io.dropwizard.metrics.BaseReporterFactory;
@JsonTypeName("json")
public class JsonMetricsReporterFactory extends BaseReporterFactory {
@JsonProperty
@NotNull
private String hostname;
@JsonProperty
@NotNull
private String token;
@Override
public ScheduledReporter build(MetricRegistry metricRegistry) {
try {
return JsonMetricsReporter.forRegistry(metricRegistry)
.withHostname(hostname)
.withToken(token)
.convertRatesTo(getRateUnit())
.convertDurationsTo(getDurationUnit())
.filter(getFilter())
.build();
} catch (UnknownHostException e) {
throw new IllegalArgumentException(e);
}
}
}

View File

@@ -0,0 +1,198 @@
package org.whispersystems.textsecuregcm.metrics;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import org.hibernate.validator.constraints.NotEmpty;
import org.productivity.java.syslog4j.SyslogConfigIF;
import org.productivity.java.syslog4j.SyslogIF;
import org.productivity.java.syslog4j.SyslogRuntimeException;
import org.productivity.java.syslog4j.impl.net.tcp.ssl.SSLTCPNetSyslogConfig;
import javax.validation.constraints.NotNull;
import java.util.TimeZone;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.classic.util.LevelToSyslogSeverity;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.AppenderBase;
import ch.qos.logback.core.Layout;
import io.dropwizard.logging.AbstractAppenderFactory;
import io.dropwizard.logging.async.AsyncAppenderFactory;
import io.dropwizard.logging.filter.LevelFilterFactory;
import io.dropwizard.logging.layout.LayoutFactory;
import io.dropwizard.validation.PortRange;
@JsonTypeName("papertrail")
public class LoggingNetworkAppenderFactory extends AbstractAppenderFactory<ILoggingEvent> {
@JsonProperty
@NotEmpty
private String ident;
@JsonProperty
@NotEmpty
private String host;
@JsonProperty
@PortRange(min = 1)
@NotNull
private Integer port;
@JsonProperty
private TimeZone timeZone = TimeZone.getTimeZone("UTC");
@JsonProperty
@NotEmpty
private String facility = "USER";
@JsonProperty
private boolean sendLocalName = true;
public String getIdent() {
return ident;
}
public void setIdent(final String ident) {
this.ident = ident;
}
public String getHost() {
return host;
}
public void setHost(final String host) {
this.host = host;
}
public Integer getPort() {
return port;
}
public void setPort(final Integer port) {
this.port = port;
}
public TimeZone getTimeZone() {
return timeZone;
}
public void setTimeZone(final TimeZone timeZone) {
this.timeZone = timeZone;
}
public String getFacility() {
return facility;
}
public void setFacility(final String facility) {
this.facility = facility;
}
public boolean isSendLocalName() {
return sendLocalName;
}
public void setSendLocalName(final boolean sendLocalName) {
this.sendLocalName = sendLocalName;
}
@Override
public Appender<ILoggingEvent> build(LoggerContext context,
String applicationName,
LayoutFactory<ILoggingEvent> layoutFactory,
LevelFilterFactory<ILoggingEvent> levelFilterFactory,
AsyncAppenderFactory<ILoggingEvent> asyncAppenderFactory)
{
final Syslog4jAppender<ILoggingEvent> syslogAppender = new Syslog4jAppender<>();
syslogAppender.setContext(context);
syslogAppender.setLayout(buildLayout(context, layoutFactory));
syslogAppender.setName("SYSLOG-TLS");
final SSLTCPNetSyslogConfig syslogConfig = new SSLTCPNetSyslogConfig();
syslogConfig.setIdent(ident);
syslogConfig.setHost(host);
syslogConfig.setPort(port);
syslogConfig.setFacility(facility);
syslogConfig.setSendLocalName(sendLocalName);
syslogAppender.setSyslogConfig(syslogConfig);
syslogAppender.addFilter(levelFilterFactory.build(threshold));
syslogAppender.start();
return syslogAppender;
}
private static class Syslog4jAppender<E> extends AppenderBase<E> {
SyslogIF syslog;
SyslogConfigIF syslogConfig;
Layout<E> layout;
@Override
protected void append(E loggingEvent) {
syslog.log(getSeverityForEvent(loggingEvent), layout.doLayout(loggingEvent));
}
@Override
public void start() {
super.start();
synchronized (this) {
try {
Class syslogClass = syslogConfig.getSyslogClass();
syslog = (SyslogIF) syslogClass.newInstance();
syslog.initialize(syslogClass.getSimpleName(), syslogConfig);
} catch (ClassCastException | IllegalAccessException | InstantiationException cse) {
throw new SyslogRuntimeException(cse);
}
}
}
@Override
public void stop() {
super.stop();
synchronized(this) {
if (syslog != null) {
syslog.shutdown();
syslog = null;
}
}
}
/**
* Convert a level to equivalent syslog severity. Only levels for printing
* methods i.e DEBUG, WARN, INFO and ERROR are converted.
*
* @see ch.qos.logback.core.net.SyslogAppenderBase#getSeverityForEvent(java.lang.Object)
*/
public int getSeverityForEvent(Object eventObject) {
if (eventObject instanceof ILoggingEvent) {
ILoggingEvent event = (ILoggingEvent) eventObject;
return LevelToSyslogSeverity.convert(event);
} else {
return SyslogIF.LEVEL_INFO;
}
}
public SyslogConfigIF getSyslogConfig() {
return syslogConfig;
}
public void setSyslogConfig(SyslogConfigIF syslogConfig) {
this.syslogConfig = syslogConfig;
}
public Layout<E> getLayout() {
return layout;
}
public void setLayout(Layout<E> layout) {
this.layout = layout;
}
}
}

View File

@@ -0,0 +1,36 @@
package org.whispersystems.textsecuregcm.metrics;
import com.codahale.metrics.Gauge;
import org.whispersystems.textsecuregcm.util.Pair;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
public abstract class NetworkGauge implements Gauge<Double> {
protected Pair<Long, Long> getSentReceived() throws IOException {
File proc = new File("/proc/net/dev");
BufferedReader reader = new BufferedReader(new FileReader(proc));
String header = reader.readLine();
String header2 = reader.readLine();
long bytesSent = 0;
long bytesReceived = 0;
String interfaceStats;
while ((interfaceStats = reader.readLine()) != null) {
String[] stats = interfaceStats.split("\\s+");
if (!stats[1].equals("lo:")) {
bytesReceived += Long.parseLong(stats[2]);
bytesSent += Long.parseLong(stats[10]);
}
}
return new Pair<>(bytesSent, bytesReceived);
}
}

View File

@@ -0,0 +1,44 @@
package org.whispersystems.textsecuregcm.metrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.util.Pair;
import java.io.IOException;
public class NetworkReceivedGauge extends NetworkGauge {
private final Logger logger = LoggerFactory.getLogger(NetworkSentGauge.class);
private long lastTimestamp;
private long lastReceived;
public NetworkReceivedGauge() {
try {
this.lastTimestamp = System.currentTimeMillis();
this.lastReceived = getSentReceived().second();
} catch (IOException e) {
logger.warn(NetworkReceivedGauge.class.getSimpleName(), e);
}
}
@Override
public Double getValue() {
try {
long timestamp = System.currentTimeMillis();
Pair<Long, Long> sentAndReceived = getSentReceived();
double bytesReceived = sentAndReceived.second() - lastReceived;
double secondsElapsed = (timestamp - this.lastTimestamp) / 1000;
double result = bytesReceived / secondsElapsed;
this.lastTimestamp = timestamp;
this.lastReceived = sentAndReceived.second();
return result;
} catch (IOException e) {
logger.warn("NetworkReceivedGauge", e);
return -1D;
}
}
}

View File

@@ -0,0 +1,43 @@
package org.whispersystems.textsecuregcm.metrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.whispersystems.textsecuregcm.util.Pair;
import java.io.IOException;
public class NetworkSentGauge extends NetworkGauge {
private final Logger logger = LoggerFactory.getLogger(NetworkSentGauge.class);
private long lastTimestamp;
private long lastSent;
public NetworkSentGauge() {
try {
this.lastTimestamp = System.currentTimeMillis();
this.lastSent = getSentReceived().first();
} catch (IOException e) {
logger.warn(NetworkSentGauge.class.getSimpleName(), e);
}
}
@Override
public Double getValue() {
try {
long timestamp = System.currentTimeMillis();
Pair<Long, Long> sentAndReceived = getSentReceived();
double bytesTransmitted = sentAndReceived.first() - lastSent;
double secondsElapsed = (timestamp - this.lastTimestamp) / 1000;
double result = bytesTransmitted / secondsElapsed;
this.lastSent = sentAndReceived.first();
this.lastTimestamp = timestamp;
return result;
} catch (IOException e) {
logger.warn("NetworkSentGauge", e);
return -1D;
}
}
}