001package io.prometheus.metrics.core.exemplars; 002 003import io.prometheus.metrics.tracer.common.SpanContext; 004import io.prometheus.metrics.tracer.initializer.SpanContextSupplier; 005import io.prometheus.metrics.model.snapshots.Exemplar; 006import io.prometheus.metrics.model.snapshots.Exemplars; 007import io.prometheus.metrics.model.snapshots.Labels; 008import io.prometheus.metrics.core.util.Scheduler; 009 010import java.util.ArrayList; 011import java.util.List; 012import java.util.concurrent.TimeUnit; 013import java.util.concurrent.atomic.AtomicBoolean; 014import java.util.function.LongSupplier; 015 016/** 017 * The ExemplarSampler selects Spans as exemplars. 018 * <p> 019 * There are two types of Exemplars: Regular exemplars are sampled implicitly if a supported tracing 020 * library is detected. Custom exemplars are provided explicitly in code, for example if a developer 021 * wants to make sure an Exemplar is created for a specific code path. 022 * <p> 023 * Spans will be marked as being an Exemplar by calling {@link SpanContext#markCurrentSpanAsExemplar()}. 024 * The tracer implementation should set a Span attribute to mark the current Span as an Exemplar. 025 * This attribute can be used by a trace sampling algorithm to make sure traces with Exemplars are sampled. 026 * <p> 027 * The ExemplarSample is rate-limited, so only a small fraction of Spans will be marked as Exemplars in 028 * an application with a large number of requests. 029 * <p> 030 * See {@link ExemplarSamplerConfig} for configuration options. 031 */ 032public class ExemplarSampler { 033 034 private final ExemplarSamplerConfig config; 035 private final Exemplar[] exemplars; 036 private final Exemplar[] customExemplars; // Separate from exemplars, because we don't want custom exemplars 037 // to be overwritten by automatic exemplar sampling. exemplars.lengt == customExemplars.length 038 private final AtomicBoolean acceptingNewExemplars = new AtomicBoolean(true); 039 private final AtomicBoolean acceptingNewCustomExemplars = new AtomicBoolean(true); 040 041 public ExemplarSampler(ExemplarSamplerConfig config) { 042 this.config = config; 043 this.exemplars = new Exemplar[config.getNumberOfExemplars()]; 044 this.customExemplars = new Exemplar[exemplars.length]; 045 } 046 047 public Exemplars collect() { 048 // this may run in parallel with observe() 049 long now = System.currentTimeMillis(); 050 List<Exemplar> result = new ArrayList<>(exemplars.length); 051 for (int i = 0; i < customExemplars.length; i++) { 052 Exemplar exemplar = customExemplars[i]; 053 if (exemplar != null) { 054 if (now - exemplar.getTimestampMillis() > config.getMaxRetentionPeriodMillis()) { 055 customExemplars[i] = null; 056 } else { 057 result.add(exemplar); 058 } 059 } 060 } 061 for (int i = 0; i < exemplars.length && result.size() < exemplars.length; i++) { 062 Exemplar exemplar = exemplars[i]; 063 if (exemplar != null) { 064 if (now - exemplar.getTimestampMillis() > config.getMaxRetentionPeriodMillis()) { 065 exemplars[i] = null; 066 } else { 067 result.add(exemplar); 068 } 069 } 070 } 071 return Exemplars.of(result); 072 } 073 074 public void reset() { 075 for (int i = 0; i < exemplars.length; i++) { 076 exemplars[i] = null; 077 customExemplars[i] = null; 078 } 079 } 080 081 public void observe(double value) { 082 if (!acceptingNewExemplars.get()) { 083 return; // This is the hot path in a high-throughput application and should be as efficient as possible. 084 } 085 rateLimitedObserve(acceptingNewExemplars, value, exemplars, () -> doObserve(value)); 086 } 087 088 public void observeWithExemplar(double value, Labels labels) { 089 if (!acceptingNewCustomExemplars.get()) { 090 return; // This is the hot path in a high-throughput application and should be as efficient as possible. 091 } 092 rateLimitedObserve(acceptingNewCustomExemplars, value, customExemplars, () -> doObserveWithExemplar(value, labels)); 093 } 094 095 private long doObserve(double value) { 096 if (exemplars.length == 1) { 097 return doObserveSingleExemplar(value); 098 } else if (config.getHistogramClassicUpperBounds() != null) { 099 return doObserveWithUpperBounds(value); 100 } else { 101 return doObserveWithoutUpperBounds(value); 102 } 103 } 104 105 private long doObserveSingleExemplar(double value) { 106 long now = System.currentTimeMillis(); 107 Exemplar current = exemplars[0]; 108 if (current == null || now - current.getTimestampMillis() > config.getMinRetentionPeriodMillis()) { 109 return updateExemplar(0, value, now); 110 } 111 return 0; 112 } 113 114 private long doObserveWithUpperBounds(double value) { 115 long now = System.currentTimeMillis(); 116 double[] upperBounds = config.getHistogramClassicUpperBounds(); 117 for (int i = 0; i < upperBounds.length; i++) { 118 if (value <= upperBounds[i]) { 119 Exemplar previous = exemplars[i]; 120 if (previous == null || now - previous.getTimestampMillis() > config.getMinRetentionPeriodMillis()) { 121 return updateExemplar(i, value, now); 122 } else { 123 return 0; 124 } 125 } 126 } 127 return 0; // will never happen, as upperBounds contains +Inf 128 } 129 130 private long doObserveWithoutUpperBounds(double value) { 131 final long now = System.currentTimeMillis(); 132 Exemplar smallest = null; 133 int smallestIndex = -1; 134 Exemplar largest = null; 135 int largestIndex = -1; 136 int nullIndex = -1; 137 for (int i = exemplars.length - 1; i >= 0; i--) { 138 Exemplar exemplar = exemplars[i]; 139 if (exemplar == null) { 140 nullIndex = i; 141 } else if (now - exemplar.getTimestampMillis() > config.getMaxRetentionPeriodMillis()) { 142 exemplars[i] = null; 143 nullIndex = i; 144 } else { 145 if (smallest == null || exemplar.getValue() < smallest.getValue()) { 146 smallest = exemplar; 147 smallestIndex = i; 148 } 149 if (largest == null || exemplar.getValue() > largest.getValue()) { 150 largest = exemplar; 151 largestIndex = i; 152 } 153 } 154 } 155 if (nullIndex >= 0) { 156 return updateExemplar(nullIndex, value, now); 157 } 158 if (now - smallest.getTimestampMillis() > config.getMinRetentionPeriodMillis() && value < smallest.getValue()) { 159 return updateExemplar(smallestIndex, value, now); 160 } 161 if (now - largest.getTimestampMillis() > config.getMinRetentionPeriodMillis() && value > largest.getValue()) { 162 return updateExemplar(largestIndex, value, now); 163 } 164 long oldestTimestamp = 0; 165 int oldestIndex = -1; 166 for (int i = 0; i < exemplars.length; i++) { 167 Exemplar exemplar = exemplars[i]; 168 if (exemplar != null && exemplar != smallest && exemplar != largest) { 169 if (oldestTimestamp == 0 || exemplar.getTimestampMillis() < oldestTimestamp) { 170 oldestTimestamp = exemplar.getTimestampMillis(); 171 oldestIndex = i; 172 } 173 } 174 } 175 if (oldestIndex != -1 && now - oldestTimestamp > config.getMinRetentionPeriodMillis()) { 176 return updateExemplar(oldestIndex, value, now); 177 } 178 return 0; 179 } 180 181 // Returns the timestamp of the newly added Exemplar (which is System.currentTimeMillis()) 182 // or 0 if no Exemplar was added. 183 private long doObserveWithExemplar(double amount, Labels labels) { 184 if (customExemplars.length == 1) { 185 return doObserveSingleExemplar(amount, labels); 186 } else if (config.getHistogramClassicUpperBounds() != null) { 187 return doObserveWithExemplarWithUpperBounds(amount, labels); 188 } else { 189 return doObserveWithExemplarWithoutUpperBounds(amount, labels); 190 } 191 } 192 193 private long doObserveSingleExemplar(double amount, Labels labels) { 194 long now = System.currentTimeMillis(); 195 Exemplar current = customExemplars[0]; 196 if (current == null || now - current.getTimestampMillis() > config.getMinRetentionPeriodMillis()) { 197 return updateCustomExemplar(0, amount, labels, now); 198 } 199 return 0; 200 } 201 202 private long doObserveWithExemplarWithUpperBounds(double value, Labels labels) { 203 long now = System.currentTimeMillis(); 204 double[] upperBounds = config.getHistogramClassicUpperBounds(); 205 for (int i = 0; i < upperBounds.length; i++) { 206 if (value <= upperBounds[i]) { 207 Exemplar previous = customExemplars[i]; 208 if (previous == null || now - previous.getTimestampMillis() > config.getMinRetentionPeriodMillis()) { 209 return updateCustomExemplar(i, value, labels, now); 210 } else { 211 return 0; 212 } 213 } 214 } 215 return 0; // will never happen, as upperBounds contains +Inf 216 } 217 218 private long doObserveWithExemplarWithoutUpperBounds(double amount, Labels labels) { 219 final long now = System.currentTimeMillis(); 220 int nullPos = -1; 221 int oldestPos = -1; 222 Exemplar oldest = null; 223 for (int i = customExemplars.length - 1; i >= 0; i--) { 224 Exemplar exemplar = customExemplars[i]; 225 if (exemplar == null) { 226 nullPos = i; 227 } else if (now - exemplar.getTimestampMillis() > config.getMaxRetentionPeriodMillis()) { 228 customExemplars[i] = null; 229 nullPos = i; 230 } else { 231 if (oldest == null || exemplar.getTimestampMillis() < oldest.getTimestampMillis()) { 232 oldest = exemplar; 233 oldestPos = i; 234 } 235 } 236 } 237 if (nullPos != -1) { 238 return updateCustomExemplar(nullPos, amount, labels, now); 239 } else if (now - oldest.getTimestampMillis() > config.getMinRetentionPeriodMillis()) { 240 return updateCustomExemplar(oldestPos, amount, labels, now); 241 } else { 242 return 0; 243 } 244 } 245 246 /** 247 * Observing requires a system call to {@link System#currentTimeMillis()}, 248 * and it requires iterating over the existing exemplars to check if one of the existing 249 * exemplars can be replaced. 250 * <p> 251 * To avoid performance issues, we rate limit observing exemplars to 252 * {@link ExemplarSamplerConfig#getSampleIntervalMillis()} milliseconds. 253 */ 254 private void rateLimitedObserve(AtomicBoolean accepting, double value, Exemplar[] exemplars, LongSupplier observeFunc) { 255 if (Double.isNaN(value)) { 256 return; 257 } 258 if (!accepting.compareAndSet(true, false)) { 259 return; 260 } 261 // observeFunc returns the current timestamp or 0 if no Exemplar was added. 262 long now = observeFunc.getAsLong(); 263 long sleepTime = now == 0 ? config.getSampleIntervalMillis() : durationUntilNextExemplarExpires(now); 264 Scheduler.schedule(() -> accepting.compareAndSet(false, true), sleepTime, TimeUnit.MILLISECONDS); 265 } 266 267 private long durationUntilNextExemplarExpires(long now) { 268 long oldestTimestamp = now; 269 for (Exemplar exemplar : exemplars) { 270 if (exemplar == null) { 271 return config.getSampleIntervalMillis(); 272 } else if (exemplar.getTimestampMillis() < oldestTimestamp) { 273 oldestTimestamp = exemplar.getTimestampMillis(); 274 } 275 } 276 long oldestAge = now - oldestTimestamp; 277 if (oldestAge < config.getMinRetentionPeriodMillis()) { 278 return config.getMinRetentionPeriodMillis() - oldestAge; 279 } 280 return config.getSampleIntervalMillis(); 281 } 282 283 private long updateCustomExemplar(int index, double value, Labels labels, long now) { 284 if (!labels.contains(Exemplar.TRACE_ID) && !labels.contains(Exemplar.SPAN_ID)) { 285 labels = labels.merge(doSampleExemplar()); 286 } 287 customExemplars[index] = Exemplar.builder() 288 .value(value) 289 .labels(labels) 290 .timestampMillis(now) 291 .build(); 292 return now; 293 } 294 295 private long updateExemplar(int index, double value, long now) { 296 Labels traceLabels = doSampleExemplar(); 297 if (!traceLabels.isEmpty()) { 298 exemplars[index] = Exemplar.builder() 299 .value(value) 300 .labels(traceLabels) 301 .timestampMillis(now) 302 .build(); 303 return now; 304 } else { 305 return 0; 306 } 307 } 308 309 private Labels doSampleExemplar() { 310 try { 311 SpanContext spanContext = SpanContextSupplier.getSpanContext(); 312 if (spanContext != null) { 313 if (spanContext.isCurrentSpanSampled()) { 314 String spanId = spanContext.getCurrentSpanId(); 315 String traceId = spanContext.getCurrentTraceId(); 316 if (spanId != null && traceId != null) { 317 spanContext.markCurrentSpanAsExemplar(); 318 return Labels.of(Exemplar.TRACE_ID, traceId, Exemplar.SPAN_ID, spanId); 319 } 320 } 321 } 322 } catch (NoClassDefFoundError ignored) { 323 } 324 return Labels.EMPTY; 325 } 326}