001package io.prometheus.metrics.core.exemplars;
002
003import io.prometheus.metrics.tracer.common.SpanContext;
004import io.prometheus.metrics.tracer.initializer.SpanContextSupplier;
005import io.prometheus.metrics.model.snapshots.Exemplar;
006import io.prometheus.metrics.model.snapshots.Exemplars;
007import io.prometheus.metrics.model.snapshots.Labels;
008import io.prometheus.metrics.core.util.Scheduler;
009
010import java.util.ArrayList;
011import java.util.List;
012import java.util.concurrent.TimeUnit;
013import java.util.concurrent.atomic.AtomicBoolean;
014import java.util.function.LongSupplier;
015
016/**
017 * The ExemplarSampler selects Spans as exemplars.
018 * <p>
019 * There are two types of Exemplars: Regular exemplars are sampled implicitly if a supported tracing
020 * library is detected. Custom exemplars are provided explicitly in code, for example if a developer
021 * wants to make sure an Exemplar is created for a specific code path.
022 * <p>
023 * Spans will be marked as being an Exemplar by calling {@link SpanContext#markCurrentSpanAsExemplar()}.
024 * The tracer implementation should set a Span attribute to mark the current Span as an Exemplar.
025 * This attribute can be used by a trace sampling algorithm to make sure traces with Exemplars are sampled.
026 * <p>
027 * The ExemplarSample is rate-limited, so only a small fraction of Spans will be marked as Exemplars in
028 * an application with a large number of requests.
029 * <p>
030 * See {@link ExemplarSamplerConfig} for configuration options.
031 */
032public class ExemplarSampler {
033
034    private final ExemplarSamplerConfig config;
035    private final Exemplar[] exemplars;
036    private final Exemplar[] customExemplars; // Separate from exemplars, because we don't want custom exemplars
037    // to be overwritten by automatic exemplar sampling. exemplars.lengt == customExemplars.length
038    private final AtomicBoolean acceptingNewExemplars = new AtomicBoolean(true);
039    private final AtomicBoolean acceptingNewCustomExemplars = new AtomicBoolean(true);
040
041    public ExemplarSampler(ExemplarSamplerConfig config) {
042        this.config = config;
043        this.exemplars = new Exemplar[config.getNumberOfExemplars()];
044        this.customExemplars = new Exemplar[exemplars.length];
045    }
046
047    public Exemplars collect() {
048        // this may run in parallel with observe()
049        long now = System.currentTimeMillis();
050        List<Exemplar> result = new ArrayList<>(exemplars.length);
051        for (int i = 0; i < customExemplars.length; i++) {
052            Exemplar exemplar = customExemplars[i];
053            if (exemplar != null) {
054                if (now - exemplar.getTimestampMillis() > config.getMaxRetentionPeriodMillis()) {
055                    customExemplars[i] = null;
056                } else {
057                    result.add(exemplar);
058                }
059            }
060        }
061        for (int i = 0; i < exemplars.length && result.size() < exemplars.length; i++) {
062            Exemplar exemplar = exemplars[i];
063            if (exemplar != null) {
064                if (now - exemplar.getTimestampMillis() > config.getMaxRetentionPeriodMillis()) {
065                    exemplars[i] = null;
066                } else {
067                    result.add(exemplar);
068                }
069            }
070        }
071        return Exemplars.of(result);
072    }
073
074    public void reset() {
075        for (int i = 0; i < exemplars.length; i++) {
076            exemplars[i] = null;
077            customExemplars[i] = null;
078        }
079    }
080
081    public void observe(double value) {
082        if (!acceptingNewExemplars.get()) {
083            return; // This is the hot path in a high-throughput application and should be as efficient as possible.
084        }
085        rateLimitedObserve(acceptingNewExemplars, value, exemplars, () -> doObserve(value));
086    }
087
088    public void observeWithExemplar(double value, Labels labels) {
089        if (!acceptingNewCustomExemplars.get()) {
090            return; // This is the hot path in a high-throughput application and should be as efficient as possible.
091        }
092        rateLimitedObserve(acceptingNewCustomExemplars, value, customExemplars, () -> doObserveWithExemplar(value, labels));
093    }
094
095    private long doObserve(double value) {
096        if (exemplars.length == 1) {
097            return doObserveSingleExemplar(value);
098        } else if (config.getHistogramClassicUpperBounds() != null) {
099            return doObserveWithUpperBounds(value);
100        } else {
101            return doObserveWithoutUpperBounds(value);
102        }
103    }
104
105    private long doObserveSingleExemplar(double value) {
106        long now = System.currentTimeMillis();
107        Exemplar current = exemplars[0];
108        if (current == null || now - current.getTimestampMillis() > config.getMinRetentionPeriodMillis()) {
109            return updateExemplar(0, value, now);
110        }
111        return 0;
112    }
113
114    private long doObserveWithUpperBounds(double value) {
115        long now = System.currentTimeMillis();
116        double[] upperBounds = config.getHistogramClassicUpperBounds();
117        for (int i = 0; i < upperBounds.length; i++) {
118            if (value <= upperBounds[i]) {
119                Exemplar previous = exemplars[i];
120                if (previous == null || now - previous.getTimestampMillis() > config.getMinRetentionPeriodMillis()) {
121                    return updateExemplar(i, value, now);
122                } else {
123                    return 0;
124                }
125            }
126        }
127        return 0; // will never happen, as upperBounds contains +Inf
128    }
129
130    private long doObserveWithoutUpperBounds(double value) {
131        final long now = System.currentTimeMillis();
132        Exemplar smallest = null;
133        int smallestIndex = -1;
134        Exemplar largest = null;
135        int largestIndex = -1;
136        int nullIndex = -1;
137        for (int i = exemplars.length - 1; i >= 0; i--) {
138            Exemplar exemplar = exemplars[i];
139            if (exemplar == null) {
140                nullIndex = i;
141            } else if (now - exemplar.getTimestampMillis() > config.getMaxRetentionPeriodMillis()) {
142                exemplars[i] = null;
143                nullIndex = i;
144            } else {
145                if (smallest == null || exemplar.getValue() < smallest.getValue()) {
146                    smallest = exemplar;
147                    smallestIndex = i;
148                }
149                if (largest == null || exemplar.getValue() > largest.getValue()) {
150                    largest = exemplar;
151                    largestIndex = i;
152                }
153            }
154        }
155        if (nullIndex >= 0) {
156            return updateExemplar(nullIndex, value, now);
157        }
158        if (now - smallest.getTimestampMillis() > config.getMinRetentionPeriodMillis() && value < smallest.getValue()) {
159            return updateExemplar(smallestIndex, value, now);
160        }
161        if (now - largest.getTimestampMillis() > config.getMinRetentionPeriodMillis() && value > largest.getValue()) {
162            return updateExemplar(largestIndex, value, now);
163        }
164        long oldestTimestamp = 0;
165        int oldestIndex = -1;
166        for (int i = 0; i < exemplars.length; i++) {
167            Exemplar exemplar = exemplars[i];
168            if (exemplar != null && exemplar != smallest && exemplar != largest) {
169                if (oldestTimestamp == 0 || exemplar.getTimestampMillis() < oldestTimestamp) {
170                    oldestTimestamp = exemplar.getTimestampMillis();
171                    oldestIndex = i;
172                }
173            }
174        }
175        if (oldestIndex != -1 && now - oldestTimestamp > config.getMinRetentionPeriodMillis()) {
176            return updateExemplar(oldestIndex, value, now);
177        }
178        return 0;
179    }
180
181    // Returns the timestamp of the newly added Exemplar (which is System.currentTimeMillis())
182    // or 0 if no Exemplar was added.
183    private long doObserveWithExemplar(double amount, Labels labels) {
184        if (customExemplars.length == 1) {
185            return doObserveSingleExemplar(amount, labels);
186        } else if (config.getHistogramClassicUpperBounds() != null) {
187            return doObserveWithExemplarWithUpperBounds(amount, labels);
188        } else {
189            return doObserveWithExemplarWithoutUpperBounds(amount, labels);
190        }
191    }
192
193    private long doObserveSingleExemplar(double amount, Labels labels) {
194        long now = System.currentTimeMillis();
195        Exemplar current = customExemplars[0];
196        if (current == null || now - current.getTimestampMillis() > config.getMinRetentionPeriodMillis()) {
197            return updateCustomExemplar(0, amount, labels, now);
198        }
199        return 0;
200    }
201
202    private long doObserveWithExemplarWithUpperBounds(double value, Labels labels) {
203        long now = System.currentTimeMillis();
204        double[] upperBounds = config.getHistogramClassicUpperBounds();
205        for (int i = 0; i < upperBounds.length; i++) {
206            if (value <= upperBounds[i]) {
207                Exemplar previous = customExemplars[i];
208                if (previous == null || now - previous.getTimestampMillis() > config.getMinRetentionPeriodMillis()) {
209                    return updateCustomExemplar(i, value, labels, now);
210                } else {
211                    return 0;
212                }
213            }
214        }
215        return 0; // will never happen, as upperBounds contains +Inf
216    }
217
218    private long doObserveWithExemplarWithoutUpperBounds(double amount, Labels labels) {
219        final long now = System.currentTimeMillis();
220        int nullPos = -1;
221        int oldestPos = -1;
222        Exemplar oldest = null;
223        for (int i = customExemplars.length - 1; i >= 0; i--) {
224            Exemplar exemplar = customExemplars[i];
225            if (exemplar == null) {
226                nullPos = i;
227            } else if (now - exemplar.getTimestampMillis() > config.getMaxRetentionPeriodMillis()) {
228                customExemplars[i] = null;
229                nullPos = i;
230            } else {
231                if (oldest == null || exemplar.getTimestampMillis() < oldest.getTimestampMillis()) {
232                    oldest = exemplar;
233                    oldestPos = i;
234                }
235            }
236        }
237        if (nullPos != -1) {
238            return updateCustomExemplar(nullPos, amount, labels, now);
239        } else if (now - oldest.getTimestampMillis() > config.getMinRetentionPeriodMillis()) {
240            return updateCustomExemplar(oldestPos, amount, labels, now);
241        } else {
242            return 0;
243        }
244    }
245
246    /**
247     * Observing requires a system call to {@link System#currentTimeMillis()},
248     * and it requires iterating over the existing exemplars to check if one of the existing
249     * exemplars can be replaced.
250     * <p>
251     * To avoid performance issues, we rate limit observing exemplars to
252     * {@link ExemplarSamplerConfig#getSampleIntervalMillis()} milliseconds.
253     */
254    private void rateLimitedObserve(AtomicBoolean accepting, double value, Exemplar[] exemplars, LongSupplier observeFunc) {
255        if (Double.isNaN(value)) {
256            return;
257        }
258        if (!accepting.compareAndSet(true, false)) {
259            return;
260        }
261        // observeFunc returns the current timestamp or 0 if no Exemplar was added.
262        long now = observeFunc.getAsLong();
263        long sleepTime = now == 0 ? config.getSampleIntervalMillis() : durationUntilNextExemplarExpires(now);
264        Scheduler.schedule(() -> accepting.compareAndSet(false, true), sleepTime, TimeUnit.MILLISECONDS);
265    }
266
267    private long durationUntilNextExemplarExpires(long now) {
268        long oldestTimestamp = now;
269        for (Exemplar exemplar : exemplars) {
270            if (exemplar == null) {
271                return config.getSampleIntervalMillis();
272            } else if (exemplar.getTimestampMillis() < oldestTimestamp) {
273                oldestTimestamp = exemplar.getTimestampMillis();
274            }
275        }
276        long oldestAge = now - oldestTimestamp;
277        if (oldestAge < config.getMinRetentionPeriodMillis()) {
278            return config.getMinRetentionPeriodMillis() - oldestAge;
279        }
280        return config.getSampleIntervalMillis();
281    }
282
283    private long updateCustomExemplar(int index, double value, Labels labels, long now) {
284        if (!labels.contains(Exemplar.TRACE_ID) && !labels.contains(Exemplar.SPAN_ID)) {
285            labels = labels.merge(doSampleExemplar());
286        }
287        customExemplars[index] = Exemplar.builder()
288                .value(value)
289                .labels(labels)
290                .timestampMillis(now)
291                .build();
292        return now;
293    }
294
295    private long updateExemplar(int index, double value, long now) {
296        Labels traceLabels = doSampleExemplar();
297        if (!traceLabels.isEmpty()) {
298            exemplars[index] = Exemplar.builder()
299                    .value(value)
300                    .labels(traceLabels)
301                    .timestampMillis(now)
302                    .build();
303            return now;
304        } else {
305            return 0;
306        }
307    }
308
309    private Labels doSampleExemplar() {
310        try {
311            SpanContext spanContext = SpanContextSupplier.getSpanContext();
312            if (spanContext != null) {
313                if (spanContext.isCurrentSpanSampled()) {
314                    String spanId = spanContext.getCurrentSpanId();
315                    String traceId = spanContext.getCurrentTraceId();
316                    if (spanId != null && traceId != null) {
317                        spanContext.markCurrentSpanAsExemplar();
318                        return Labels.of(Exemplar.TRACE_ID, traceId, Exemplar.SPAN_ID, spanId);
319                    }
320                }
321            }
322        } catch (NoClassDefFoundError ignored) {
323        }
324        return Labels.EMPTY;
325    }
326}