1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368

// SPDXLicenseIdentifier: GPL2.0
/*
* Copyright (C) 2016, Linaro Ltd  Daniel Lezcano <daniel.lezcano@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
*/
#include <linux/kernel.h>
#include <linux/percpu.h>
#include <linux/slab.h>
#include <linux/static_key.h>
#include <linux/interrupt.h>
#include <linux/idr.h>
#include <linux/irq.h>
#include <linux/math64.h>
#include <trace/events/irq.h>
#include "internals.h"
DEFINE_STATIC_KEY_FALSE(irq_timing_enabled);
DEFINE_PER_CPU(struct irq_timings, irq_timings);
struct irqt_stat {
u64 next_evt;
u64 last_ts;
u64 variance;
u32 avg;
u32 nr_samples;
int anomalies;
int valid;
};
static DEFINE_IDR(irqt_stats);
void irq_timings_enable(void)
{
static_branch_enable(&irq_timing_enabled);
}
void irq_timings_disable(void)
{
static_branch_disable(&irq_timing_enabled);
}
/**
* irqs_update  update the irq timing statistics with a new timestamp
*
* @irqs: an irqt_stat struct pointer
* @ts: the new timestamp
*
* The statistics are computed online, in other words, the code is
* designed to compute the statistics on a stream of values rather
* than doing multiple passes on the values to compute the average,
* then the variance. The integer division introduces a loss of
* precision but with an acceptable error margin regarding the results
* we would have with the double floating precision: we are dealing
* with nanosec, so big numbers, consequently the mantisse is
* negligeable, especially when converting the time in usec
* afterwards.
*
* The computation happens at idle time. When the CPU is not idle, the
* interrupts' timestamps are stored in the circular buffer, when the
* CPU goes idle and this routine is called, all the buffer's values
* are injected in the statistical model continuying to extend the
* statistics from the previous busyidle cycle.
*
* The observations showed a device will trigger a burst of periodic
* interrupts followed by one or two peaks of longer time, for
* instance when a SD card device flushes its cache, then the periodic
* intervals occur again. A one second inactivity period resets the
* stats, that gives us the certitude the statistical values won't
* exceed 1x10^9, thus the computation won't overflow.
*
* Basically, the purpose of the algorithm is to watch the periodic
* interrupts and eliminate the peaks.
*
* An interrupt is considered periodically stable if the interval of
* its occurences follow the normal distribution, thus the values
* comply with:
*
* avg  3 x stddev < value < avg + 3 x stddev
*
* Which can be simplified to:
*
* 3 x stddev < value  avg < 3 x stddev
*
* abs(value  avg) < 3 x stddev
*
* In order to save a costly square root computation, we use the
* variance. For the record, stddev = sqrt(variance). The equation
* above becomes:
*
* abs(value  avg) < 3 x sqrt(variance)
*
* And finally we square it:
*
* (value  avg) ^ 2 < (3 x sqrt(variance)) ^ 2
*
* (value  avg) x (value  avg) < 9 x variance
*
* Statistically speaking, any values out of this interval is
* considered as an anomaly and is discarded. However, a normal
* distribution appears when the number of samples is 30 (it is the
* rule of thumb in statistics, cf. "30 samples" on Internet). When
* there are three consecutive anomalies, the statistics are resetted.
*
*/
static void irqs_update(struct irqt_stat *irqs, u64 ts)
{
u64 old_ts = irqs>last_ts;
u64 variance = 0;
u64 interval;
s64 diff;
/*
* The timestamps are absolute time values, we need to compute
* the timing interval between two interrupts.
*/
irqs>last_ts = ts;
/*
* The interval type is u64 in order to deal with the same
* type in our computation, that prevent mindfuck issues with
* overflow, sign and division.
*/
interval = ts  old_ts;
/*
* The interrupt triggered more than one second apart, that
* ends the sequence as predictible for our purpose. In this
* case, assume we have the beginning of a sequence and the
* timestamp is the first value. As it is impossible to
* predict anything at this point, return.
*
* Note the first timestamp of the sequence will always fall
* in this test because the old_ts is zero. That is what we
* want as we need another timestamp to compute an interval.
*/
if (interval >= NSEC_PER_SEC) {
memset(irqs, 0, sizeof(*irqs));
irqs>last_ts = ts;
return;
}
/*
* Precompute the delta with the average as the result is
* used several times in this function.
*/
diff = interval  irqs>avg;
/*
* Increment the number of samples.
*/
irqs>nr_samples++;
/*
* Online variance divided by the number of elements if there
* is more than one sample. Normally the formula is division
* by nr_samples  1 but we assume the number of element will be
* more than 32 and dividing by 32 instead of 31 is enough
* precise.
*/
if (likely(irqs>nr_samples > 1))
variance = irqs>variance >> IRQ_TIMINGS_SHIFT;
/*
* The rule of thumb in statistics for the normal distribution
* is having at least 30 samples in order to have the model to
* apply. Values outside the interval are considered as an
* anomaly.
*/
if ((irqs>nr_samples >= 30) && ((diff * diff) > (9 * variance))) {
/*
* After three consecutive anomalies, we reset the
* stats as it is no longer stable enough.
*/
if (irqs>anomalies++ >= 3) {
memset(irqs, 0, sizeof(*irqs));
irqs>last_ts = ts;
return;
}
} else {
/*
* The anomalies must be consecutives, so at this
* point, we reset the anomalies counter.
*/
irqs>anomalies = 0;
}
/*
* The interrupt is considered stable enough to try to predict
* the next event on it.
*/
irqs>valid = 1;
/*
* Online average algorithm:
*
* new_average = average + ((value  average) / count)
*
* The variance computation depends on the new average
* to be computed here first.
*
*/
irqs>avg = irqs>avg + (diff >> IRQ_TIMINGS_SHIFT);
/*
* Online variance algorithm:
*
* new_variance = variance + (value  average) x (value  new_average)
*
* Warning: irqs>avg is updated with the line above, hence
* 'interval  irqs>avg' is no longer equal to 'diff'
*/
irqs>variance = irqs>variance + (diff * (interval  irqs>avg));
/*
* Update the next event
*/
irqs>next_evt = ts + irqs>avg;
}
/**
* irq_timings_next_event  Return when the next event is supposed to arrive
*
* During the last busy cycle, the number of interrupts is incremented
* and stored in the irq_timings structure. This information is
* necessary to:
*
*  know if the index in the table wrapped up:
*
* If more than the array size interrupts happened during the
* last busy/idle cycle, the index wrapped up and we have to
* begin with the next element in the array which is the last one
* in the sequence, otherwise it is a the index 0.
*
*  have an indication of the interrupts activity on this CPU
* (eg. irq/sec)
*
* The values are 'consumed' after inserting in the statistical model,
* thus the count is reinitialized.
*
* The array of values **must** be browsed in the time direction, the
* timestamp must increase between an element and the next one.
*
* Returns a nanosec time based estimation of the earliest interrupt,
* U64_MAX otherwise.
*/
u64 irq_timings_next_event(u64 now)
{
struct irq_timings *irqts = this_cpu_ptr(&irq_timings);
struct irqt_stat *irqs;
struct irqt_stat __percpu *s;
u64 ts, next_evt = U64_MAX;
int i, irq = 0;
/*
* This function must be called with the local irq disabled in
* order to prevent the timings circular buffer to be updated
* while we are reading it.
*/
lockdep_assert_irqs_disabled();
/*
* Number of elements in the circular buffer: If it happens it
* was flushed before, then the number of elements could be
* smaller than IRQ_TIMINGS_SIZE, so the count is used,
* otherwise the array size is used as we wrapped. The index
* begins from zero when we did not wrap. That could be done
* in a nicer way with the proper circular array structure
* type but with the cost of extra computation in the
* interrupt handler hot path. We choose efficiency.
*
* Inject measured irq/timestamp to the statistical model
* while decrementing the counter because we consume the data
* from our circular buffer.
*/
for (i = irqts>count & IRQ_TIMINGS_MASK,
irqts>count = min(IRQ_TIMINGS_SIZE, irqts>count);
irqts>count > 0; irqts>count, i = (i + 1) & IRQ_TIMINGS_MASK) {
irq = irq_timing_decode(irqts>values[i], &ts);
s = idr_find(&irqt_stats, irq);
if (s) {
irqs = this_cpu_ptr(s);
irqs_update(irqs, ts);
}
}
/*
* Look in the list of interrupts' statistics, the earliest
* next event.
*/
idr_for_each_entry(&irqt_stats, s, i) {
irqs = this_cpu_ptr(s);
if (!irqs>valid)
continue;
if (irqs>next_evt <= now) {
irq = i;
next_evt = now;
/*
* This interrupt mustn't use in the future
* until new events occur and update the
* statistics.
*/
irqs>valid = 0;
break;
}
if (irqs>next_evt < next_evt) {
irq = i;
next_evt = irqs>next_evt;
}
}
return next_evt;
}
void irq_timings_free(int irq)
{
struct irqt_stat __percpu *s;
s = idr_find(&irqt_stats, irq);
if (s) {
free_percpu(s);
idr_remove(&irqt_stats, irq);
}
}
int irq_timings_alloc(int irq)
{
struct irqt_stat __percpu *s;
int id;
/*
* Some platforms can have the same private interrupt per cpu,
* so this function may be be called several times with the
* same interrupt number. Just bail out in case the per cpu
* stat structure is already allocated.
*/
s = idr_find(&irqt_stats, irq);
if (s)
return 0;
s = alloc_percpu(*s);
if (!s)
return ENOMEM;
idr_preload(GFP_KERNEL);
id = idr_alloc(&irqt_stats, s, irq, irq + 1, GFP_NOWAIT);
idr_preload_end();
if (id < 0) {
free_percpu(s);
return id;
}
return 0;
}
