Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229// SPDX-License-Identifier: GPL-2.0/* * Block stat tracking code * * Copyright (C) 2016 Jens Axboe */#include <linux/kernel.h>#include <linux/rculist.h>
#include "blk-stat.h"#include "blk-mq.h"#include "blk.h"
struct blk_queue_stats { struct list_head callbacks; spinlock_t lock; int accounting;};
void blk_rq_stat_init(struct blk_rq_stat *stat){ stat->min = -1ULL; stat->max = stat->nr_samples = stat->mean = 0; stat->batch = 0;}
/* src is a per-cpu stat, mean isn't initialized */void blk_rq_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src){ if (dst->nr_samples + src->nr_samples <= dst->nr_samples) return;
dst->min = min(dst->min, src->min); dst->max = max(dst->max, src->max);
dst->mean = div_u64(src->batch + dst->mean * dst->nr_samples, dst->nr_samples + src->nr_samples);
dst->nr_samples += src->nr_samples;}
void blk_rq_stat_add(struct blk_rq_stat *stat, u64 value){ stat->min = min(stat->min, value); stat->max = max(stat->max, value); stat->batch += value; stat->nr_samples++;}
void blk_stat_add(struct request *rq, u64 now){ struct request_queue *q = rq->q; struct blk_stat_callback *cb; struct blk_rq_stat *stat; int bucket, cpu; u64 value;
value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;
rcu_read_lock(); cpu = get_cpu(); list_for_each_entry_rcu(cb, &q->stats->callbacks, list) { if (!blk_stat_is_active(cb)) continue;
bucket = cb->bucket_fn(rq); if (bucket < 0) continue;
stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket]; blk_rq_stat_add(stat, value); } put_cpu(); rcu_read_unlock();}
static void blk_stat_timer_fn(struct timer_list *t){ struct blk_stat_callback *cb = timer_container_of(cb, t, timer); unsigned int bucket; int cpu;
for (bucket = 0; bucket < cb->buckets; bucket++) blk_rq_stat_init(&cb->stat[bucket]);
for_each_online_cpu(cpu) { struct blk_rq_stat *cpu_stat;
cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); for (bucket = 0; bucket < cb->buckets; bucket++) { blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]); blk_rq_stat_init(&cpu_stat[bucket]); } }
cb->timer_fn(cb);}
struct blk_stat_callback *blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *), int (*bucket_fn)(const struct request *), unsigned int buckets, void *data){ struct blk_stat_callback *cb;
cb = kmalloc_obj(*cb); if (!cb) return NULL;
cb->stat = kmalloc_objs(struct blk_rq_stat, buckets); if (!cb->stat) { kfree(cb); return NULL; } cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat), __alignof__(struct blk_rq_stat)); if (!cb->cpu_stat) { kfree(cb->stat); kfree(cb); return NULL; }
cb->timer_fn = timer_fn; cb->bucket_fn = bucket_fn; cb->data = data; cb->buckets = buckets; timer_setup(&cb->timer, blk_stat_timer_fn, 0);
return cb;}
void blk_stat_add_callback(struct request_queue *q, struct blk_stat_callback *cb){ unsigned int bucket; unsigned long flags; int cpu;
for_each_possible_cpu(cpu) { struct blk_rq_stat *cpu_stat;
cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); for (bucket = 0; bucket < cb->buckets; bucket++) blk_rq_stat_init(&cpu_stat[bucket]); }
spin_lock_irqsave(&q->stats->lock, flags); list_add_tail_rcu(&cb->list, &q->stats->callbacks); blk_queue_flag_set(QUEUE_FLAG_STATS, q); spin_unlock_irqrestore(&q->stats->lock, flags);}
void blk_stat_remove_callback(struct request_queue *q, struct blk_stat_callback *cb){ unsigned long flags;
spin_lock_irqsave(&q->stats->lock, flags); list_del_rcu(&cb->list); if (list_empty(&q->stats->callbacks) && !q->stats->accounting) blk_queue_flag_clear(QUEUE_FLAG_STATS, q); spin_unlock_irqrestore(&q->stats->lock, flags);
timer_delete_sync(&cb->timer);}
static void blk_stat_free_callback_rcu(struct rcu_head *head){ struct blk_stat_callback *cb;
cb = container_of(head, struct blk_stat_callback, rcu); free_percpu(cb->cpu_stat); kfree(cb->stat); kfree(cb);}
void blk_stat_free_callback(struct blk_stat_callback *cb){ if (cb) call_rcu(&cb->rcu, blk_stat_free_callback_rcu);}
void blk_stat_disable_accounting(struct request_queue *q){ unsigned long flags;
spin_lock_irqsave(&q->stats->lock, flags); if (!--q->stats->accounting && list_empty(&q->stats->callbacks)) blk_queue_flag_clear(QUEUE_FLAG_STATS, q); spin_unlock_irqrestore(&q->stats->lock, flags);}EXPORT_SYMBOL_GPL(blk_stat_disable_accounting);
void blk_stat_enable_accounting(struct request_queue *q){ unsigned long flags;
spin_lock_irqsave(&q->stats->lock, flags); if (!q->stats->accounting++ && list_empty(&q->stats->callbacks)) blk_queue_flag_set(QUEUE_FLAG_STATS, q); spin_unlock_irqrestore(&q->stats->lock, flags);}EXPORT_SYMBOL_GPL(blk_stat_enable_accounting);
struct blk_queue_stats *blk_alloc_queue_stats(void){ struct blk_queue_stats *stats;
stats = kmalloc_obj(*stats); if (!stats) return NULL;
INIT_LIST_HEAD(&stats->callbacks); spin_lock_init(&stats->lock); stats->accounting = 0;
return stats;}
void blk_free_queue_stats(struct blk_queue_stats *stats){ if (!stats) return;
WARN_ON(!list_empty(&stats->callbacks));
kfree(stats);}