diff options
Diffstat (limited to 'block/blk-cgroup.c')
| -rw-r--r-- | block/blk-cgroup.c | 150 |
1 files changed, 85 insertions, 65 deletions
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 0eb39981e7c..91f9824be5c 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -78,14 +78,14 @@ struct blkio_cgroup *task_blkio_cgroup(struct task_struct *tsk) } EXPORT_SYMBOL_GPL(task_blkio_cgroup); -static inline void -blkio_update_group_weight(struct blkio_group *blkg, unsigned int weight) +static inline void blkio_update_group_weight(struct blkio_group *blkg, + int plid, unsigned int weight) { struct blkio_policy_type *blkiop; list_for_each_entry(blkiop, &blkio_list, list) { /* If this policy does not own the blkg, do not send updates */ - if (blkiop->plid != blkg->plid) + if (blkiop->plid != plid) continue; if (blkiop->ops.blkio_update_group_weight_fn) blkiop->ops.blkio_update_group_weight_fn(blkg->q, @@ -93,15 +93,15 @@ blkio_update_group_weight(struct blkio_group *blkg, unsigned int weight) } } -static inline void blkio_update_group_bps(struct blkio_group *blkg, u64 bps, - int fileid) +static inline void blkio_update_group_bps(struct blkio_group *blkg, int plid, + u64 bps, int fileid) { struct blkio_policy_type *blkiop; list_for_each_entry(blkiop, &blkio_list, list) { /* If this policy does not own the blkg, do not send updates */ - if (blkiop->plid != blkg->plid) + if (blkiop->plid != plid) continue; if (fileid == BLKIO_THROTL_read_bps_device @@ -117,14 +117,15 @@ static inline void blkio_update_group_bps(struct blkio_group *blkg, u64 bps, } static inline void blkio_update_group_iops(struct blkio_group *blkg, - unsigned int iops, int fileid) + int plid, unsigned int iops, + int fileid) { struct blkio_policy_type *blkiop; list_for_each_entry(blkiop, &blkio_list, list) { /* If this policy does not own the blkg, do not send updates */ - if (blkiop->plid != blkg->plid) + if (blkiop->plid != plid) continue; if (fileid == BLKIO_THROTL_read_iops_device @@ -182,9 +183,10 @@ static void blkio_check_and_dec_stat(uint64_t *stat, bool direction, bool sync) #ifdef CONFIG_DEBUG_BLK_CGROUP /* This should be called with the blkg->stats_lock held. */ static void blkio_set_start_group_wait_time(struct blkio_group *blkg, - struct blkio_group *curr_blkg) + struct blkio_policy_type *pol, + struct blkio_group *curr_blkg) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[pol->plid]; if (blkio_blkg_waiting(&pd->stats)) return; @@ -222,9 +224,10 @@ static void blkio_end_empty_time(struct blkio_group_stats *stats) blkio_clear_blkg_empty(stats); } -void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg) +void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg, + struct blkio_policy_type *pol) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[pol->plid]; unsigned long flags; spin_lock_irqsave(&blkg->stats_lock, flags); @@ -235,9 +238,10 @@ void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg) } EXPORT_SYMBOL_GPL(blkiocg_update_set_idle_time_stats); -void blkiocg_update_idle_time_stats(struct blkio_group *blkg) +void blkiocg_update_idle_time_stats(struct blkio_group *blkg, + struct blkio_policy_type *pol) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[pol->plid]; unsigned long flags; unsigned long long now; struct blkio_group_stats *stats; @@ -254,9 +258,10 @@ void blkiocg_update_idle_time_stats(struct blkio_group *blkg) } EXPORT_SYMBOL_GPL(blkiocg_update_idle_time_stats); -void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg) +void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg, + struct blkio_policy_type *pol) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[pol->plid]; unsigned long flags; struct blkio_group_stats *stats; @@ -271,9 +276,10 @@ void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg) } EXPORT_SYMBOL_GPL(blkiocg_update_avg_queue_size_stats); -void blkiocg_set_start_empty_time(struct blkio_group *blkg) +void blkiocg_set_start_empty_time(struct blkio_group *blkg, + struct blkio_policy_type *pol) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[pol->plid]; unsigned long flags; struct blkio_group_stats *stats; @@ -303,39 +309,43 @@ void blkiocg_set_start_empty_time(struct blkio_group *blkg) EXPORT_SYMBOL_GPL(blkiocg_set_start_empty_time); void blkiocg_update_dequeue_stats(struct blkio_group *blkg, - unsigned long dequeue) + struct blkio_policy_type *pol, + unsigned long dequeue) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[pol->plid]; pd->stats.dequeue += dequeue; } EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats); #else static inline void blkio_set_start_group_wait_time(struct blkio_group *blkg, - struct blkio_group *curr_blkg) {} -static inline void blkio_end_empty_time(struct blkio_group_stats *stats) {} + struct blkio_policy_type *pol, + struct blkio_group *curr_blkg) { } +static inline void blkio_end_empty_time(struct blkio_group_stats *stats) { } #endif void blkiocg_update_io_add_stats(struct blkio_group *blkg, - struct blkio_group *curr_blkg, bool direction, - bool sync) + struct blkio_policy_type *pol, + struct blkio_group *curr_blkg, bool direction, + bool sync) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[pol->plid]; unsigned long flags; spin_lock_irqsave(&blkg->stats_lock, flags); blkio_add_stat(pd->stats.stat_arr[BLKIO_STAT_QUEUED], 1, direction, sync); blkio_end_empty_time(&pd->stats); - blkio_set_start_group_wait_time(blkg, curr_blkg); + blkio_set_start_group_wait_time(blkg, pol, curr_blkg); spin_unlock_irqrestore(&blkg->stats_lock, flags); } EXPORT_SYMBOL_GPL(blkiocg_update_io_add_stats); void blkiocg_update_io_remove_stats(struct blkio_group *blkg, - bool direction, bool sync) + struct blkio_policy_type *pol, + bool direction, bool sync) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[pol->plid]; unsigned long flags; spin_lock_irqsave(&blkg->stats_lock, flags); @@ -345,10 +355,12 @@ void blkiocg_update_io_remove_stats(struct blkio_group *blkg, } EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats); -void blkiocg_update_timeslice_used(struct blkio_group *blkg, unsigned long time, - unsigned long unaccounted_time) +void blkiocg_update_timeslice_used(struct blkio_group *blkg, + struct blkio_policy_type *pol, + unsigned long time, + unsigned long unaccounted_time) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[pol->plid]; unsigned long flags; spin_lock_irqsave(&blkg->stats_lock, flags); @@ -365,9 +377,10 @@ EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used); * is valid. */ void blkiocg_update_dispatch_stats(struct blkio_group *blkg, - uint64_t bytes, bool direction, bool sync) + struct blkio_policy_type *pol, + uint64_t bytes, bool direction, bool sync) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[pol->plid]; struct blkio_group_stats_cpu *stats_cpu; unsigned long flags; @@ -392,9 +405,12 @@ void blkiocg_update_dispatch_stats(struct blkio_group *blkg, EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats); void blkiocg_update_completion_stats(struct blkio_group *blkg, - uint64_t start_time, uint64_t io_start_time, bool direction, bool sync) + struct blkio_policy_type *pol, + uint64_t start_time, + uint64_t io_start_time, bool direction, + bool sync) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[pol->plid]; struct blkio_group_stats *stats; unsigned long flags; unsigned long long now = sched_clock(); @@ -412,10 +428,11 @@ void blkiocg_update_completion_stats(struct blkio_group *blkg, EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats); /* Merged stats are per cpu. */ -void blkiocg_update_io_merged_stats(struct blkio_group *blkg, bool direction, - bool sync) +void blkiocg_update_io_merged_stats(struct blkio_group *blkg, + struct blkio_policy_type *pol, + bool direction, bool sync) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[pol->plid]; struct blkio_group_stats_cpu *stats_cpu; unsigned long flags; @@ -681,9 +698,9 @@ void __blkg_release(struct blkio_group *blkg) } EXPORT_SYMBOL_GPL(__blkg_release); -static void blkio_reset_stats_cpu(struct blkio_group *blkg) +static void blkio_reset_stats_cpu(struct blkio_group *blkg, int plid) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[plid]; struct blkio_group_stats_cpu *stats_cpu; int i, j, k; /* @@ -754,7 +771,7 @@ blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val) spin_unlock(&blkg->stats_lock); /* Reset Per cpu stats which don't take blkg->stats_lock */ - blkio_reset_stats_cpu(blkg); + blkio_reset_stats_cpu(blkg, blkg->plid); } spin_unlock_irq(&blkcg->lock); @@ -803,10 +820,10 @@ static uint64_t blkio_fill_stat(char *str, int chars_left, uint64_t val, } -static uint64_t blkio_read_stat_cpu(struct blkio_group *blkg, +static uint64_t blkio_read_stat_cpu(struct blkio_group *blkg, int plid, enum stat_type_cpu type, enum stat_sub_type sub_type) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[plid]; int cpu; struct blkio_group_stats_cpu *stats_cpu; u64 val = 0, tval; @@ -829,7 +846,7 @@ static uint64_t blkio_read_stat_cpu(struct blkio_group *blkg, return val; } -static uint64_t blkio_get_stat_cpu(struct blkio_group *blkg, +static uint64_t blkio_get_stat_cpu(struct blkio_group *blkg, int plid, struct cgroup_map_cb *cb, const char *dname, enum stat_type_cpu type) { @@ -838,7 +855,7 @@ static uint64_t blkio_get_stat_cpu(struct blkio_group *blkg, enum stat_sub_type sub_type; if (type == BLKIO_STAT_CPU_SECTORS) { - val = blkio_read_stat_cpu(blkg, type, 0); + val = blkio_read_stat_cpu(blkg, plid, type, 0); return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, val, cb, dname); } @@ -847,12 +864,12 @@ static uint64_t blkio_get_stat_cpu(struct blkio_group *blkg, sub_type++) { blkio_get_key_name(sub_type, dname, key_str, MAX_KEY_LEN, false); - val = blkio_read_stat_cpu(blkg, type, sub_type); + val = blkio_read_stat_cpu(blkg, plid, type, sub_type); cb->fill(cb, key_str, val); } - disk_total = blkio_read_stat_cpu(blkg, type, BLKIO_STAT_READ) + - blkio_read_stat_cpu(blkg, type, BLKIO_STAT_WRITE); + disk_total = blkio_read_stat_cpu(blkg, plid, type, BLKIO_STAT_READ) + + blkio_read_stat_cpu(blkg, plid, type, BLKIO_STAT_WRITE); blkio_get_key_name(BLKIO_STAT_TOTAL, dname, key_str, MAX_KEY_LEN, false); @@ -861,11 +878,11 @@ static uint64_t blkio_get_stat_cpu(struct blkio_group *blkg, } /* This should be called with blkg->stats_lock held */ -static uint64_t blkio_get_stat(struct blkio_group *blkg, +static uint64_t blkio_get_stat(struct blkio_group *blkg, int plid, struct cgroup_map_cb *cb, const char *dname, enum stat_type type) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; + struct blkg_policy_data *pd = blkg->pd[plid]; uint64_t disk_total; char key_str[MAX_KEY_LEN]; enum stat_sub_type sub_type; @@ -989,29 +1006,29 @@ static int blkio_policy_parse_and_set(char *buf, enum blkio_policy_id plid, goto out_unlock; pd->conf.weight = temp; - blkio_update_group_weight(blkg, temp ?: blkcg->weight); + blkio_update_group_weight(blkg, plid, temp ?: blkcg->weight); break; case BLKIO_POLICY_THROTL: switch(fileid) { case BLKIO_THROTL_read_bps_device: pd->conf.bps[READ] = temp; - blkio_update_group_bps(blkg, temp ?: -1, fileid); + blkio_update_group_bps(blkg, plid, temp ?: -1, fileid); break; case BLKIO_THROTL_write_bps_device: pd->conf.bps[WRITE] = temp; - blkio_update_group_bps(blkg, temp ?: -1, fileid); + blkio_update_group_bps(blkg, plid, temp ?: -1, fileid); break; case BLKIO_THROTL_read_iops_device: if (temp > THROTL_IOPS_MAX) goto out_unlock; pd->conf.iops[READ] = temp; - blkio_update_group_iops(blkg, temp ?: -1, fileid); + blkio_update_group_iops(blkg, plid, temp ?: -1, fileid); break; case BLKIO_THROTL_write_iops_device: if (temp > THROTL_IOPS_MAX) goto out_unlock; pd->conf.iops[WRITE] = temp; - blkio_update_group_iops(blkg, temp ?: -1, fileid); + blkio_update_group_iops(blkg, plid, temp ?: -1, fileid); break; } break; @@ -1066,15 +1083,16 @@ static const char *blkg_dev_name(struct blkio_group *blkg) static void blkio_print_group_conf(struct cftype *cft, struct blkio_group *blkg, struct seq_file *m) { - struct blkg_policy_data *pd = blkg->pd[blkg->plid]; - const char *dname = blkg_dev_name(blkg); + int plid = BLKIOFILE_POLICY(cft->private); int fileid = BLKIOFILE_ATTR(cft->private); + struct blkg_policy_data *pd = blkg->pd[plid]; + const char *dname = blkg_dev_name(blkg); int rw = WRITE; if (!dname) return; - switch (blkg->plid) { + switch (plid) { case BLKIO_POLICY_PROP: if (pd->conf.weight) seq_printf(m, "%s\t%u\n", @@ -1166,15 +1184,17 @@ static int blkio_read_blkg_stats(struct blkio_cgroup *blkcg, rcu_read_lock(); hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node) { const char *dname = blkg_dev_name(blkg); + int plid = BLKIOFILE_POLICY(cft->private); - if (!dname || BLKIOFILE_POLICY(cft->private) != blkg->plid) + if (!dname || plid != blkg->plid) continue; - if (pcpu) - cgroup_total += blkio_get_stat_cpu(blkg, cb, dname, - type); - else { + if (pcpu) { + cgroup_total += blkio_get_stat_cpu(blkg, plid, + cb, dname, type); + } else { spin_lock_irq(&blkg->stats_lock); - cgroup_total += blkio_get_stat(blkg, cb, dname, type); + cgroup_total += blkio_get_stat(blkg, plid, + cb, dname, type); spin_unlock_irq(&blkg->stats_lock); } } @@ -1280,7 +1300,7 @@ static int blkio_weight_write(struct blkio_cgroup *blkcg, int plid, u64 val) struct blkg_policy_data *pd = blkg->pd[blkg->plid]; if (blkg->plid == plid && !pd->conf.weight) - blkio_update_group_weight(blkg, blkcg->weight); + blkio_update_group_weight(blkg, plid, blkcg->weight); } spin_unlock_irq(&blkcg->lock); |