mirror of
https://github.com/a52xq-dev/kernel.git
synced 2026-10-18 21:58:11 +07:00
for-4.19/block-20180812
-----BEGIN PGP SIGNATURE-----
iQJEBAABCAAuFiEEwPw5LcreJtl1+l5K99NY+ylx4KYFAltwvasQHGF4Ym9lQGtl
cm5lbC5kawAKCRD301j7KXHgpv65EACTq5gSLnJBI6ZPr1RAHruVDnjfzO2Veitl
tUtjm0XfWmnEiwQ3dYvnyhy99xbyaG3900d9BClCTlH6xaUdSiQkDpcKG/R2F36J
5mZitYukQcpFAQJWF8YKsTTE7JPl4VglCIDqYiC4+C3rOSVi8lrKn2qp4J4MMCFn
thRg3jCcq7c5s9Eigsop1pXWQSasubkXfk55Krcp4oybKYpYRKXXf74Mj14QAbwJ
QHN3VisyAUWoBRg7UQZo1Npe2oPk6bbnJypnjf8M0M2EnlvddEkIlHob91sodka8
6p4APOEu5cbyXOBCAQsw/koff14mb8aEadqeQA68WvXfIdX9ZjfxCX0OoC3sBEXk
yqJhZ0C980AM13zIBD8ejv4uasGcPca8W+47mE5P8sRiI++5kBsFWDZPCtUBna0X
2Kh24NsmEya9XRR5vsB84dsIPQ3tLMkxg/IgQRVDaSnfJz0c/+zm54xDyKRaFT4l
5iERk2WSkm9+8jNfVmWG0edrv6nRAXjpGwFfOCPh6/LCSCi4xQRULYN7sVzsX8ZK
FRjt24HftBI8mJbh4BtweJvg+ppVe1gAk3IO3HvxAQhv29Hz+uvFYe9kL+3N8LJA
Qosr9n9O4+wKYizJcDnw+5iPqCHfAwOm9th4pyedR+R7SmNcP3yNC8AbbheNBiF5
Zolos5H+JA==
=b9ib
-----END PGP SIGNATURE-----
Merge tag 'for-4.19/block-20180812' of git://git.kernel.dk/linux-block
Pull block updates from Jens Axboe:
"First pull request for this merge window, there will also be a
followup request with some stragglers.
This pull request contains:
- Fix for a thundering heard issue in the wbt block code (Anchal
Agarwal)
- A few NVMe pull requests:
* Improved tracepoints (Keith)
* Larger inline data support for RDMA (Steve Wise)
* RDMA setup/teardown fixes (Sagi)
* Effects log suppor for NVMe target (Chaitanya Kulkarni)
* Buffered IO suppor for NVMe target (Chaitanya Kulkarni)
* TP4004 (ANA) support (Christoph)
* Various NVMe fixes
- Block io-latency controller support. Much needed support for
properly containing block devices. (Josef)
- Series improving how we handle sense information on the stack
(Kees)
- Lightnvm fixes and updates/improvements (Mathias/Javier et al)
- Zoned device support for null_blk (Matias)
- AIX partition fixes (Mauricio Faria de Oliveira)
- DIF checksum code made generic (Max Gurtovoy)
- Add support for discard in iostats (Michael Callahan / Tejun)
- Set of updates for BFQ (Paolo)
- Removal of async write support for bsg (Christoph)
- Bio page dirtying and clone fixups (Christoph)
- Set of bcache fix/changes (via Coly)
- Series improving blk-mq queue setup/teardown speed (Ming)
- Series improving merging performance on blk-mq (Ming)
- Lots of other fixes and cleanups from a slew of folks"
* tag 'for-4.19/block-20180812' of git://git.kernel.dk/linux-block: (190 commits)
blkcg: Make blkg_root_lookup() work for queues in bypass mode
bcache: fix error setting writeback_rate through sysfs interface
null_blk: add lock drop/acquire annotation
Blk-throttle: reduce tail io latency when iops limit is enforced
block: paride: pd: mark expected switch fall-throughs
block: Ensure that a request queue is dissociated from the cgroup controller
block: Introduce blk_exit_queue()
blkcg: Introduce blkg_root_lookup()
block: Remove two superfluous #include directives
blk-mq: count the hctx as active before allocating tag
block: bvec_nr_vecs() returns value for wrong slab
bcache: trivial - remove tailing backslash in macro BTREE_FLAG
bcache: make the pr_err statement used for ENOENT only in sysfs_attatch section
bcache: set max writeback rate when I/O request is idle
bcache: add code comments for bset.c
bcache: fix mistaken comments in request.c
bcache: fix mistaken code comments in bcache.h
bcache: add a comment in super.c
bcache: avoid unncessary cache prefetch bch_btree_node_get()
bcache: display rate debug parameters to 0 when writeback is not running
...
This commit is contained in:
Linus Torvalds
commit
73ba2fb33c
172 changed files with 6029 additions and 2659 deletions
|
|
@ -5,6 +5,7 @@ Description:
|
|||
The /proc/diskstats file displays the I/O statistics
|
||||
of block devices. Each line contains the following 14
|
||||
fields:
|
||||
|
||||
1 - major number
|
||||
2 - minor mumber
|
||||
3 - device name
|
||||
|
|
@ -19,4 +20,13 @@ Description:
|
|||
12 - I/Os currently in progress
|
||||
13 - time spent doing I/Os (ms)
|
||||
14 - weighted time spent doing I/Os (ms)
|
||||
|
||||
Kernel 4.18+ appends four more fields for discard
|
||||
tracking putting the total at 18:
|
||||
|
||||
15 - discards completed successfully
|
||||
16 - discards merged
|
||||
17 - sectors discarded
|
||||
18 - time spent discarding
|
||||
|
||||
For more details refer to Documentation/iostats.txt
|
||||
|
|
|
|||
|
|
@ -51,6 +51,9 @@ v1 is available under Documentation/cgroup-v1/.
|
|||
5-3. IO
|
||||
5-3-1. IO Interface Files
|
||||
5-3-2. Writeback
|
||||
5-3-3. IO Latency
|
||||
5-3-3-1. How IO Latency Throttling Works
|
||||
5-3-3-2. IO Latency Interface Files
|
||||
5-4. PID
|
||||
5-4-1. PID Interface Files
|
||||
5-5. Device
|
||||
|
|
@ -1314,17 +1317,19 @@ IO Interface Files
|
|||
Lines are keyed by $MAJ:$MIN device numbers and not ordered.
|
||||
The following nested keys are defined.
|
||||
|
||||
====== ===================
|
||||
====== =====================
|
||||
rbytes Bytes read
|
||||
wbytes Bytes written
|
||||
rios Number of read IOs
|
||||
wios Number of write IOs
|
||||
====== ===================
|
||||
dbytes Bytes discarded
|
||||
dios Number of discard IOs
|
||||
====== =====================
|
||||
|
||||
An example read output follows:
|
||||
|
||||
8:16 rbytes=1459200 wbytes=314773504 rios=192 wios=353
|
||||
8:0 rbytes=90430464 wbytes=299008000 rios=8950 wios=1252
|
||||
8:16 rbytes=1459200 wbytes=314773504 rios=192 wios=353 dbytes=0 dios=0
|
||||
8:0 rbytes=90430464 wbytes=299008000 rios=8950 wios=1252 dbytes=50331648 dios=3021
|
||||
|
||||
io.weight
|
||||
A read-write flat-keyed file which exists on non-root cgroups.
|
||||
|
|
@ -1446,6 +1451,85 @@ writeback as follows.
|
|||
vm.dirty[_background]_ratio.
|
||||
|
||||
|
||||
IO Latency
|
||||
~~~~~~~~~~
|
||||
|
||||
This is a cgroup v2 controller for IO workload protection. You provide a group
|
||||
with a latency target, and if the average latency exceeds that target the
|
||||
controller will throttle any peers that have a lower latency target than the
|
||||
protected workload.
|
||||
|
||||
The limits are only applied at the peer level in the hierarchy. This means that
|
||||
in the diagram below, only groups A, B, and C will influence each other, and
|
||||
groups D and F will influence each other. Group G will influence nobody.
|
||||
|
||||
[root]
|
||||
/ | \
|
||||
A B C
|
||||
/ \ |
|
||||
D F G
|
||||
|
||||
|
||||
So the ideal way to configure this is to set io.latency in groups A, B, and C.
|
||||
Generally you do not want to set a value lower than the latency your device
|
||||
supports. Experiment to find the value that works best for your workload.
|
||||
Start at higher than the expected latency for your device and watch the
|
||||
avg_lat value in io.stat for your workload group to get an idea of the
|
||||
latency you see during normal operation. Use the avg_lat value as a basis for
|
||||
your real setting, setting at 10-15% higher than the value in io.stat.
|
||||
|
||||
How IO Latency Throttling Works
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
io.latency is work conserving; so as long as everybody is meeting their latency
|
||||
target the controller doesn't do anything. Once a group starts missing its
|
||||
target it begins throttling any peer group that has a higher target than itself.
|
||||
This throttling takes 2 forms:
|
||||
|
||||
- Queue depth throttling. This is the number of outstanding IO's a group is
|
||||
allowed to have. We will clamp down relatively quickly, starting at no limit
|
||||
and going all the way down to 1 IO at a time.
|
||||
|
||||
- Artificial delay induction. There are certain types of IO that cannot be
|
||||
throttled without possibly adversely affecting higher priority groups. This
|
||||
includes swapping and metadata IO. These types of IO are allowed to occur
|
||||
normally, however they are "charged" to the originating group. If the
|
||||
originating group is being throttled you will see the use_delay and delay
|
||||
fields in io.stat increase. The delay value is how many microseconds that are
|
||||
being added to any process that runs in this group. Because this number can
|
||||
grow quite large if there is a lot of swapping or metadata IO occurring we
|
||||
limit the individual delay events to 1 second at a time.
|
||||
|
||||
Once the victimized group starts meeting its latency target again it will start
|
||||
unthrottling any peer groups that were throttled previously. If the victimized
|
||||
group simply stops doing IO the global counter will unthrottle appropriately.
|
||||
|
||||
IO Latency Interface Files
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
io.latency
|
||||
This takes a similar format as the other controllers.
|
||||
|
||||
"MAJOR:MINOR target=<target time in microseconds"
|
||||
|
||||
io.stat
|
||||
If the controller is enabled you will see extra stats in io.stat in
|
||||
addition to the normal ones.
|
||||
|
||||
depth
|
||||
This is the current queue depth for the group.
|
||||
|
||||
avg_lat
|
||||
This is an exponential moving average with a decay rate of 1/exp
|
||||
bound by the sampling interval. The decay rate interval can be
|
||||
calculated by multiplying the win value in io.stat by the
|
||||
corresponding number of samples based on the win value.
|
||||
|
||||
win
|
||||
The sampling window size in milliseconds. This is the minimum
|
||||
duration of time between evaluation events. Windows only elapse
|
||||
with IO activity. Idle periods extend the most recent window.
|
||||
|
||||
PID
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -85,3 +85,10 @@ shared_tags=[0/1]: Default: 0
|
|||
0: Tag set is not shared.
|
||||
1: Tag set shared between devices for blk-mq. Only makes sense with
|
||||
nr_devices > 1, otherwise there's no tag set to share.
|
||||
|
||||
zoned=[0/1]: Default: 0
|
||||
0: Block device is exposed as a random-access block device.
|
||||
1: Block device is exposed as a host-managed zoned block device.
|
||||
|
||||
zone_size=[MB]: Default: 256
|
||||
Per zone size when exposed as a zoned block device. Must be a power of two.
|
||||
|
|
|
|||
|
|
@ -31,28 +31,32 @@ write ticks milliseconds total wait time for write requests
|
|||
in_flight requests number of I/Os currently in flight
|
||||
io_ticks milliseconds total time this block device has been active
|
||||
time_in_queue milliseconds total wait time for all requests
|
||||
discard I/Os requests number of discard I/Os processed
|
||||
discard merges requests number of discard I/Os merged with in-queue I/O
|
||||
discard sectors sectors number of sectors discarded
|
||||
discard ticks milliseconds total wait time for discard requests
|
||||
|
||||
read I/Os, write I/Os
|
||||
=====================
|
||||
read I/Os, write I/Os, discard I/0s
|
||||
===================================
|
||||
|
||||
These values increment when an I/O request completes.
|
||||
|
||||
read merges, write merges
|
||||
=========================
|
||||
read merges, write merges, discard merges
|
||||
=========================================
|
||||
|
||||
These values increment when an I/O request is merged with an
|
||||
already-queued I/O request.
|
||||
|
||||
read sectors, write sectors
|
||||
===========================
|
||||
read sectors, write sectors, discard_sectors
|
||||
============================================
|
||||
|
||||
These values count the number of sectors read from or written to this
|
||||
block device. The "sectors" in question are the standard UNIX 512-byte
|
||||
sectors, not any device- or filesystem-specific block size. The
|
||||
counters are incremented when the I/O completes.
|
||||
These values count the number of sectors read from, written to, or
|
||||
discarded from this block device. The "sectors" in question are the
|
||||
standard UNIX 512-byte sectors, not any device- or filesystem-specific
|
||||
block size. The counters are incremented when the I/O completes.
|
||||
|
||||
read ticks, write ticks
|
||||
=======================
|
||||
read ticks, write ticks, discard ticks
|
||||
======================================
|
||||
|
||||
These values count the number of milliseconds that I/O requests have
|
||||
waited on this block device. If there are multiple I/O requests waiting,
|
||||
|
|
|
|||
|
|
@ -31,6 +31,9 @@ Here are examples of these different formats::
|
|||
3 0 hda 446216 784926 9550688 4382310 424847 312726 5922052 19310380 0 3376340 23705160
|
||||
3 1 hda1 35486 38030 38030 38030
|
||||
|
||||
4.18+ diskstats:
|
||||
3 0 hda 446216 784926 9550688 4382310 424847 312726 5922052 19310380 0 3376340 23705160 0 0 0 0
|
||||
|
||||
On 2.4 you might execute ``grep 'hda ' /proc/partitions``. On 2.6+, you have
|
||||
a choice of ``cat /sys/block/hda/stat`` or ``grep 'hda ' /proc/diskstats``.
|
||||
|
||||
|
|
@ -101,6 +104,18 @@ Field 11 -- weighted # of milliseconds spent doing I/Os
|
|||
last update of this field. This can provide an easy measure of both
|
||||
I/O completion time and the backlog that may be accumulating.
|
||||
|
||||
Field 12 -- # of discards completed
|
||||
This is the total number of discards completed successfully.
|
||||
|
||||
Field 13 -- # of discards merged
|
||||
See the description of field 2
|
||||
|
||||
Field 14 -- # of sectors discarded
|
||||
This is the total number of sectors discarded successfully.
|
||||
|
||||
Field 15 -- # of milliseconds spent discarding
|
||||
This is the total number of milliseconds spent by all discards (as
|
||||
measured from __make_request() to end_that_request_last()).
|
||||
|
||||
To avoid introducing performance bottlenecks, no locks are held while
|
||||
modifying these counters. This implies that minor inaccuracies may be
|
||||
|
|
|
|||
|
|
@ -149,6 +149,18 @@ config BLK_WBT
|
|||
dynamically on an algorithm loosely based on CoDel, factoring in
|
||||
the realtime performance of the disk.
|
||||
|
||||
config BLK_CGROUP_IOLATENCY
|
||||
bool "Enable support for latency based cgroup IO protection"
|
||||
depends on BLK_CGROUP=y
|
||||
default n
|
||||
---help---
|
||||
Enabling this option enables the .latency interface for IO throttling.
|
||||
The IO controller will attempt to maintain average IO latencies below
|
||||
the configured latency target, throttling anybody with a higher latency
|
||||
target than the victimized group.
|
||||
|
||||
Note, this is an experimental interface and could be changed someday.
|
||||
|
||||
config BLK_WBT_SQ
|
||||
bool "Single queue writeback throttling"
|
||||
default n
|
||||
|
|
@ -177,6 +189,10 @@ config BLK_DEBUG_FS
|
|||
Unless you are building a kernel for a tiny system, you should
|
||||
say Y here.
|
||||
|
||||
config BLK_DEBUG_FS_ZONED
|
||||
bool
|
||||
default BLK_DEBUG_FS && BLK_DEV_ZONED
|
||||
|
||||
config BLK_SED_OPAL
|
||||
bool "Logic for interfacing with Opal enabled SEDs"
|
||||
---help---
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \
|
|||
blk-lib.o blk-mq.o blk-mq-tag.o blk-stat.o \
|
||||
blk-mq-sysfs.o blk-mq-cpumap.o blk-mq-sched.o ioctl.o \
|
||||
genhd.o partition-generic.o ioprio.o \
|
||||
badblocks.o partitions/
|
||||
badblocks.o partitions/ blk-rq-qos.o
|
||||
|
||||
obj-$(CONFIG_BOUNCE) += bounce.o
|
||||
obj-$(CONFIG_BLK_SCSI_REQUEST) += scsi_ioctl.o
|
||||
|
|
@ -17,6 +17,7 @@ obj-$(CONFIG_BLK_DEV_BSG) += bsg.o
|
|||
obj-$(CONFIG_BLK_DEV_BSGLIB) += bsg-lib.o
|
||||
obj-$(CONFIG_BLK_CGROUP) += blk-cgroup.o
|
||||
obj-$(CONFIG_BLK_DEV_THROTTLING) += blk-throttle.o
|
||||
obj-$(CONFIG_BLK_CGROUP_IOLATENCY) += blk-iolatency.o
|
||||
obj-$(CONFIG_IOSCHED_NOOP) += noop-iosched.o
|
||||
obj-$(CONFIG_IOSCHED_DEADLINE) += deadline-iosched.o
|
||||
obj-$(CONFIG_IOSCHED_CFQ) += cfq-iosched.o
|
||||
|
|
@ -34,4 +35,5 @@ obj-$(CONFIG_BLK_MQ_RDMA) += blk-mq-rdma.o
|
|||
obj-$(CONFIG_BLK_DEV_ZONED) += blk-zoned.o
|
||||
obj-$(CONFIG_BLK_WBT) += blk-wbt.o
|
||||
obj-$(CONFIG_BLK_DEBUG_FS) += blk-mq-debugfs.o
|
||||
obj-$(CONFIG_BLK_DEBUG_FS_ZONED)+= blk-mq-debugfs-zoned.o
|
||||
obj-$(CONFIG_BLK_SED_OPAL) += sed-opal.o
|
||||
|
|
|
|||
|
|
@ -634,7 +634,7 @@ static bool bfq_differentiated_weights(struct bfq_data *bfqd)
|
|||
* The following function returns true if every queue must receive the
|
||||
* same share of the throughput (this condition is used when deciding
|
||||
* whether idling may be disabled, see the comments in the function
|
||||
* bfq_bfqq_may_idle()).
|
||||
* bfq_better_to_idle()).
|
||||
*
|
||||
* Such a scenario occurs when:
|
||||
* 1) all active queues have the same weight,
|
||||
|
|
@ -742,8 +742,9 @@ inc_counter:
|
|||
* See the comments to the function bfq_weights_tree_add() for considerations
|
||||
* about overhead.
|
||||
*/
|
||||
void bfq_weights_tree_remove(struct bfq_data *bfqd, struct bfq_entity *entity,
|
||||
struct rb_root *root)
|
||||
void __bfq_weights_tree_remove(struct bfq_data *bfqd,
|
||||
struct bfq_entity *entity,
|
||||
struct rb_root *root)
|
||||
{
|
||||
if (!entity->weight_counter)
|
||||
return;
|
||||
|
|
@ -759,6 +760,43 @@ reset_entity_pointer:
|
|||
entity->weight_counter = NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Invoke __bfq_weights_tree_remove on bfqq and all its inactive
|
||||
* parent entities.
|
||||
*/
|
||||
void bfq_weights_tree_remove(struct bfq_data *bfqd,
|
||||
struct bfq_queue *bfqq)
|
||||
{
|
||||
struct bfq_entity *entity = bfqq->entity.parent;
|
||||
|
||||
__bfq_weights_tree_remove(bfqd, &bfqq->entity,
|
||||
&bfqd->queue_weights_tree);
|
||||
|
||||
for_each_entity(entity) {
|
||||
struct bfq_sched_data *sd = entity->my_sched_data;
|
||||
|
||||
if (sd->next_in_service || sd->in_service_entity) {
|
||||
/*
|
||||
* entity is still active, because either
|
||||
* next_in_service or in_service_entity is not
|
||||
* NULL (see the comments on the definition of
|
||||
* next_in_service for details on why
|
||||
* in_service_entity must be checked too).
|
||||
*
|
||||
* As a consequence, the weight of entity is
|
||||
* not to be removed. In addition, if entity
|
||||
* is active, then its parent entities are
|
||||
* active as well, and thus their weights are
|
||||
* not to be removed either. In the end, this
|
||||
* loop must stop here.
|
||||
*/
|
||||
break;
|
||||
}
|
||||
__bfq_weights_tree_remove(bfqd, entity,
|
||||
&bfqd->group_weights_tree);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Return expired entry, or NULL to just start from scratch in rbtree.
|
||||
*/
|
||||
|
|
@ -1344,18 +1382,30 @@ static bool bfq_bfqq_update_budg_for_activation(struct bfq_data *bfqd,
|
|||
* remain unchanged after such an expiration, and the
|
||||
* following statement therefore assigns to
|
||||
* entity->budget the remaining budget on such an
|
||||
* expiration. For clarity, entity->service is not
|
||||
* updated on expiration in any case, and, in normal
|
||||
* operation, is reset only when bfqq is selected for
|
||||
* service (see bfq_get_next_queue).
|
||||
* expiration.
|
||||
*/
|
||||
entity->budget = min_t(unsigned long,
|
||||
bfq_bfqq_budget_left(bfqq),
|
||||
bfqq->max_budget);
|
||||
|
||||
/*
|
||||
* At this point, we have used entity->service to get
|
||||
* the budget left (needed for updating
|
||||
* entity->budget). Thus we finally can, and have to,
|
||||
* reset entity->service. The latter must be reset
|
||||
* because bfqq would otherwise be charged again for
|
||||
* the service it has received during its previous
|
||||
* service slot(s).
|
||||
*/
|
||||
entity->service = 0;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* We can finally complete expiration, by setting service to 0.
|
||||
*/
|
||||
entity->service = 0;
|
||||
entity->budget = max_t(unsigned long, bfqq->max_budget,
|
||||
bfq_serv_to_charge(bfqq->next_rq, bfqq));
|
||||
bfq_clear_bfqq_non_blocking_wait_rq(bfqq);
|
||||
|
|
@ -3233,11 +3283,21 @@ void bfq_bfqq_expire(struct bfq_data *bfqd,
|
|||
ref = bfqq->ref;
|
||||
__bfq_bfqq_expire(bfqd, bfqq);
|
||||
|
||||
if (ref == 1) /* bfqq is gone, no more actions on it */
|
||||
return;
|
||||
|
||||
/* mark bfqq as waiting a request only if a bic still points to it */
|
||||
if (ref > 1 && !bfq_bfqq_busy(bfqq) &&
|
||||
if (!bfq_bfqq_busy(bfqq) &&
|
||||
reason != BFQQE_BUDGET_TIMEOUT &&
|
||||
reason != BFQQE_BUDGET_EXHAUSTED)
|
||||
reason != BFQQE_BUDGET_EXHAUSTED) {
|
||||
bfq_mark_bfqq_non_blocking_wait_rq(bfqq);
|
||||
/*
|
||||
* Not setting service to 0, because, if the next rq
|
||||
* arrives in time, the queue will go on receiving
|
||||
* service with this same budget (as if it never expired)
|
||||
*/
|
||||
} else
|
||||
entity->service = 0;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -3295,7 +3355,7 @@ static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq)
|
|||
* issues taken into account are not trivial. We discuss these issues
|
||||
* individually while introducing the variables.
|
||||
*/
|
||||
static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
|
||||
static bool bfq_better_to_idle(struct bfq_queue *bfqq)
|
||||
{
|
||||
struct bfq_data *bfqd = bfqq->bfqd;
|
||||
bool rot_without_queueing =
|
||||
|
|
@ -3528,19 +3588,19 @@ static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq)
|
|||
}
|
||||
|
||||
/*
|
||||
* If the in-service queue is empty but the function bfq_bfqq_may_idle
|
||||
* If the in-service queue is empty but the function bfq_better_to_idle
|
||||
* returns true, then:
|
||||
* 1) the queue must remain in service and cannot be expired, and
|
||||
* 2) the device must be idled to wait for the possible arrival of a new
|
||||
* request for the queue.
|
||||
* See the comments on the function bfq_bfqq_may_idle for the reasons
|
||||
* See the comments on the function bfq_better_to_idle for the reasons
|
||||
* why performing device idling is the best choice to boost the throughput
|
||||
* and preserve service guarantees when bfq_bfqq_may_idle itself
|
||||
* and preserve service guarantees when bfq_better_to_idle itself
|
||||
* returns true.
|
||||
*/
|
||||
static bool bfq_bfqq_must_idle(struct bfq_queue *bfqq)
|
||||
{
|
||||
return RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_may_idle(bfqq);
|
||||
return RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_better_to_idle(bfqq);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -3559,8 +3619,14 @@ static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd)
|
|||
|
||||
bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue");
|
||||
|
||||
/*
|
||||
* Do not expire bfqq for budget timeout if bfqq may be about
|
||||
* to enjoy device idling. The reason why, in this case, we
|
||||
* prevent bfqq from expiring is the same as in the comments
|
||||
* on the case where bfq_bfqq_must_idle() returns true, in
|
||||
* bfq_completed_request().
|
||||
*/
|
||||
if (bfq_may_expire_for_budg_timeout(bfqq) &&
|
||||
!bfq_bfqq_wait_request(bfqq) &&
|
||||
!bfq_bfqq_must_idle(bfqq))
|
||||
goto expire;
|
||||
|
||||
|
|
@ -3620,7 +3686,7 @@ check_queue:
|
|||
* may idle after their completion, then keep it anyway.
|
||||
*/
|
||||
if (bfq_bfqq_wait_request(bfqq) ||
|
||||
(bfqq->dispatched != 0 && bfq_bfqq_may_idle(bfqq))) {
|
||||
(bfqq->dispatched != 0 && bfq_better_to_idle(bfqq))) {
|
||||
bfqq = NULL;
|
||||
goto keep_queue;
|
||||
}
|
||||
|
|
@ -4582,8 +4648,7 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd)
|
|||
*/
|
||||
bfqq->budget_timeout = jiffies;
|
||||
|
||||
bfq_weights_tree_remove(bfqd, &bfqq->entity,
|
||||
&bfqd->queue_weights_tree);
|
||||
bfq_weights_tree_remove(bfqd, bfqq);
|
||||
}
|
||||
|
||||
now_ns = ktime_get_ns();
|
||||
|
|
@ -4637,15 +4702,39 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd)
|
|||
* or if we want to idle in case it has no pending requests.
|
||||
*/
|
||||
if (bfqd->in_service_queue == bfqq) {
|
||||
if (bfqq->dispatched == 0 && bfq_bfqq_must_idle(bfqq)) {
|
||||
bfq_arm_slice_timer(bfqd);
|
||||
if (bfq_bfqq_must_idle(bfqq)) {
|
||||
if (bfqq->dispatched == 0)
|
||||
bfq_arm_slice_timer(bfqd);
|
||||
/*
|
||||
* If we get here, we do not expire bfqq, even
|
||||
* if bfqq was in budget timeout or had no
|
||||
* more requests (as controlled in the next
|
||||
* conditional instructions). The reason for
|
||||
* not expiring bfqq is as follows.
|
||||
*
|
||||
* Here bfqq->dispatched > 0 holds, but
|
||||
* bfq_bfqq_must_idle() returned true. This
|
||||
* implies that, even if no request arrives
|
||||
* for bfqq before bfqq->dispatched reaches 0,
|
||||
* bfqq will, however, not be expired on the
|
||||
* completion event that causes bfqq->dispatch
|
||||
* to reach zero. In contrast, on this event,
|
||||
* bfqq will start enjoying device idling
|
||||
* (I/O-dispatch plugging).
|
||||
*
|
||||
* But, if we expired bfqq here, bfqq would
|
||||
* not have the chance to enjoy device idling
|
||||
* when bfqq->dispatched finally reaches
|
||||
* zero. This would expose bfqq to violation
|
||||
* of its reserved service guarantees.
|
||||
*/
|
||||
return;
|
||||
} else if (bfq_may_expire_for_budg_timeout(bfqq))
|
||||
bfq_bfqq_expire(bfqd, bfqq, false,
|
||||
BFQQE_BUDGET_TIMEOUT);
|
||||
else if (RB_EMPTY_ROOT(&bfqq->sort_list) &&
|
||||
(bfqq->dispatched == 0 ||
|
||||
!bfq_bfqq_may_idle(bfqq)))
|
||||
!bfq_better_to_idle(bfqq)))
|
||||
bfq_bfqq_expire(bfqd, bfqq, false,
|
||||
BFQQE_NO_MORE_REQUESTS);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -827,8 +827,11 @@ struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic);
|
|||
void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq);
|
||||
void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_entity *entity,
|
||||
struct rb_root *root);
|
||||
void bfq_weights_tree_remove(struct bfq_data *bfqd, struct bfq_entity *entity,
|
||||
struct rb_root *root);
|
||||
void __bfq_weights_tree_remove(struct bfq_data *bfqd,
|
||||
struct bfq_entity *entity,
|
||||
struct rb_root *root);
|
||||
void bfq_weights_tree_remove(struct bfq_data *bfqd,
|
||||
struct bfq_queue *bfqq);
|
||||
void bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq,
|
||||
bool compensate, enum bfqq_expiration reason);
|
||||
void bfq_put_queue(struct bfq_queue *bfqq);
|
||||
|
|
|
|||
|
|
@ -499,9 +499,6 @@ static void bfq_active_insert(struct bfq_service_tree *st,
|
|||
if (bfqq)
|
||||
list_add(&bfqq->bfqq_list, &bfqq->bfqd->active_list);
|
||||
#ifdef CONFIG_BFQ_GROUP_IOSCHED
|
||||
else /* bfq_group */
|
||||
bfq_weights_tree_add(bfqd, entity, &bfqd->group_weights_tree);
|
||||
|
||||
if (bfqg != bfqd->root_group)
|
||||
bfqg->active_entities++;
|
||||
#endif
|
||||
|
|
@ -601,10 +598,6 @@ static void bfq_active_extract(struct bfq_service_tree *st,
|
|||
if (bfqq)
|
||||
list_del(&bfqq->bfqq_list);
|
||||
#ifdef CONFIG_BFQ_GROUP_IOSCHED
|
||||
else /* bfq_group */
|
||||
bfq_weights_tree_remove(bfqd, entity,
|
||||
&bfqd->group_weights_tree);
|
||||
|
||||
if (bfqg != bfqd->root_group)
|
||||
bfqg->active_entities--;
|
||||
#endif
|
||||
|
|
@ -799,7 +792,7 @@ __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st,
|
|||
if (prev_weight != new_weight) {
|
||||
root = bfqq ? &bfqd->queue_weights_tree :
|
||||
&bfqd->group_weights_tree;
|
||||
bfq_weights_tree_remove(bfqd, entity, root);
|
||||
__bfq_weights_tree_remove(bfqd, entity, root);
|
||||
}
|
||||
entity->weight = new_weight;
|
||||
/*
|
||||
|
|
@ -971,7 +964,7 @@ static void bfq_update_fin_time_enqueue(struct bfq_entity *entity,
|
|||
* one of its children receives a new request.
|
||||
*
|
||||
* Basically, this function updates the timestamps of entity and
|
||||
* inserts entity into its active tree, ater possibly extracting it
|
||||
* inserts entity into its active tree, after possibly extracting it
|
||||
* from its idle tree.
|
||||
*/
|
||||
static void __bfq_activate_entity(struct bfq_entity *entity,
|
||||
|
|
@ -1015,6 +1008,16 @@ static void __bfq_activate_entity(struct bfq_entity *entity,
|
|||
entity->on_st = true;
|
||||
}
|
||||
|
||||
#ifdef BFQ_GROUP_IOSCHED_ENABLED
|
||||
if (!bfq_entity_to_bfqq(entity)) { /* bfq_group */
|
||||
struct bfq_group *bfqg =
|
||||
container_of(entity, struct bfq_group, entity);
|
||||
|
||||
bfq_weights_tree_add(bfqg->bfqd, entity,
|
||||
&bfqd->group_weights_tree);
|
||||
}
|
||||
#endif
|
||||
|
||||
bfq_update_fin_time_enqueue(entity, st, backshifted);
|
||||
}
|
||||
|
||||
|
|
@ -1541,12 +1544,6 @@ struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd)
|
|||
entity = sd->next_in_service;
|
||||
sd->in_service_entity = entity;
|
||||
|
||||
/*
|
||||
* Reset the accumulator of the amount of service that
|
||||
* the entity is about to receive.
|
||||
*/
|
||||
entity->service = 0;
|
||||
|
||||
/*
|
||||
* If entity is no longer a candidate for next
|
||||
* service, then it must be extracted from its active
|
||||
|
|
@ -1664,8 +1661,7 @@ void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq,
|
|||
bfqd->busy_queues--;
|
||||
|
||||
if (!bfqq->dispatched)
|
||||
bfq_weights_tree_remove(bfqd, &bfqq->entity,
|
||||
&bfqd->queue_weights_tree);
|
||||
bfq_weights_tree_remove(bfqd, bfqq);
|
||||
|
||||
if (bfqq->wr_coeff > 1)
|
||||
bfqd->wr_busy_queues--;
|
||||
|
|
|
|||
|
|
@ -159,28 +159,6 @@ int bio_integrity_add_page(struct bio *bio, struct page *page,
|
|||
}
|
||||
EXPORT_SYMBOL(bio_integrity_add_page);
|
||||
|
||||
/**
|
||||
* bio_integrity_intervals - Return number of integrity intervals for a bio
|
||||
* @bi: blk_integrity profile for device
|
||||
* @sectors: Size of the bio in 512-byte sectors
|
||||
*
|
||||
* Description: The block layer calculates everything in 512 byte
|
||||
* sectors but integrity metadata is done in terms of the data integrity
|
||||
* interval size of the storage device. Convert the block layer sectors
|
||||
* to the appropriate number of integrity intervals.
|
||||
*/
|
||||
static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
|
||||
unsigned int sectors)
|
||||
{
|
||||
return sectors >> (bi->interval_exp - 9);
|
||||
}
|
||||
|
||||
static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi,
|
||||
unsigned int sectors)
|
||||
{
|
||||
return bio_integrity_intervals(bi, sectors) * bi->tuple_size;
|
||||
}
|
||||
|
||||
/**
|
||||
* bio_integrity_process - Process integrity metadata for a bio
|
||||
* @bio: bio to generate/verify integrity metadata for
|
||||
|
|
|
|||
208
block/bio.c
208
block/bio.c
|
|
@ -28,9 +28,11 @@
|
|||
#include <linux/mempool.h>
|
||||
#include <linux/workqueue.h>
|
||||
#include <linux/cgroup.h>
|
||||
#include <linux/blk-cgroup.h>
|
||||
|
||||
#include <trace/events/block.h>
|
||||
#include "blk.h"
|
||||
#include "blk-rq-qos.h"
|
||||
|
||||
/*
|
||||
* Test patch to inline a certain number of bi_io_vec's inside the bio
|
||||
|
|
@ -156,7 +158,7 @@ out:
|
|||
|
||||
unsigned int bvec_nr_vecs(unsigned short idx)
|
||||
{
|
||||
return bvec_slabs[idx].nr_vecs;
|
||||
return bvec_slabs[--idx].nr_vecs;
|
||||
}
|
||||
|
||||
void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
|
||||
|
|
@ -644,83 +646,6 @@ struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
|
|||
}
|
||||
EXPORT_SYMBOL(bio_clone_fast);
|
||||
|
||||
/**
|
||||
* bio_clone_bioset - clone a bio
|
||||
* @bio_src: bio to clone
|
||||
* @gfp_mask: allocation priority
|
||||
* @bs: bio_set to allocate from
|
||||
*
|
||||
* Clone bio. Caller will own the returned bio, but not the actual data it
|
||||
* points to. Reference count of returned bio will be one.
|
||||
*/
|
||||
struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask,
|
||||
struct bio_set *bs)
|
||||
{
|
||||
struct bvec_iter iter;
|
||||
struct bio_vec bv;
|
||||
struct bio *bio;
|
||||
|
||||
/*
|
||||
* Pre immutable biovecs, __bio_clone() used to just do a memcpy from
|
||||
* bio_src->bi_io_vec to bio->bi_io_vec.
|
||||
*
|
||||
* We can't do that anymore, because:
|
||||
*
|
||||
* - The point of cloning the biovec is to produce a bio with a biovec
|
||||
* the caller can modify: bi_idx and bi_bvec_done should be 0.
|
||||
*
|
||||
* - The original bio could've had more than BIO_MAX_PAGES biovecs; if
|
||||
* we tried to clone the whole thing bio_alloc_bioset() would fail.
|
||||
* But the clone should succeed as long as the number of biovecs we
|
||||
* actually need to allocate is fewer than BIO_MAX_PAGES.
|
||||
*
|
||||
* - Lastly, bi_vcnt should not be looked at or relied upon by code
|
||||
* that does not own the bio - reason being drivers don't use it for
|
||||
* iterating over the biovec anymore, so expecting it to be kept up
|
||||
* to date (i.e. for clones that share the parent biovec) is just
|
||||
* asking for trouble and would force extra work on
|
||||
* __bio_clone_fast() anyways.
|
||||
*/
|
||||
|
||||
bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
|
||||
if (!bio)
|
||||
return NULL;
|
||||
bio->bi_disk = bio_src->bi_disk;
|
||||
bio->bi_opf = bio_src->bi_opf;
|
||||
bio->bi_write_hint = bio_src->bi_write_hint;
|
||||
bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
|
||||
bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
|
||||
|
||||
switch (bio_op(bio)) {
|
||||
case REQ_OP_DISCARD:
|
||||
case REQ_OP_SECURE_ERASE:
|
||||
case REQ_OP_WRITE_ZEROES:
|
||||
break;
|
||||
case REQ_OP_WRITE_SAME:
|
||||
bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0];
|
||||
break;
|
||||
default:
|
||||
bio_for_each_segment(bv, bio_src, iter)
|
||||
bio->bi_io_vec[bio->bi_vcnt++] = bv;
|
||||
break;
|
||||
}
|
||||
|
||||
if (bio_integrity(bio_src)) {
|
||||
int ret;
|
||||
|
||||
ret = bio_integrity_clone(bio, bio_src, gfp_mask);
|
||||
if (ret < 0) {
|
||||
bio_put(bio);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
bio_clone_blkcg_association(bio, bio_src);
|
||||
|
||||
return bio;
|
||||
}
|
||||
EXPORT_SYMBOL(bio_clone_bioset);
|
||||
|
||||
/**
|
||||
* bio_add_pc_page - attempt to add page to bio
|
||||
* @q: the target queue
|
||||
|
|
@ -1661,10 +1586,8 @@ void bio_set_pages_dirty(struct bio *bio)
|
|||
int i;
|
||||
|
||||
bio_for_each_segment_all(bvec, bio, i) {
|
||||
struct page *page = bvec->bv_page;
|
||||
|
||||
if (page && !PageCompound(page))
|
||||
set_page_dirty_lock(page);
|
||||
if (!PageCompound(bvec->bv_page))
|
||||
set_page_dirty_lock(bvec->bv_page);
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(bio_set_pages_dirty);
|
||||
|
|
@ -1674,19 +1597,15 @@ static void bio_release_pages(struct bio *bio)
|
|||
struct bio_vec *bvec;
|
||||
int i;
|
||||
|
||||
bio_for_each_segment_all(bvec, bio, i) {
|
||||
struct page *page = bvec->bv_page;
|
||||
|
||||
if (page)
|
||||
put_page(page);
|
||||
}
|
||||
bio_for_each_segment_all(bvec, bio, i)
|
||||
put_page(bvec->bv_page);
|
||||
}
|
||||
|
||||
/*
|
||||
* bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
|
||||
* If they are, then fine. If, however, some pages are clean then they must
|
||||
* have been written out during the direct-IO read. So we take another ref on
|
||||
* the BIO and the offending pages and re-dirty the pages in process context.
|
||||
* the BIO and re-dirty the pages in process context.
|
||||
*
|
||||
* It is expected that bio_check_pages_dirty() will wholly own the BIO from
|
||||
* here on. It will run one put_page() against each page and will run one
|
||||
|
|
@ -1704,78 +1623,70 @@ static struct bio *bio_dirty_list;
|
|||
*/
|
||||
static void bio_dirty_fn(struct work_struct *work)
|
||||
{
|
||||
unsigned long flags;
|
||||
struct bio *bio;
|
||||
struct bio *bio, *next;
|
||||
|
||||
spin_lock_irqsave(&bio_dirty_lock, flags);
|
||||
bio = bio_dirty_list;
|
||||
spin_lock_irq(&bio_dirty_lock);
|
||||
next = bio_dirty_list;
|
||||
bio_dirty_list = NULL;
|
||||
spin_unlock_irqrestore(&bio_dirty_lock, flags);
|
||||
spin_unlock_irq(&bio_dirty_lock);
|
||||
|
||||
while (bio) {
|
||||
struct bio *next = bio->bi_private;
|
||||
while ((bio = next) != NULL) {
|
||||
next = bio->bi_private;
|
||||
|
||||
bio_set_pages_dirty(bio);
|
||||
bio_release_pages(bio);
|
||||
bio_put(bio);
|
||||
bio = next;
|
||||
}
|
||||
}
|
||||
|
||||
void bio_check_pages_dirty(struct bio *bio)
|
||||
{
|
||||
struct bio_vec *bvec;
|
||||
int nr_clean_pages = 0;
|
||||
unsigned long flags;
|
||||
int i;
|
||||
|
||||
bio_for_each_segment_all(bvec, bio, i) {
|
||||
struct page *page = bvec->bv_page;
|
||||
|
||||
if (PageDirty(page) || PageCompound(page)) {
|
||||
put_page(page);
|
||||
bvec->bv_page = NULL;
|
||||
} else {
|
||||
nr_clean_pages++;
|
||||
}
|
||||
if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page))
|
||||
goto defer;
|
||||
}
|
||||
|
||||
if (nr_clean_pages) {
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&bio_dirty_lock, flags);
|
||||
bio->bi_private = bio_dirty_list;
|
||||
bio_dirty_list = bio;
|
||||
spin_unlock_irqrestore(&bio_dirty_lock, flags);
|
||||
schedule_work(&bio_dirty_work);
|
||||
} else {
|
||||
bio_put(bio);
|
||||
}
|
||||
bio_release_pages(bio);
|
||||
bio_put(bio);
|
||||
return;
|
||||
defer:
|
||||
spin_lock_irqsave(&bio_dirty_lock, flags);
|
||||
bio->bi_private = bio_dirty_list;
|
||||
bio_dirty_list = bio;
|
||||
spin_unlock_irqrestore(&bio_dirty_lock, flags);
|
||||
schedule_work(&bio_dirty_work);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(bio_check_pages_dirty);
|
||||
|
||||
void generic_start_io_acct(struct request_queue *q, int rw,
|
||||
void generic_start_io_acct(struct request_queue *q, int op,
|
||||
unsigned long sectors, struct hd_struct *part)
|
||||
{
|
||||
const int sgrp = op_stat_group(op);
|
||||
int cpu = part_stat_lock();
|
||||
|
||||
part_round_stats(q, cpu, part);
|
||||
part_stat_inc(cpu, part, ios[rw]);
|
||||
part_stat_add(cpu, part, sectors[rw], sectors);
|
||||
part_inc_in_flight(q, part, rw);
|
||||
part_stat_inc(cpu, part, ios[sgrp]);
|
||||
part_stat_add(cpu, part, sectors[sgrp], sectors);
|
||||
part_inc_in_flight(q, part, op_is_write(op));
|
||||
|
||||
part_stat_unlock();
|
||||
}
|
||||
EXPORT_SYMBOL(generic_start_io_acct);
|
||||
|
||||
void generic_end_io_acct(struct request_queue *q, int rw,
|
||||
void generic_end_io_acct(struct request_queue *q, int req_op,
|
||||
struct hd_struct *part, unsigned long start_time)
|
||||
{
|
||||
unsigned long duration = jiffies - start_time;
|
||||
const int sgrp = op_stat_group(req_op);
|
||||
int cpu = part_stat_lock();
|
||||
|
||||
part_stat_add(cpu, part, ticks[rw], duration);
|
||||
part_stat_add(cpu, part, ticks[sgrp], duration);
|
||||
part_round_stats(q, cpu, part);
|
||||
part_dec_in_flight(q, part, rw);
|
||||
part_dec_in_flight(q, part, op_is_write(req_op));
|
||||
|
||||
part_stat_unlock();
|
||||
}
|
||||
|
|
@ -1834,6 +1745,9 @@ again:
|
|||
if (!bio_integrity_endio(bio))
|
||||
return;
|
||||
|
||||
if (bio->bi_disk)
|
||||
rq_qos_done_bio(bio->bi_disk->queue, bio);
|
||||
|
||||
/*
|
||||
* Need to have a real endio function for chained bios, otherwise
|
||||
* various corner cases will break (like stacking block devices that
|
||||
|
|
@ -2042,6 +1956,30 @@ EXPORT_SYMBOL(bioset_init_from_src);
|
|||
|
||||
#ifdef CONFIG_BLK_CGROUP
|
||||
|
||||
#ifdef CONFIG_MEMCG
|
||||
/**
|
||||
* bio_associate_blkcg_from_page - associate a bio with the page's blkcg
|
||||
* @bio: target bio
|
||||
* @page: the page to lookup the blkcg from
|
||||
*
|
||||
* Associate @bio with the blkcg from @page's owning memcg. This works like
|
||||
* every other associate function wrt references.
|
||||
*/
|
||||
int bio_associate_blkcg_from_page(struct bio *bio, struct page *page)
|
||||
{
|
||||
struct cgroup_subsys_state *blkcg_css;
|
||||
|
||||
if (unlikely(bio->bi_css))
|
||||
return -EBUSY;
|
||||
if (!page->mem_cgroup)
|
||||
return 0;
|
||||
blkcg_css = cgroup_get_e_css(page->mem_cgroup->css.cgroup,
|
||||
&io_cgrp_subsys);
|
||||
bio->bi_css = blkcg_css;
|
||||
return 0;
|
||||
}
|
||||
#endif /* CONFIG_MEMCG */
|
||||
|
||||
/**
|
||||
* bio_associate_blkcg - associate a bio with the specified blkcg
|
||||
* @bio: target bio
|
||||
|
|
@ -2064,6 +2002,24 @@ int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(bio_associate_blkcg);
|
||||
|
||||
/**
|
||||
* bio_associate_blkg - associate a bio with the specified blkg
|
||||
* @bio: target bio
|
||||
* @blkg: the blkg to associate
|
||||
*
|
||||
* Associate @bio with the blkg specified by @blkg. This is the queue specific
|
||||
* blkcg information associated with the @bio, a reference will be taken on the
|
||||
* @blkg and will be freed when the bio is freed.
|
||||
*/
|
||||
int bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg)
|
||||
{
|
||||
if (unlikely(bio->bi_blkg))
|
||||
return -EBUSY;
|
||||
blkg_get(blkg);
|
||||
bio->bi_blkg = blkg;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* bio_disassociate_task - undo bio_associate_current()
|
||||
* @bio: target bio
|
||||
|
|
@ -2078,6 +2034,10 @@ void bio_disassociate_task(struct bio *bio)
|
|||
css_put(bio->bi_css);
|
||||
bio->bi_css = NULL;
|
||||
}
|
||||
if (bio->bi_blkg) {
|
||||
blkg_put(bio->bi_blkg);
|
||||
bio->bi_blkg = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@
|
|||
#include <linux/atomic.h>
|
||||
#include <linux/ctype.h>
|
||||
#include <linux/blk-cgroup.h>
|
||||
#include <linux/tracehook.h>
|
||||
#include "blk.h"
|
||||
|
||||
#define MAX_KEY_LEN 100
|
||||
|
|
@ -50,6 +51,8 @@ static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
|
|||
|
||||
static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
|
||||
|
||||
static bool blkcg_debug_stats = false;
|
||||
|
||||
static bool blkcg_policy_enabled(struct request_queue *q,
|
||||
const struct blkcg_policy *pol)
|
||||
{
|
||||
|
|
@ -564,6 +567,7 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
|
|||
[BLKG_RWSTAT_WRITE] = "Write",
|
||||
[BLKG_RWSTAT_SYNC] = "Sync",
|
||||
[BLKG_RWSTAT_ASYNC] = "Async",
|
||||
[BLKG_RWSTAT_DISCARD] = "Discard",
|
||||
};
|
||||
const char *dname = blkg_dev_name(pd->blkg);
|
||||
u64 v;
|
||||
|
|
@ -577,7 +581,8 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
|
|||
(unsigned long long)atomic64_read(&rwstat->aux_cnt[i]));
|
||||
|
||||
v = atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_READ]) +
|
||||
atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]);
|
||||
atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]) +
|
||||
atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_DISCARD]);
|
||||
seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
|
||||
return v;
|
||||
}
|
||||
|
|
@ -954,30 +959,77 @@ static int blkcg_print_stat(struct seq_file *sf, void *v)
|
|||
|
||||
hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
|
||||
const char *dname;
|
||||
char *buf;
|
||||
struct blkg_rwstat rwstat;
|
||||
u64 rbytes, wbytes, rios, wios;
|
||||
u64 rbytes, wbytes, rios, wios, dbytes, dios;
|
||||
size_t size = seq_get_buf(sf, &buf), off = 0;
|
||||
int i;
|
||||
bool has_stats = false;
|
||||
|
||||
dname = blkg_dev_name(blkg);
|
||||
if (!dname)
|
||||
continue;
|
||||
|
||||
/*
|
||||
* Hooray string manipulation, count is the size written NOT
|
||||
* INCLUDING THE \0, so size is now count+1 less than what we
|
||||
* had before, but we want to start writing the next bit from
|
||||
* the \0 so we only add count to buf.
|
||||
*/
|
||||
off += scnprintf(buf+off, size-off, "%s ", dname);
|
||||
|
||||
spin_lock_irq(blkg->q->queue_lock);
|
||||
|
||||
rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
|
||||
offsetof(struct blkcg_gq, stat_bytes));
|
||||
rbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
|
||||
wbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
|
||||
dbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_DISCARD]);
|
||||
|
||||
rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
|
||||
offsetof(struct blkcg_gq, stat_ios));
|
||||
rios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
|
||||
wios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
|
||||
dios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_DISCARD]);
|
||||
|
||||
spin_unlock_irq(blkg->q->queue_lock);
|
||||
|
||||
if (rbytes || wbytes || rios || wios)
|
||||
seq_printf(sf, "%s rbytes=%llu wbytes=%llu rios=%llu wios=%llu\n",
|
||||
dname, rbytes, wbytes, rios, wios);
|
||||
if (rbytes || wbytes || rios || wios) {
|
||||
has_stats = true;
|
||||
off += scnprintf(buf+off, size-off,
|
||||
"rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
|
||||
rbytes, wbytes, rios, wios,
|
||||
dbytes, dios);
|
||||
}
|
||||
|
||||
if (!blkcg_debug_stats)
|
||||
goto next;
|
||||
|
||||
if (atomic_read(&blkg->use_delay)) {
|
||||
has_stats = true;
|
||||
off += scnprintf(buf+off, size-off,
|
||||
" use_delay=%d delay_nsec=%llu",
|
||||
atomic_read(&blkg->use_delay),
|
||||
(unsigned long long)atomic64_read(&blkg->delay_nsec));
|
||||
}
|
||||
|
||||
for (i = 0; i < BLKCG_MAX_POLS; i++) {
|
||||
struct blkcg_policy *pol = blkcg_policy[i];
|
||||
size_t written;
|
||||
|
||||
if (!blkg->pd[i] || !pol->pd_stat_fn)
|
||||
continue;
|
||||
|
||||
written = pol->pd_stat_fn(blkg->pd[i], buf+off, size-off);
|
||||
if (written)
|
||||
has_stats = true;
|
||||
off += written;
|
||||
}
|
||||
next:
|
||||
if (has_stats) {
|
||||
off += scnprintf(buf+off, size-off, "\n");
|
||||
seq_commit(sf, off);
|
||||
}
|
||||
}
|
||||
|
||||
rcu_read_unlock();
|
||||
|
|
@ -1191,6 +1243,14 @@ int blkcg_init_queue(struct request_queue *q)
|
|||
if (preloaded)
|
||||
radix_tree_preload_end();
|
||||
|
||||
ret = blk_iolatency_init(q);
|
||||
if (ret) {
|
||||
spin_lock_irq(q->queue_lock);
|
||||
blkg_destroy_all(q);
|
||||
spin_unlock_irq(q->queue_lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = blk_throtl_init(q);
|
||||
if (ret) {
|
||||
spin_lock_irq(q->queue_lock);
|
||||
|
|
@ -1288,6 +1348,13 @@ static void blkcg_bind(struct cgroup_subsys_state *root_css)
|
|||
mutex_unlock(&blkcg_pol_mutex);
|
||||
}
|
||||
|
||||
static void blkcg_exit(struct task_struct *tsk)
|
||||
{
|
||||
if (tsk->throttle_queue)
|
||||
blk_put_queue(tsk->throttle_queue);
|
||||
tsk->throttle_queue = NULL;
|
||||
}
|
||||
|
||||
struct cgroup_subsys io_cgrp_subsys = {
|
||||
.css_alloc = blkcg_css_alloc,
|
||||
.css_offline = blkcg_css_offline,
|
||||
|
|
@ -1297,6 +1364,7 @@ struct cgroup_subsys io_cgrp_subsys = {
|
|||
.dfl_cftypes = blkcg_files,
|
||||
.legacy_cftypes = blkcg_legacy_files,
|
||||
.legacy_name = "blkio",
|
||||
.exit = blkcg_exit,
|
||||
#ifdef CONFIG_MEMCG
|
||||
/*
|
||||
* This ensures that, if available, memcg is automatically enabled
|
||||
|
|
@ -1547,3 +1615,209 @@ out_unlock:
|
|||
mutex_unlock(&blkcg_pol_register_mutex);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
|
||||
|
||||
/*
|
||||
* Scale the accumulated delay based on how long it has been since we updated
|
||||
* the delay. We only call this when we are adding delay, in case it's been a
|
||||
* while since we added delay, and when we are checking to see if we need to
|
||||
* delay a task, to account for any delays that may have occurred.
|
||||
*/
|
||||
static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now)
|
||||
{
|
||||
u64 old = atomic64_read(&blkg->delay_start);
|
||||
|
||||
/*
|
||||
* We only want to scale down every second. The idea here is that we
|
||||
* want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain
|
||||
* time window. We only want to throttle tasks for recent delay that
|
||||
* has occurred, in 1 second time windows since that's the maximum
|
||||
* things can be throttled. We save the current delay window in
|
||||
* blkg->last_delay so we know what amount is still left to be charged
|
||||
* to the blkg from this point onward. blkg->last_use keeps track of
|
||||
* the use_delay counter. The idea is if we're unthrottling the blkg we
|
||||
* are ok with whatever is happening now, and we can take away more of
|
||||
* the accumulated delay as we've already throttled enough that
|
||||
* everybody is happy with their IO latencies.
|
||||
*/
|
||||
if (time_before64(old + NSEC_PER_SEC, now) &&
|
||||
atomic64_cmpxchg(&blkg->delay_start, old, now) == old) {
|
||||
u64 cur = atomic64_read(&blkg->delay_nsec);
|
||||
u64 sub = min_t(u64, blkg->last_delay, now - old);
|
||||
int cur_use = atomic_read(&blkg->use_delay);
|
||||
|
||||
/*
|
||||
* We've been unthrottled, subtract a larger chunk of our
|
||||
* accumulated delay.
|
||||
*/
|
||||
if (cur_use < blkg->last_use)
|
||||
sub = max_t(u64, sub, blkg->last_delay >> 1);
|
||||
|
||||
/*
|
||||
* This shouldn't happen, but handle it anyway. Our delay_nsec
|
||||
* should only ever be growing except here where we subtract out
|
||||
* min(last_delay, 1 second), but lord knows bugs happen and I'd
|
||||
* rather not end up with negative numbers.
|
||||
*/
|
||||
if (unlikely(cur < sub)) {
|
||||
atomic64_set(&blkg->delay_nsec, 0);
|
||||
blkg->last_delay = 0;
|
||||
} else {
|
||||
atomic64_sub(sub, &blkg->delay_nsec);
|
||||
blkg->last_delay = cur - sub;
|
||||
}
|
||||
blkg->last_use = cur_use;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* This is called when we want to actually walk up the hierarchy and check to
|
||||
* see if we need to throttle, and then actually throttle if there is some
|
||||
* accumulated delay. This should only be called upon return to user space so
|
||||
* we're not holding some lock that would induce a priority inversion.
|
||||
*/
|
||||
static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)
|
||||
{
|
||||
u64 now = ktime_to_ns(ktime_get());
|
||||
u64 exp;
|
||||
u64 delay_nsec = 0;
|
||||
int tok;
|
||||
|
||||
while (blkg->parent) {
|
||||
if (atomic_read(&blkg->use_delay)) {
|
||||
blkcg_scale_delay(blkg, now);
|
||||
delay_nsec = max_t(u64, delay_nsec,
|
||||
atomic64_read(&blkg->delay_nsec));
|
||||
}
|
||||
blkg = blkg->parent;
|
||||
}
|
||||
|
||||
if (!delay_nsec)
|
||||
return;
|
||||
|
||||
/*
|
||||
* Let's not sleep for all eternity if we've amassed a huge delay.
|
||||
* Swapping or metadata IO can accumulate 10's of seconds worth of
|
||||
* delay, and we want userspace to be able to do _something_ so cap the
|
||||
* delays at 1 second. If there's 10's of seconds worth of delay then
|
||||
* the tasks will be delayed for 1 second for every syscall.
|
||||
*/
|
||||
delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
|
||||
|
||||
/*
|
||||
* TODO: the use_memdelay flag is going to be for the upcoming psi stuff
|
||||
* that hasn't landed upstream yet. Once that stuff is in place we need
|
||||
* to do a psi_memstall_enter/leave if memdelay is set.
|
||||
*/
|
||||
|
||||
exp = ktime_add_ns(now, delay_nsec);
|
||||
tok = io_schedule_prepare();
|
||||
do {
|
||||
__set_current_state(TASK_KILLABLE);
|
||||
if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS))
|
||||
break;
|
||||
} while (!fatal_signal_pending(current));
|
||||
io_schedule_finish(tok);
|
||||
}
|
||||
|
||||
/**
|
||||
* blkcg_maybe_throttle_current - throttle the current task if it has been marked
|
||||
*
|
||||
* This is only called if we've been marked with set_notify_resume(). Obviously
|
||||
* we can be set_notify_resume() for reasons other than blkcg throttling, so we
|
||||
* check to see if current->throttle_queue is set and if not this doesn't do
|
||||
* anything. This should only ever be called by the resume code, it's not meant
|
||||
* to be called by people willy-nilly as it will actually do the work to
|
||||
* throttle the task if it is setup for throttling.
|
||||
*/
|
||||
void blkcg_maybe_throttle_current(void)
|
||||
{
|
||||
struct request_queue *q = current->throttle_queue;
|
||||
struct cgroup_subsys_state *css;
|
||||
struct blkcg *blkcg;
|
||||
struct blkcg_gq *blkg;
|
||||
bool use_memdelay = current->use_memdelay;
|
||||
|
||||
if (!q)
|
||||
return;
|
||||
|
||||
current->throttle_queue = NULL;
|
||||
current->use_memdelay = false;
|
||||
|
||||
rcu_read_lock();
|
||||
css = kthread_blkcg();
|
||||
if (css)
|
||||
blkcg = css_to_blkcg(css);
|
||||
else
|
||||
blkcg = css_to_blkcg(task_css(current, io_cgrp_id));
|
||||
|
||||
if (!blkcg)
|
||||
goto out;
|
||||
blkg = blkg_lookup(blkcg, q);
|
||||
if (!blkg)
|
||||
goto out;
|
||||
blkg = blkg_try_get(blkg);
|
||||
if (!blkg)
|
||||
goto out;
|
||||
rcu_read_unlock();
|
||||
|
||||
blkcg_maybe_throttle_blkg(blkg, use_memdelay);
|
||||
blkg_put(blkg);
|
||||
blk_put_queue(q);
|
||||
return;
|
||||
out:
|
||||
rcu_read_unlock();
|
||||
blk_put_queue(q);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(blkcg_maybe_throttle_current);
|
||||
|
||||
/**
|
||||
* blkcg_schedule_throttle - this task needs to check for throttling
|
||||
* @q - the request queue IO was submitted on
|
||||
* @use_memdelay - do we charge this to memory delay for PSI
|
||||
*
|
||||
* This is called by the IO controller when we know there's delay accumulated
|
||||
* for the blkg for this task. We do not pass the blkg because there are places
|
||||
* we call this that may not have that information, the swapping code for
|
||||
* instance will only have a request_queue at that point. This set's the
|
||||
* notify_resume for the task to check and see if it requires throttling before
|
||||
* returning to user space.
|
||||
*
|
||||
* We will only schedule once per syscall. You can call this over and over
|
||||
* again and it will only do the check once upon return to user space, and only
|
||||
* throttle once. If the task needs to be throttled again it'll need to be
|
||||
* re-set at the next time we see the task.
|
||||
*/
|
||||
void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay)
|
||||
{
|
||||
if (unlikely(current->flags & PF_KTHREAD))
|
||||
return;
|
||||
|
||||
if (!blk_get_queue(q))
|
||||
return;
|
||||
|
||||
if (current->throttle_queue)
|
||||
blk_put_queue(current->throttle_queue);
|
||||
current->throttle_queue = q;
|
||||
if (use_memdelay)
|
||||
current->use_memdelay = use_memdelay;
|
||||
set_notify_resume(current);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(blkcg_schedule_throttle);
|
||||
|
||||
/**
|
||||
* blkcg_add_delay - add delay to this blkg
|
||||
* @now - the current time in nanoseconds
|
||||
* @delta - how many nanoseconds of delay to add
|
||||
*
|
||||
* Charge @delta to the blkg's current delay accumulation. This is used to
|
||||
* throttle tasks if an IO controller thinks we need more throttling.
|
||||
*/
|
||||
void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
|
||||
{
|
||||
blkcg_scale_delay(blkg, now);
|
||||
atomic64_add(delta, &blkg->delay_nsec);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(blkcg_add_delay);
|
||||
|
||||
module_param(blkcg_debug_stats, bool, 0644);
|
||||
MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not");
|
||||
|
|
|
|||
106
block/blk-core.c
106
block/blk-core.c
|
|
@ -42,7 +42,7 @@
|
|||
#include "blk.h"
|
||||
#include "blk-mq.h"
|
||||
#include "blk-mq-sched.h"
|
||||
#include "blk-wbt.h"
|
||||
#include "blk-rq-qos.h"
|
||||
|
||||
#ifdef CONFIG_DEBUG_FS
|
||||
struct dentry *blk_debugfs_root;
|
||||
|
|
@ -715,6 +715,35 @@ void blk_set_queue_dying(struct request_queue *q)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(blk_set_queue_dying);
|
||||
|
||||
/* Unconfigure the I/O scheduler and dissociate from the cgroup controller. */
|
||||
void blk_exit_queue(struct request_queue *q)
|
||||
{
|
||||
/*
|
||||
* Since the I/O scheduler exit code may access cgroup information,
|
||||
* perform I/O scheduler exit before disassociating from the block
|
||||
* cgroup controller.
|
||||
*/
|
||||
if (q->elevator) {
|
||||
ioc_clear_queue(q);
|
||||
elevator_exit(q, q->elevator);
|
||||
q->elevator = NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Remove all references to @q from the block cgroup controller before
|
||||
* restoring @q->queue_lock to avoid that restoring this pointer causes
|
||||
* e.g. blkcg_print_blkgs() to crash.
|
||||
*/
|
||||
blkcg_exit_queue(q);
|
||||
|
||||
/*
|
||||
* Since the cgroup code may dereference the @q->backing_dev_info
|
||||
* pointer, only decrease its reference count after having removed the
|
||||
* association with the block cgroup controller.
|
||||
*/
|
||||
bdi_put(q->backing_dev_info);
|
||||
}
|
||||
|
||||
/**
|
||||
* blk_cleanup_queue - shutdown a request queue
|
||||
* @q: request queue to shutdown
|
||||
|
|
@ -762,9 +791,13 @@ void blk_cleanup_queue(struct request_queue *q)
|
|||
* make sure all in-progress dispatch are completed because
|
||||
* blk_freeze_queue() can only complete all requests, and
|
||||
* dispatch may still be in-progress since we dispatch requests
|
||||
* from more than one contexts
|
||||
* from more than one contexts.
|
||||
*
|
||||
* No need to quiesce queue if it isn't initialized yet since
|
||||
* blk_freeze_queue() should be enough for cases of passthrough
|
||||
* request.
|
||||
*/
|
||||
if (q->mq_ops)
|
||||
if (q->mq_ops && blk_queue_init_done(q))
|
||||
blk_mq_quiesce_queue(q);
|
||||
|
||||
/* for synchronous bio-based driver finish in-flight integrity i/o */
|
||||
|
|
@ -780,30 +813,7 @@ void blk_cleanup_queue(struct request_queue *q)
|
|||
*/
|
||||
WARN_ON_ONCE(q->kobj.state_in_sysfs);
|
||||
|
||||
/*
|
||||
* Since the I/O scheduler exit code may access cgroup information,
|
||||
* perform I/O scheduler exit before disassociating from the block
|
||||
* cgroup controller.
|
||||
*/
|
||||
if (q->elevator) {
|
||||
ioc_clear_queue(q);
|
||||
elevator_exit(q, q->elevator);
|
||||
q->elevator = NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Remove all references to @q from the block cgroup controller before
|
||||
* restoring @q->queue_lock to avoid that restoring this pointer causes
|
||||
* e.g. blkcg_print_blkgs() to crash.
|
||||
*/
|
||||
blkcg_exit_queue(q);
|
||||
|
||||
/*
|
||||
* Since the cgroup code may dereference the @q->backing_dev_info
|
||||
* pointer, only decrease its reference count after having removed the
|
||||
* association with the block cgroup controller.
|
||||
*/
|
||||
bdi_put(q->backing_dev_info);
|
||||
blk_exit_queue(q);
|
||||
|
||||
if (q->mq_ops)
|
||||
blk_mq_free_queue(q);
|
||||
|
|
@ -1180,6 +1190,7 @@ out_exit_flush_rq:
|
|||
q->exit_rq_fn(q, q->fq->flush_rq);
|
||||
out_free_flush_queue:
|
||||
blk_free_flush_queue(q->fq);
|
||||
q->fq = NULL;
|
||||
return -ENOMEM;
|
||||
}
|
||||
EXPORT_SYMBOL(blk_init_allocated_queue);
|
||||
|
|
@ -1641,7 +1652,7 @@ void blk_requeue_request(struct request_queue *q, struct request *rq)
|
|||
blk_delete_timer(rq);
|
||||
blk_clear_rq_complete(rq);
|
||||
trace_block_rq_requeue(q, rq);
|
||||
wbt_requeue(q->rq_wb, rq);
|
||||
rq_qos_requeue(q, rq);
|
||||
|
||||
if (rq->rq_flags & RQF_QUEUED)
|
||||
blk_queue_end_tag(q, rq);
|
||||
|
|
@ -1748,7 +1759,7 @@ void __blk_put_request(struct request_queue *q, struct request *req)
|
|||
/* this is a bio leak */
|
||||
WARN_ON(req->bio != NULL);
|
||||
|
||||
wbt_done(q->rq_wb, req);
|
||||
rq_qos_done(q, req);
|
||||
|
||||
/*
|
||||
* Request may not have originated from ll_rw_blk. if not,
|
||||
|
|
@ -1982,7 +1993,6 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
|
|||
int where = ELEVATOR_INSERT_SORT;
|
||||
struct request *req, *free;
|
||||
unsigned int request_count = 0;
|
||||
unsigned int wb_acct;
|
||||
|
||||
/*
|
||||
* low level driver can indicate that it wants pages above a
|
||||
|
|
@ -2040,7 +2050,7 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
|
|||
}
|
||||
|
||||
get_rq:
|
||||
wb_acct = wbt_wait(q->rq_wb, bio, q->queue_lock);
|
||||
rq_qos_throttle(q, bio, q->queue_lock);
|
||||
|
||||
/*
|
||||
* Grab a free request. This is might sleep but can not fail.
|
||||
|
|
@ -2050,7 +2060,7 @@ get_rq:
|
|||
req = get_request(q, bio->bi_opf, bio, 0, GFP_NOIO);
|
||||
if (IS_ERR(req)) {
|
||||
blk_queue_exit(q);
|
||||
__wbt_done(q->rq_wb, wb_acct);
|
||||
rq_qos_cleanup(q, bio);
|
||||
if (PTR_ERR(req) == -ENOMEM)
|
||||
bio->bi_status = BLK_STS_RESOURCE;
|
||||
else
|
||||
|
|
@ -2059,7 +2069,7 @@ get_rq:
|
|||
goto out_unlock;
|
||||
}
|
||||
|
||||
wbt_track(req, wb_acct);
|
||||
rq_qos_track(q, req, bio);
|
||||
|
||||
/*
|
||||
* After dropping the lock and possibly sleeping here, our request
|
||||
|
|
@ -2700,13 +2710,13 @@ EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
|
|||
void blk_account_io_completion(struct request *req, unsigned int bytes)
|
||||
{
|
||||
if (blk_do_io_stat(req)) {
|
||||
const int rw = rq_data_dir(req);
|
||||
const int sgrp = op_stat_group(req_op(req));
|
||||
struct hd_struct *part;
|
||||
int cpu;
|
||||
|
||||
cpu = part_stat_lock();
|
||||
part = req->part;
|
||||
part_stat_add(cpu, part, sectors[rw], bytes >> 9);
|
||||
part_stat_add(cpu, part, sectors[sgrp], bytes >> 9);
|
||||
part_stat_unlock();
|
||||
}
|
||||
}
|
||||
|
|
@ -2720,7 +2730,7 @@ void blk_account_io_done(struct request *req, u64 now)
|
|||
*/
|
||||
if (blk_do_io_stat(req) && !(req->rq_flags & RQF_FLUSH_SEQ)) {
|
||||
unsigned long duration;
|
||||
const int rw = rq_data_dir(req);
|
||||
const int sgrp = op_stat_group(req_op(req));
|
||||
struct hd_struct *part;
|
||||
int cpu;
|
||||
|
||||
|
|
@ -2728,10 +2738,10 @@ void blk_account_io_done(struct request *req, u64 now)
|
|||
cpu = part_stat_lock();
|
||||
part = req->part;
|
||||
|
||||
part_stat_inc(cpu, part, ios[rw]);
|
||||
part_stat_add(cpu, part, ticks[rw], duration);
|
||||
part_stat_inc(cpu, part, ios[sgrp]);
|
||||
part_stat_add(cpu, part, ticks[sgrp], duration);
|
||||
part_round_stats(req->q, cpu, part);
|
||||
part_dec_in_flight(req->q, part, rw);
|
||||
part_dec_in_flight(req->q, part, rq_data_dir(req));
|
||||
|
||||
hd_struct_put(part);
|
||||
part_stat_unlock();
|
||||
|
|
@ -2751,9 +2761,9 @@ static bool blk_pm_allow_request(struct request *rq)
|
|||
return rq->rq_flags & RQF_PM;
|
||||
case RPM_SUSPENDED:
|
||||
return false;
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
#else
|
||||
static bool blk_pm_allow_request(struct request *rq)
|
||||
|
|
@ -2980,7 +2990,7 @@ void blk_start_request(struct request *req)
|
|||
req->throtl_size = blk_rq_sectors(req);
|
||||
#endif
|
||||
req->rq_flags |= RQF_STATS;
|
||||
wbt_issue(req->q->rq_wb, req);
|
||||
rq_qos_issue(req->q, req);
|
||||
}
|
||||
|
||||
BUG_ON(blk_rq_is_complete(req));
|
||||
|
|
@ -3053,6 +3063,10 @@ EXPORT_SYMBOL_GPL(blk_steal_bios);
|
|||
* Passing the result of blk_rq_bytes() as @nr_bytes guarantees
|
||||
* %false return from this function.
|
||||
*
|
||||
* Note:
|
||||
* The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in both
|
||||
* blk_rq_bytes() and in blk_update_request().
|
||||
*
|
||||
* Return:
|
||||
* %false - this request doesn't have any more data
|
||||
* %true - this request has more data
|
||||
|
|
@ -3200,7 +3214,7 @@ void blk_finish_request(struct request *req, blk_status_t error)
|
|||
blk_account_io_done(req, now);
|
||||
|
||||
if (req->end_io) {
|
||||
wbt_done(req->q->rq_wb, req);
|
||||
rq_qos_done(q, req);
|
||||
req->end_io(req, error);
|
||||
} else {
|
||||
if (blk_bidi_rq(req))
|
||||
|
|
@ -3763,9 +3777,11 @@ EXPORT_SYMBOL(blk_finish_plug);
|
|||
*/
|
||||
void blk_pm_runtime_init(struct request_queue *q, struct device *dev)
|
||||
{
|
||||
/* not support for RQF_PM and ->rpm_status in blk-mq yet */
|
||||
if (q->mq_ops)
|
||||
/* Don't enable runtime PM for blk-mq until it is ready */
|
||||
if (q->mq_ops) {
|
||||
pm_runtime_disable(dev);
|
||||
return;
|
||||
}
|
||||
|
||||
q->dev = dev;
|
||||
q->rpm_status = RPM_ACTIVE;
|
||||
|
|
|
|||
|
|
@ -278,7 +278,7 @@ int create_task_io_context(struct task_struct *task, gfp_t gfp_flags, int node)
|
|||
atomic_set(&ioc->nr_tasks, 1);
|
||||
atomic_set(&ioc->active_ref, 1);
|
||||
spin_lock_init(&ioc->lock);
|
||||
INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC | __GFP_HIGH);
|
||||
INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC);
|
||||
INIT_HLIST_HEAD(&ioc->icq_list);
|
||||
INIT_WORK(&ioc->release_work, ioc_release_fn);
|
||||
|
||||
|
|
|
|||
955
block/blk-iolatency.c
Normal file
955
block/blk-iolatency.c
Normal file
|
|
@ -0,0 +1,955 @@
|
|||
/*
|
||||
* Block rq-qos base io controller
|
||||
*
|
||||
* This works similar to wbt with a few exceptions
|
||||
*
|
||||
* - It's bio based, so the latency covers the whole block layer in addition to
|
||||
* the actual io.
|
||||
* - We will throttle all IO that comes in here if we need to.
|
||||
* - We use the mean latency over the 100ms window. This is because writes can
|
||||
* be particularly fast, which could give us a false sense of the impact of
|
||||
* other workloads on our protected workload.
|
||||
* - By default there's no throttling, we set the queue_depth to UINT_MAX so
|
||||
* that we can have as many outstanding bio's as we're allowed to. Only at
|
||||
* throttle time do we pay attention to the actual queue depth.
|
||||
*
|
||||
* The hierarchy works like the cpu controller does, we track the latency at
|
||||
* every configured node, and each configured node has it's own independent
|
||||
* queue depth. This means that we only care about our latency targets at the
|
||||
* peer level. Some group at the bottom of the hierarchy isn't going to affect
|
||||
* a group at the end of some other path if we're only configred at leaf level.
|
||||
*
|
||||
* Consider the following
|
||||
*
|
||||
* root blkg
|
||||
* / \
|
||||
* fast (target=5ms) slow (target=10ms)
|
||||
* / \ / \
|
||||
* a b normal(15ms) unloved
|
||||
*
|
||||
* "a" and "b" have no target, but their combined io under "fast" cannot exceed
|
||||
* an average latency of 5ms. If it does then we will throttle the "slow"
|
||||
* group. In the case of "normal", if it exceeds its 15ms target, we will
|
||||
* throttle "unloved", but nobody else.
|
||||
*
|
||||
* In this example "fast", "slow", and "normal" will be the only groups actually
|
||||
* accounting their io latencies. We have to walk up the heirarchy to the root
|
||||
* on every submit and complete so we can do the appropriate stat recording and
|
||||
* adjust the queue depth of ourselves if needed.
|
||||
*
|
||||
* There are 2 ways we throttle IO.
|
||||
*
|
||||
* 1) Queue depth throttling. As we throttle down we will adjust the maximum
|
||||
* number of IO's we're allowed to have in flight. This starts at (u64)-1 down
|
||||
* to 1. If the group is only ever submitting IO for itself then this is the
|
||||
* only way we throttle.
|
||||
*
|
||||
* 2) Induced delay throttling. This is for the case that a group is generating
|
||||
* IO that has to be issued by the root cg to avoid priority inversion. So think
|
||||
* REQ_META or REQ_SWAP. If we are already at qd == 1 and we're getting a lot
|
||||
* of work done for us on behalf of the root cg and are being asked to scale
|
||||
* down more then we induce a latency at userspace return. We accumulate the
|
||||
* total amount of time we need to be punished by doing
|
||||
*
|
||||
* total_time += min_lat_nsec - actual_io_completion
|
||||
*
|
||||
* and then at throttle time will do
|
||||
*
|
||||
* throttle_time = min(total_time, NSEC_PER_SEC)
|
||||
*
|
||||
* This induced delay will throttle back the activity that is generating the
|
||||
* root cg issued io's, wethere that's some metadata intensive operation or the
|
||||
* group is using so much memory that it is pushing us into swap.
|
||||
*
|
||||
* Copyright (C) 2018 Josef Bacik
|
||||
*/
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/blk_types.h>
|
||||
#include <linux/backing-dev.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/timer.h>
|
||||
#include <linux/memcontrol.h>
|
||||
#include <linux/sched/loadavg.h>
|
||||
#include <linux/sched/signal.h>
|
||||
#include <trace/events/block.h>
|
||||
#include "blk-rq-qos.h"
|
||||
#include "blk-stat.h"
|
||||
|
||||
#define DEFAULT_SCALE_COOKIE 1000000U
|
||||
|
||||
static struct blkcg_policy blkcg_policy_iolatency;
|
||||
struct iolatency_grp;
|
||||
|
||||
struct blk_iolatency {
|
||||
struct rq_qos rqos;
|
||||
struct timer_list timer;
|
||||
atomic_t enabled;
|
||||
};
|
||||
|
||||
static inline struct blk_iolatency *BLKIOLATENCY(struct rq_qos *rqos)
|
||||
{
|
||||
return container_of(rqos, struct blk_iolatency, rqos);
|
||||
}
|
||||
|
||||
static inline bool blk_iolatency_enabled(struct blk_iolatency *blkiolat)
|
||||
{
|
||||
return atomic_read(&blkiolat->enabled) > 0;
|
||||
}
|
||||
|
||||
struct child_latency_info {
|
||||
spinlock_t lock;
|
||||
|
||||
/* Last time we adjusted the scale of everybody. */
|
||||
u64 last_scale_event;
|
||||
|
||||
/* The latency that we missed. */
|
||||
u64 scale_lat;
|
||||
|
||||
/* Total io's from all of our children for the last summation. */
|
||||
u64 nr_samples;
|
||||
|
||||
/* The guy who actually changed the latency numbers. */
|
||||
struct iolatency_grp *scale_grp;
|
||||
|
||||
/* Cookie to tell if we need to scale up or down. */
|
||||
atomic_t scale_cookie;
|
||||
};
|
||||
|
||||
struct iolatency_grp {
|
||||
struct blkg_policy_data pd;
|
||||
struct blk_rq_stat __percpu *stats;
|
||||
struct blk_iolatency *blkiolat;
|
||||
struct rq_depth rq_depth;
|
||||
struct rq_wait rq_wait;
|
||||
atomic64_t window_start;
|
||||
atomic_t scale_cookie;
|
||||
u64 min_lat_nsec;
|
||||
u64 cur_win_nsec;
|
||||
|
||||
/* total running average of our io latency. */
|
||||
u64 lat_avg;
|
||||
|
||||
/* Our current number of IO's for the last summation. */
|
||||
u64 nr_samples;
|
||||
|
||||
struct child_latency_info child_lat;
|
||||
};
|
||||
|
||||
#define BLKIOLATENCY_MIN_WIN_SIZE (100 * NSEC_PER_MSEC)
|
||||
#define BLKIOLATENCY_MAX_WIN_SIZE NSEC_PER_SEC
|
||||
/*
|
||||
* These are the constants used to fake the fixed-point moving average
|
||||
* calculation just like load average. The call to CALC_LOAD folds
|
||||
* (FIXED_1 (2048) - exp_factor) * new_sample into lat_avg. The sampling
|
||||
* window size is bucketed to try to approximately calculate average
|
||||
* latency such that 1/exp (decay rate) is [1 min, 2.5 min) when windows
|
||||
* elapse immediately. Note, windows only elapse with IO activity. Idle
|
||||
* periods extend the most recent window.
|
||||
*/
|
||||
#define BLKIOLATENCY_NR_EXP_FACTORS 5
|
||||
#define BLKIOLATENCY_EXP_BUCKET_SIZE (BLKIOLATENCY_MAX_WIN_SIZE / \
|
||||
(BLKIOLATENCY_NR_EXP_FACTORS - 1))
|
||||
static const u64 iolatency_exp_factors[BLKIOLATENCY_NR_EXP_FACTORS] = {
|
||||
2045, // exp(1/600) - 600 samples
|
||||
2039, // exp(1/240) - 240 samples
|
||||
2031, // exp(1/120) - 120 samples
|
||||
2023, // exp(1/80) - 80 samples
|
||||
2014, // exp(1/60) - 60 samples
|
||||
};
|
||||
|
||||
static inline struct iolatency_grp *pd_to_lat(struct blkg_policy_data *pd)
|
||||
{
|
||||
return pd ? container_of(pd, struct iolatency_grp, pd) : NULL;
|
||||
}
|
||||
|
||||
static inline struct iolatency_grp *blkg_to_lat(struct blkcg_gq *blkg)
|
||||
{
|
||||
return pd_to_lat(blkg_to_pd(blkg, &blkcg_policy_iolatency));
|
||||
}
|
||||
|
||||
static inline struct blkcg_gq *lat_to_blkg(struct iolatency_grp *iolat)
|
||||
{
|
||||
return pd_to_blkg(&iolat->pd);
|
||||
}
|
||||
|
||||
static inline bool iolatency_may_queue(struct iolatency_grp *iolat,
|
||||
wait_queue_entry_t *wait,
|
||||
bool first_block)
|
||||
{
|
||||
struct rq_wait *rqw = &iolat->rq_wait;
|
||||
|
||||
if (first_block && waitqueue_active(&rqw->wait) &&
|
||||
rqw->wait.head.next != &wait->entry)
|
||||
return false;
|
||||
return rq_wait_inc_below(rqw, iolat->rq_depth.max_depth);
|
||||
}
|
||||
|
||||
static void __blkcg_iolatency_throttle(struct rq_qos *rqos,
|
||||
struct iolatency_grp *iolat,
|
||||
spinlock_t *lock, bool issue_as_root,
|
||||
bool use_memdelay)
|
||||
__releases(lock)
|
||||
__acquires(lock)
|
||||
{
|
||||
struct rq_wait *rqw = &iolat->rq_wait;
|
||||
unsigned use_delay = atomic_read(&lat_to_blkg(iolat)->use_delay);
|
||||
DEFINE_WAIT(wait);
|
||||
bool first_block = true;
|
||||
|
||||
if (use_delay)
|
||||
blkcg_schedule_throttle(rqos->q, use_memdelay);
|
||||
|
||||
/*
|
||||
* To avoid priority inversions we want to just take a slot if we are
|
||||
* issuing as root. If we're being killed off there's no point in
|
||||
* delaying things, we may have been killed by OOM so throttling may
|
||||
* make recovery take even longer, so just let the IO's through so the
|
||||
* task can go away.
|
||||
*/
|
||||
if (issue_as_root || fatal_signal_pending(current)) {
|
||||
atomic_inc(&rqw->inflight);
|
||||
return;
|
||||
}
|
||||
|
||||
if (iolatency_may_queue(iolat, &wait, first_block))
|
||||
return;
|
||||
|
||||
do {
|
||||
prepare_to_wait_exclusive(&rqw->wait, &wait,
|
||||
TASK_UNINTERRUPTIBLE);
|
||||
|
||||
if (iolatency_may_queue(iolat, &wait, first_block))
|
||||
break;
|
||||
first_block = false;
|
||||
|
||||
if (lock) {
|
||||
spin_unlock_irq(lock);
|
||||
io_schedule();
|
||||
spin_lock_irq(lock);
|
||||
} else {
|
||||
io_schedule();
|
||||
}
|
||||
} while (1);
|
||||
|
||||
finish_wait(&rqw->wait, &wait);
|
||||
}
|
||||
|
||||
#define SCALE_DOWN_FACTOR 2
|
||||
#define SCALE_UP_FACTOR 4
|
||||
|
||||
static inline unsigned long scale_amount(unsigned long qd, bool up)
|
||||
{
|
||||
return max(up ? qd >> SCALE_UP_FACTOR : qd >> SCALE_DOWN_FACTOR, 1UL);
|
||||
}
|
||||
|
||||
/*
|
||||
* We scale the qd down faster than we scale up, so we need to use this helper
|
||||
* to adjust the scale_cookie accordingly so we don't prematurely get
|
||||
* scale_cookie at DEFAULT_SCALE_COOKIE and unthrottle too much.
|
||||
*
|
||||
* Each group has their own local copy of the last scale cookie they saw, so if
|
||||
* the global scale cookie goes up or down they know which way they need to go
|
||||
* based on their last knowledge of it.
|
||||
*/
|
||||
static void scale_cookie_change(struct blk_iolatency *blkiolat,
|
||||
struct child_latency_info *lat_info,
|
||||
bool up)
|
||||
{
|
||||
unsigned long qd = blk_queue_depth(blkiolat->rqos.q);
|
||||
unsigned long scale = scale_amount(qd, up);
|
||||
unsigned long old = atomic_read(&lat_info->scale_cookie);
|
||||
unsigned long max_scale = qd << 1;
|
||||
unsigned long diff = 0;
|
||||
|
||||
if (old < DEFAULT_SCALE_COOKIE)
|
||||
diff = DEFAULT_SCALE_COOKIE - old;
|
||||
|
||||
if (up) {
|
||||
if (scale + old > DEFAULT_SCALE_COOKIE)
|
||||
atomic_set(&lat_info->scale_cookie,
|
||||
DEFAULT_SCALE_COOKIE);
|
||||
else if (diff > qd)
|
||||
atomic_inc(&lat_info->scale_cookie);
|
||||
else
|
||||
atomic_add(scale, &lat_info->scale_cookie);
|
||||
} else {
|
||||
/*
|
||||
* We don't want to dig a hole so deep that it takes us hours to
|
||||
* dig out of it. Just enough that we don't throttle/unthrottle
|
||||
* with jagged workloads but can still unthrottle once pressure
|
||||
* has sufficiently dissipated.
|
||||
*/
|
||||
if (diff > qd) {
|
||||
if (diff < max_scale)
|
||||
atomic_dec(&lat_info->scale_cookie);
|
||||
} else {
|
||||
atomic_sub(scale, &lat_info->scale_cookie);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Change the queue depth of the iolatency_grp. We add/subtract 1/16th of the
|
||||
* queue depth at a time so we don't get wild swings and hopefully dial in to
|
||||
* fairer distribution of the overall queue depth.
|
||||
*/
|
||||
static void scale_change(struct iolatency_grp *iolat, bool up)
|
||||
{
|
||||
unsigned long qd = blk_queue_depth(iolat->blkiolat->rqos.q);
|
||||
unsigned long scale = scale_amount(qd, up);
|
||||
unsigned long old = iolat->rq_depth.max_depth;
|
||||
bool changed = false;
|
||||
|
||||
if (old > qd)
|
||||
old = qd;
|
||||
|
||||
if (up) {
|
||||
if (old == 1 && blkcg_unuse_delay(lat_to_blkg(iolat)))
|
||||
return;
|
||||
|
||||
if (old < qd) {
|
||||
changed = true;
|
||||
old += scale;
|
||||
old = min(old, qd);
|
||||
iolat->rq_depth.max_depth = old;
|
||||
wake_up_all(&iolat->rq_wait.wait);
|
||||
}
|
||||
} else if (old > 1) {
|
||||
old >>= 1;
|
||||
changed = true;
|
||||
iolat->rq_depth.max_depth = max(old, 1UL);
|
||||
}
|
||||
}
|
||||
|
||||
/* Check our parent and see if the scale cookie has changed. */
|
||||
static void check_scale_change(struct iolatency_grp *iolat)
|
||||
{
|
||||
struct iolatency_grp *parent;
|
||||
struct child_latency_info *lat_info;
|
||||
unsigned int cur_cookie;
|
||||
unsigned int our_cookie = atomic_read(&iolat->scale_cookie);
|
||||
u64 scale_lat;
|
||||
unsigned int old;
|
||||
int direction = 0;
|
||||
|
||||
if (lat_to_blkg(iolat)->parent == NULL)
|
||||
return;
|
||||
|
||||
parent = blkg_to_lat(lat_to_blkg(iolat)->parent);
|
||||
if (!parent)
|
||||
return;
|
||||
|
||||
lat_info = &parent->child_lat;
|
||||
cur_cookie = atomic_read(&lat_info->scale_cookie);
|
||||
scale_lat = READ_ONCE(lat_info->scale_lat);
|
||||
|
||||
if (cur_cookie < our_cookie)
|
||||
direction = -1;
|
||||
else if (cur_cookie > our_cookie)
|
||||
direction = 1;
|
||||
else
|
||||
return;
|
||||
|
||||
old = atomic_cmpxchg(&iolat->scale_cookie, our_cookie, cur_cookie);
|
||||
|
||||
/* Somebody beat us to the punch, just bail. */
|
||||
if (old != our_cookie)
|
||||
return;
|
||||
|
||||
if (direction < 0 && iolat->min_lat_nsec) {
|
||||
u64 samples_thresh;
|
||||
|
||||
if (!scale_lat || iolat->min_lat_nsec <= scale_lat)
|
||||
return;
|
||||
|
||||
/*
|
||||
* Sometimes high priority groups are their own worst enemy, so
|
||||
* instead of taking it out on some poor other group that did 5%
|
||||
* or less of the IO's for the last summation just skip this
|
||||
* scale down event.
|
||||
*/
|
||||
samples_thresh = lat_info->nr_samples * 5;
|
||||
samples_thresh = div64_u64(samples_thresh, 100);
|
||||
if (iolat->nr_samples <= samples_thresh)
|
||||
return;
|
||||
}
|
||||
|
||||
/* We're as low as we can go. */
|
||||
if (iolat->rq_depth.max_depth == 1 && direction < 0) {
|
||||
blkcg_use_delay(lat_to_blkg(iolat));
|
||||
return;
|
||||
}
|
||||
|
||||
/* We're back to the default cookie, unthrottle all the things. */
|
||||
if (cur_cookie == DEFAULT_SCALE_COOKIE) {
|
||||
blkcg_clear_delay(lat_to_blkg(iolat));
|
||||
iolat->rq_depth.max_depth = UINT_MAX;
|
||||
wake_up_all(&iolat->rq_wait.wait);
|
||||
return;
|
||||
}
|
||||
|
||||
scale_change(iolat, direction > 0);
|
||||
}
|
||||
|
||||
static void blkcg_iolatency_throttle(struct rq_qos *rqos, struct bio *bio,
|
||||
spinlock_t *lock)
|
||||
{
|
||||
struct blk_iolatency *blkiolat = BLKIOLATENCY(rqos);
|
||||
struct blkcg *blkcg;
|
||||
struct blkcg_gq *blkg;
|
||||
struct request_queue *q = rqos->q;
|
||||
bool issue_as_root = bio_issue_as_root_blkg(bio);
|
||||
|
||||
if (!blk_iolatency_enabled(blkiolat))
|
||||
return;
|
||||
|
||||
rcu_read_lock();
|
||||
blkcg = bio_blkcg(bio);
|
||||
bio_associate_blkcg(bio, &blkcg->css);
|
||||
blkg = blkg_lookup(blkcg, q);
|
||||
if (unlikely(!blkg)) {
|
||||
if (!lock)
|
||||
spin_lock_irq(q->queue_lock);
|
||||
blkg = blkg_lookup_create(blkcg, q);
|
||||
if (IS_ERR(blkg))
|
||||
blkg = NULL;
|
||||
if (!lock)
|
||||
spin_unlock_irq(q->queue_lock);
|
||||
}
|
||||
if (!blkg)
|
||||
goto out;
|
||||
|
||||
bio_issue_init(&bio->bi_issue, bio_sectors(bio));
|
||||
bio_associate_blkg(bio, blkg);
|
||||
out:
|
||||
rcu_read_unlock();
|
||||
while (blkg && blkg->parent) {
|
||||
struct iolatency_grp *iolat = blkg_to_lat(blkg);
|
||||
if (!iolat) {
|
||||
blkg = blkg->parent;
|
||||
continue;
|
||||
}
|
||||
|
||||
check_scale_change(iolat);
|
||||
__blkcg_iolatency_throttle(rqos, iolat, lock, issue_as_root,
|
||||
(bio->bi_opf & REQ_SWAP) == REQ_SWAP);
|
||||
blkg = blkg->parent;
|
||||
}
|
||||
if (!timer_pending(&blkiolat->timer))
|
||||
mod_timer(&blkiolat->timer, jiffies + HZ);
|
||||
}
|
||||
|
||||
static void iolatency_record_time(struct iolatency_grp *iolat,
|
||||
struct bio_issue *issue, u64 now,
|
||||
bool issue_as_root)
|
||||
{
|
||||
struct blk_rq_stat *rq_stat;
|
||||
u64 start = bio_issue_time(issue);
|
||||
u64 req_time;
|
||||
|
||||
/*
|
||||
* Have to do this so we are truncated to the correct time that our
|
||||
* issue is truncated to.
|
||||
*/
|
||||
now = __bio_issue_time(now);
|
||||
|
||||
if (now <= start)
|
||||
return;
|
||||
|
||||
req_time = now - start;
|
||||
|
||||
/*
|
||||
* We don't want to count issue_as_root bio's in the cgroups latency
|
||||
* statistics as it could skew the numbers downwards.
|
||||
*/
|
||||
if (unlikely(issue_as_root && iolat->rq_depth.max_depth != UINT_MAX)) {
|
||||
u64 sub = iolat->min_lat_nsec;
|
||||
if (req_time < sub)
|
||||
blkcg_add_delay(lat_to_blkg(iolat), now, sub - req_time);
|
||||
return;
|
||||
}
|
||||
|
||||
rq_stat = get_cpu_ptr(iolat->stats);
|
||||
blk_rq_stat_add(rq_stat, req_time);
|
||||
put_cpu_ptr(rq_stat);
|
||||
}
|
||||
|
||||
#define BLKIOLATENCY_MIN_ADJUST_TIME (500 * NSEC_PER_MSEC)
|
||||
#define BLKIOLATENCY_MIN_GOOD_SAMPLES 5
|
||||
|
||||
static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now)
|
||||
{
|
||||
struct blkcg_gq *blkg = lat_to_blkg(iolat);
|
||||
struct iolatency_grp *parent;
|
||||
struct child_latency_info *lat_info;
|
||||
struct blk_rq_stat stat;
|
||||
unsigned long flags;
|
||||
int cpu, exp_idx;
|
||||
|
||||
blk_rq_stat_init(&stat);
|
||||
preempt_disable();
|
||||
for_each_online_cpu(cpu) {
|
||||
struct blk_rq_stat *s;
|
||||
s = per_cpu_ptr(iolat->stats, cpu);
|
||||
blk_rq_stat_sum(&stat, s);
|
||||
blk_rq_stat_init(s);
|
||||
}
|
||||
preempt_enable();
|
||||
|
||||
parent = blkg_to_lat(blkg->parent);
|
||||
if (!parent)
|
||||
return;
|
||||
|
||||
lat_info = &parent->child_lat;
|
||||
|
||||
/*
|
||||
* CALC_LOAD takes in a number stored in fixed point representation.
|
||||
* Because we are using this for IO time in ns, the values stored
|
||||
* are significantly larger than the FIXED_1 denominator (2048).
|
||||
* Therefore, rounding errors in the calculation are negligible and
|
||||
* can be ignored.
|
||||
*/
|
||||
exp_idx = min_t(int, BLKIOLATENCY_NR_EXP_FACTORS - 1,
|
||||
div64_u64(iolat->cur_win_nsec,
|
||||
BLKIOLATENCY_EXP_BUCKET_SIZE));
|
||||
CALC_LOAD(iolat->lat_avg, iolatency_exp_factors[exp_idx], stat.mean);
|
||||
|
||||
/* Everything is ok and we don't need to adjust the scale. */
|
||||
if (stat.mean <= iolat->min_lat_nsec &&
|
||||
atomic_read(&lat_info->scale_cookie) == DEFAULT_SCALE_COOKIE)
|
||||
return;
|
||||
|
||||
/* Somebody beat us to the punch, just bail. */
|
||||
spin_lock_irqsave(&lat_info->lock, flags);
|
||||
lat_info->nr_samples -= iolat->nr_samples;
|
||||
lat_info->nr_samples += stat.nr_samples;
|
||||
iolat->nr_samples = stat.nr_samples;
|
||||
|
||||
if ((lat_info->last_scale_event >= now ||
|
||||
now - lat_info->last_scale_event < BLKIOLATENCY_MIN_ADJUST_TIME) &&
|
||||
lat_info->scale_lat <= iolat->min_lat_nsec)
|
||||
goto out;
|
||||
|
||||
if (stat.mean <= iolat->min_lat_nsec &&
|
||||
stat.nr_samples >= BLKIOLATENCY_MIN_GOOD_SAMPLES) {
|
||||
if (lat_info->scale_grp == iolat) {
|
||||
lat_info->last_scale_event = now;
|
||||
scale_cookie_change(iolat->blkiolat, lat_info, true);
|
||||
}
|
||||
} else if (stat.mean > iolat->min_lat_nsec) {
|
||||
lat_info->last_scale_event = now;
|
||||
if (!lat_info->scale_grp ||
|
||||
lat_info->scale_lat > iolat->min_lat_nsec) {
|
||||
WRITE_ONCE(lat_info->scale_lat, iolat->min_lat_nsec);
|
||||
lat_info->scale_grp = iolat;
|
||||
}
|
||||
scale_cookie_change(iolat->blkiolat, lat_info, false);
|
||||
}
|
||||
out:
|
||||
spin_unlock_irqrestore(&lat_info->lock, flags);
|
||||
}
|
||||
|
||||
static void blkcg_iolatency_done_bio(struct rq_qos *rqos, struct bio *bio)
|
||||
{
|
||||
struct blkcg_gq *blkg;
|
||||
struct rq_wait *rqw;
|
||||
struct iolatency_grp *iolat;
|
||||
u64 window_start;
|
||||
u64 now = ktime_to_ns(ktime_get());
|
||||
bool issue_as_root = bio_issue_as_root_blkg(bio);
|
||||
bool enabled = false;
|
||||
|
||||
blkg = bio->bi_blkg;
|
||||
if (!blkg)
|
||||
return;
|
||||
|
||||
iolat = blkg_to_lat(bio->bi_blkg);
|
||||
if (!iolat)
|
||||
return;
|
||||
|
||||
enabled = blk_iolatency_enabled(iolat->blkiolat);
|
||||
while (blkg && blkg->parent) {
|
||||
iolat = blkg_to_lat(blkg);
|
||||
if (!iolat) {
|
||||
blkg = blkg->parent;
|
||||
continue;
|
||||
}
|
||||
rqw = &iolat->rq_wait;
|
||||
|
||||
atomic_dec(&rqw->inflight);
|
||||
if (!enabled || iolat->min_lat_nsec == 0)
|
||||
goto next;
|
||||
iolatency_record_time(iolat, &bio->bi_issue, now,
|
||||
issue_as_root);
|
||||
window_start = atomic64_read(&iolat->window_start);
|
||||
if (now > window_start &&
|
||||
(now - window_start) >= iolat->cur_win_nsec) {
|
||||
if (atomic64_cmpxchg(&iolat->window_start,
|
||||
window_start, now) == window_start)
|
||||
iolatency_check_latencies(iolat, now);
|
||||
}
|
||||
next:
|
||||
wake_up(&rqw->wait);
|
||||
blkg = blkg->parent;
|
||||
}
|
||||
}
|
||||
|
||||
static void blkcg_iolatency_cleanup(struct rq_qos *rqos, struct bio *bio)
|
||||
{
|
||||
struct blkcg_gq *blkg;
|
||||
|
||||
blkg = bio->bi_blkg;
|
||||
while (blkg && blkg->parent) {
|
||||
struct rq_wait *rqw;
|
||||
struct iolatency_grp *iolat;
|
||||
|
||||
iolat = blkg_to_lat(blkg);
|
||||
if (!iolat)
|
||||
goto next;
|
||||
|
||||
rqw = &iolat->rq_wait;
|
||||
atomic_dec(&rqw->inflight);
|
||||
wake_up(&rqw->wait);
|
||||
next:
|
||||
blkg = blkg->parent;
|
||||
}
|
||||
}
|
||||
|
||||
static void blkcg_iolatency_exit(struct rq_qos *rqos)
|
||||
{
|
||||
struct blk_iolatency *blkiolat = BLKIOLATENCY(rqos);
|
||||
|
||||
del_timer_sync(&blkiolat->timer);
|
||||
blkcg_deactivate_policy(rqos->q, &blkcg_policy_iolatency);
|
||||
kfree(blkiolat);
|
||||
}
|
||||
|
||||
static struct rq_qos_ops blkcg_iolatency_ops = {
|
||||
.throttle = blkcg_iolatency_throttle,
|
||||
.cleanup = blkcg_iolatency_cleanup,
|
||||
.done_bio = blkcg_iolatency_done_bio,
|
||||
.exit = blkcg_iolatency_exit,
|
||||
};
|
||||
|
||||
static void blkiolatency_timer_fn(struct timer_list *t)
|
||||
{
|
||||
struct blk_iolatency *blkiolat = from_timer(blkiolat, t, timer);
|
||||
struct blkcg_gq *blkg;
|
||||
struct cgroup_subsys_state *pos_css;
|
||||
u64 now = ktime_to_ns(ktime_get());
|
||||
|
||||
rcu_read_lock();
|
||||
blkg_for_each_descendant_pre(blkg, pos_css,
|
||||
blkiolat->rqos.q->root_blkg) {
|
||||
struct iolatency_grp *iolat;
|
||||
struct child_latency_info *lat_info;
|
||||
unsigned long flags;
|
||||
u64 cookie;
|
||||
|
||||
/*
|
||||
* We could be exiting, don't access the pd unless we have a
|
||||
* ref on the blkg.
|
||||
*/
|
||||
if (!blkg_try_get(blkg))
|
||||
continue;
|
||||
|
||||
iolat = blkg_to_lat(blkg);
|
||||
if (!iolat)
|
||||
goto next;
|
||||
|
||||
lat_info = &iolat->child_lat;
|
||||
cookie = atomic_read(&lat_info->scale_cookie);
|
||||
|
||||
if (cookie >= DEFAULT_SCALE_COOKIE)
|
||||
goto next;
|
||||
|
||||
spin_lock_irqsave(&lat_info->lock, flags);
|
||||
if (lat_info->last_scale_event >= now)
|
||||
goto next_lock;
|
||||
|
||||
/*
|
||||
* We scaled down but don't have a scale_grp, scale up and carry
|
||||
* on.
|
||||
*/
|
||||
if (lat_info->scale_grp == NULL) {
|
||||
scale_cookie_change(iolat->blkiolat, lat_info, true);
|
||||
goto next_lock;
|
||||
}
|
||||
|
||||
/*
|
||||
* It's been 5 seconds since our last scale event, clear the
|
||||
* scale grp in case the group that needed the scale down isn't
|
||||
* doing any IO currently.
|
||||
*/
|
||||
if (now - lat_info->last_scale_event >=
|
||||
((u64)NSEC_PER_SEC * 5))
|
||||
lat_info->scale_grp = NULL;
|
||||
next_lock:
|
||||
spin_unlock_irqrestore(&lat_info->lock, flags);
|
||||
next:
|
||||
blkg_put(blkg);
|
||||
}
|
||||
rcu_read_unlock();
|
||||
}
|
||||
|
||||
int blk_iolatency_init(struct request_queue *q)
|
||||
{
|
||||
struct blk_iolatency *blkiolat;
|
||||
struct rq_qos *rqos;
|
||||
int ret;
|
||||
|
||||
blkiolat = kzalloc(sizeof(*blkiolat), GFP_KERNEL);
|
||||
if (!blkiolat)
|
||||
return -ENOMEM;
|
||||
|
||||
rqos = &blkiolat->rqos;
|
||||
rqos->id = RQ_QOS_CGROUP;
|
||||
rqos->ops = &blkcg_iolatency_ops;
|
||||
rqos->q = q;
|
||||
|
||||
rq_qos_add(q, rqos);
|
||||
|
||||
ret = blkcg_activate_policy(q, &blkcg_policy_iolatency);
|
||||
if (ret) {
|
||||
rq_qos_del(q, rqos);
|
||||
kfree(blkiolat);
|
||||
return ret;
|
||||
}
|
||||
|
||||
timer_setup(&blkiolat->timer, blkiolatency_timer_fn, 0);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void iolatency_set_min_lat_nsec(struct blkcg_gq *blkg, u64 val)
|
||||
{
|
||||
struct iolatency_grp *iolat = blkg_to_lat(blkg);
|
||||
struct blk_iolatency *blkiolat = iolat->blkiolat;
|
||||
u64 oldval = iolat->min_lat_nsec;
|
||||
|
||||
iolat->min_lat_nsec = val;
|
||||
iolat->cur_win_nsec = max_t(u64, val << 4, BLKIOLATENCY_MIN_WIN_SIZE);
|
||||
iolat->cur_win_nsec = min_t(u64, iolat->cur_win_nsec,
|
||||
BLKIOLATENCY_MAX_WIN_SIZE);
|
||||
|
||||
if (!oldval && val)
|
||||
atomic_inc(&blkiolat->enabled);
|
||||
if (oldval && !val)
|
||||
atomic_dec(&blkiolat->enabled);
|
||||
}
|
||||
|
||||
static void iolatency_clear_scaling(struct blkcg_gq *blkg)
|
||||
{
|
||||
if (blkg->parent) {
|
||||
struct iolatency_grp *iolat = blkg_to_lat(blkg->parent);
|
||||
struct child_latency_info *lat_info;
|
||||
if (!iolat)
|
||||
return;
|
||||
|
||||
lat_info = &iolat->child_lat;
|
||||
spin_lock(&lat_info->lock);
|
||||
atomic_set(&lat_info->scale_cookie, DEFAULT_SCALE_COOKIE);
|
||||
lat_info->last_scale_event = 0;
|
||||
lat_info->scale_grp = NULL;
|
||||
lat_info->scale_lat = 0;
|
||||
spin_unlock(&lat_info->lock);
|
||||
}
|
||||
}
|
||||
|
||||
static ssize_t iolatency_set_limit(struct kernfs_open_file *of, char *buf,
|
||||
size_t nbytes, loff_t off)
|
||||
{
|
||||
struct blkcg *blkcg = css_to_blkcg(of_css(of));
|
||||
struct blkcg_gq *blkg;
|
||||
struct blk_iolatency *blkiolat;
|
||||
struct blkg_conf_ctx ctx;
|
||||
struct iolatency_grp *iolat;
|
||||
char *p, *tok;
|
||||
u64 lat_val = 0;
|
||||
u64 oldval;
|
||||
int ret;
|
||||
|
||||
ret = blkg_conf_prep(blkcg, &blkcg_policy_iolatency, buf, &ctx);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
iolat = blkg_to_lat(ctx.blkg);
|
||||
blkiolat = iolat->blkiolat;
|
||||
p = ctx.body;
|
||||
|
||||
ret = -EINVAL;
|
||||
while ((tok = strsep(&p, " "))) {
|
||||
char key[16];
|
||||
char val[21]; /* 18446744073709551616 */
|
||||
|
||||
if (sscanf(tok, "%15[^=]=%20s", key, val) != 2)
|
||||
goto out;
|
||||
|
||||
if (!strcmp(key, "target")) {
|
||||
u64 v;
|
||||
|
||||
if (!strcmp(val, "max"))
|
||||
lat_val = 0;
|
||||
else if (sscanf(val, "%llu", &v) == 1)
|
||||
lat_val = v * NSEC_PER_USEC;
|
||||
else
|
||||
goto out;
|
||||
} else {
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
/* Walk up the tree to see if our new val is lower than it should be. */
|
||||
blkg = ctx.blkg;
|
||||
oldval = iolat->min_lat_nsec;
|
||||
|
||||
iolatency_set_min_lat_nsec(blkg, lat_val);
|
||||
if (oldval != iolat->min_lat_nsec) {
|
||||
iolatency_clear_scaling(blkg);
|
||||
}
|
||||
|
||||
ret = 0;
|
||||
out:
|
||||
blkg_conf_finish(&ctx);
|
||||
return ret ?: nbytes;
|
||||
}
|
||||
|
||||
static u64 iolatency_prfill_limit(struct seq_file *sf,
|
||||
struct blkg_policy_data *pd, int off)
|
||||
{
|
||||
struct iolatency_grp *iolat = pd_to_lat(pd);
|
||||
const char *dname = blkg_dev_name(pd->blkg);
|
||||
|
||||
if (!dname || !iolat->min_lat_nsec)
|
||||
return 0;
|
||||
seq_printf(sf, "%s target=%llu\n",
|
||||
dname, div_u64(iolat->min_lat_nsec, NSEC_PER_USEC));
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int iolatency_print_limit(struct seq_file *sf, void *v)
|
||||
{
|
||||
blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
|
||||
iolatency_prfill_limit,
|
||||
&blkcg_policy_iolatency, seq_cft(sf)->private, false);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t iolatency_pd_stat(struct blkg_policy_data *pd, char *buf,
|
||||
size_t size)
|
||||
{
|
||||
struct iolatency_grp *iolat = pd_to_lat(pd);
|
||||
unsigned long long avg_lat = div64_u64(iolat->lat_avg, NSEC_PER_USEC);
|
||||
unsigned long long cur_win = div64_u64(iolat->cur_win_nsec, NSEC_PER_MSEC);
|
||||
|
||||
if (iolat->rq_depth.max_depth == UINT_MAX)
|
||||
return scnprintf(buf, size, " depth=max avg_lat=%llu win=%llu",
|
||||
avg_lat, cur_win);
|
||||
|
||||
return scnprintf(buf, size, " depth=%u avg_lat=%llu win=%llu",
|
||||
iolat->rq_depth.max_depth, avg_lat, cur_win);
|
||||
}
|
||||
|
||||
|
||||
static struct blkg_policy_data *iolatency_pd_alloc(gfp_t gfp, int node)
|
||||
{
|
||||
struct iolatency_grp *iolat;
|
||||
|
||||
iolat = kzalloc_node(sizeof(*iolat), gfp, node);
|
||||
if (!iolat)
|
||||
return NULL;
|
||||
iolat->stats = __alloc_percpu_gfp(sizeof(struct blk_rq_stat),
|
||||
__alignof__(struct blk_rq_stat), gfp);
|
||||
if (!iolat->stats) {
|
||||
kfree(iolat);
|
||||
return NULL;
|
||||
}
|
||||
return &iolat->pd;
|
||||
}
|
||||
|
||||
static void iolatency_pd_init(struct blkg_policy_data *pd)
|
||||
{
|
||||
struct iolatency_grp *iolat = pd_to_lat(pd);
|
||||
struct blkcg_gq *blkg = lat_to_blkg(iolat);
|
||||
struct rq_qos *rqos = blkcg_rq_qos(blkg->q);
|
||||
struct blk_iolatency *blkiolat = BLKIOLATENCY(rqos);
|
||||
u64 now = ktime_to_ns(ktime_get());
|
||||
int cpu;
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
struct blk_rq_stat *stat;
|
||||
stat = per_cpu_ptr(iolat->stats, cpu);
|
||||
blk_rq_stat_init(stat);
|
||||
}
|
||||
|
||||
rq_wait_init(&iolat->rq_wait);
|
||||
spin_lock_init(&iolat->child_lat.lock);
|
||||
iolat->rq_depth.queue_depth = blk_queue_depth(blkg->q);
|
||||
iolat->rq_depth.max_depth = UINT_MAX;
|
||||
iolat->rq_depth.default_depth = iolat->rq_depth.queue_depth;
|
||||
iolat->blkiolat = blkiolat;
|
||||
iolat->cur_win_nsec = 100 * NSEC_PER_MSEC;
|
||||
atomic64_set(&iolat->window_start, now);
|
||||
|
||||
/*
|
||||
* We init things in list order, so the pd for the parent may not be
|
||||
* init'ed yet for whatever reason.
|
||||
*/
|
||||
if (blkg->parent && blkg_to_pd(blkg->parent, &blkcg_policy_iolatency)) {
|
||||
struct iolatency_grp *parent = blkg_to_lat(blkg->parent);
|
||||
atomic_set(&iolat->scale_cookie,
|
||||
atomic_read(&parent->child_lat.scale_cookie));
|
||||
} else {
|
||||
atomic_set(&iolat->scale_cookie, DEFAULT_SCALE_COOKIE);
|
||||
}
|
||||
|
||||
atomic_set(&iolat->child_lat.scale_cookie, DEFAULT_SCALE_COOKIE);
|
||||
}
|
||||
|
||||
static void iolatency_pd_offline(struct blkg_policy_data *pd)
|
||||
{
|
||||
struct iolatency_grp *iolat = pd_to_lat(pd);
|
||||
struct blkcg_gq *blkg = lat_to_blkg(iolat);
|
||||
|
||||
iolatency_set_min_lat_nsec(blkg, 0);
|
||||
iolatency_clear_scaling(blkg);
|
||||
}
|
||||
|
||||
static void iolatency_pd_free(struct blkg_policy_data *pd)
|
||||
{
|
||||
struct iolatency_grp *iolat = pd_to_lat(pd);
|
||||
free_percpu(iolat->stats);
|
||||
kfree(iolat);
|
||||
}
|
||||
|
||||
static struct cftype iolatency_files[] = {
|
||||
{
|
||||
.name = "latency",
|
||||
.flags = CFTYPE_NOT_ON_ROOT,
|
||||
.seq_show = iolatency_print_limit,
|
||||
.write = iolatency_set_limit,
|
||||
},
|
||||
{}
|
||||
};
|
||||
|
||||
static struct blkcg_policy blkcg_policy_iolatency = {
|
||||
.dfl_cftypes = iolatency_files,
|
||||
.pd_alloc_fn = iolatency_pd_alloc,
|
||||
.pd_init_fn = iolatency_pd_init,
|
||||
.pd_offline_fn = iolatency_pd_offline,
|
||||
.pd_free_fn = iolatency_pd_free,
|
||||
.pd_stat_fn = iolatency_pd_stat,
|
||||
};
|
||||
|
||||
static int __init iolatency_init(void)
|
||||
{
|
||||
return blkcg_policy_register(&blkcg_policy_iolatency);
|
||||
}
|
||||
|
||||
static void __exit iolatency_exit(void)
|
||||
{
|
||||
return blkcg_policy_unregister(&blkcg_policy_iolatency);
|
||||
}
|
||||
|
||||
module_init(iolatency_init);
|
||||
module_exit(iolatency_exit);
|
||||
|
|
@ -68,6 +68,8 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
|
|||
*/
|
||||
req_sects = min_t(sector_t, nr_sects,
|
||||
q->limits.max_discard_sectors);
|
||||
if (!req_sects)
|
||||
goto fail;
|
||||
if (req_sects > UINT_MAX >> 9)
|
||||
req_sects = UINT_MAX >> 9;
|
||||
|
||||
|
|
@ -105,6 +107,14 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
|
|||
|
||||
*biop = bio;
|
||||
return 0;
|
||||
|
||||
fail:
|
||||
if (bio) {
|
||||
submit_bio_wait(bio);
|
||||
bio_put(bio);
|
||||
}
|
||||
*biop = NULL;
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
EXPORT_SYMBOL(__blkdev_issue_discard);
|
||||
|
||||
|
|
|
|||
24
block/blk-mq-debugfs-zoned.c
Normal file
24
block/blk-mq-debugfs-zoned.c
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Copyright (C) 2017 Western Digital Corporation or its affiliates.
|
||||
*
|
||||
* This file is released under the GPL.
|
||||
*/
|
||||
|
||||
#include <linux/blkdev.h>
|
||||
#include "blk-mq-debugfs.h"
|
||||
|
||||
int queue_zone_wlock_show(void *data, struct seq_file *m)
|
||||
{
|
||||
struct request_queue *q = data;
|
||||
unsigned int i;
|
||||
|
||||
if (!q->seq_zones_wlock)
|
||||
return 0;
|
||||
|
||||
for (i = 0; i < q->nr_zones; i++)
|
||||
if (test_bit(i, q->seq_zones_wlock))
|
||||
seq_printf(m, "%u\n", i);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -206,21 +206,6 @@ static ssize_t queue_write_hint_store(void *data, const char __user *buf,
|
|||
return count;
|
||||
}
|
||||
|
||||
static int queue_zone_wlock_show(void *data, struct seq_file *m)
|
||||
{
|
||||
struct request_queue *q = data;
|
||||
unsigned int i;
|
||||
|
||||
if (!q->seq_zones_wlock)
|
||||
return 0;
|
||||
|
||||
for (i = 0; i < blk_queue_nr_zones(q); i++)
|
||||
if (test_bit(i, q->seq_zones_wlock))
|
||||
seq_printf(m, "%u\n", i);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct blk_mq_debugfs_attr blk_mq_debugfs_queue_attrs[] = {
|
||||
{ "poll_stat", 0400, queue_poll_stat_show },
|
||||
{ "requeue_list", 0400, .seq_ops = &queue_requeue_list_seq_ops },
|
||||
|
|
@ -637,6 +622,14 @@ static int hctx_active_show(void *data, struct seq_file *m)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int hctx_dispatch_busy_show(void *data, struct seq_file *m)
|
||||
{
|
||||
struct blk_mq_hw_ctx *hctx = data;
|
||||
|
||||
seq_printf(m, "%u\n", hctx->dispatch_busy);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void *ctx_rq_list_start(struct seq_file *m, loff_t *pos)
|
||||
__acquires(&ctx->lock)
|
||||
{
|
||||
|
|
@ -798,6 +791,7 @@ static const struct blk_mq_debugfs_attr blk_mq_debugfs_hctx_attrs[] = {
|
|||
{"queued", 0600, hctx_queued_show, hctx_queued_write},
|
||||
{"run", 0600, hctx_run_show, hctx_run_write},
|
||||
{"active", 0400, hctx_active_show},
|
||||
{"dispatch_busy", 0400, hctx_dispatch_busy_show},
|
||||
{},
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -80,4 +80,13 @@ static inline void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hc
|
|||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_BLK_DEBUG_FS_ZONED
|
||||
int queue_zone_wlock_show(void *data, struct seq_file *m);
|
||||
#else
|
||||
static inline int queue_zone_wlock_show(void *data, struct seq_file *m)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue