Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
Merge branch 'docs-next' of git://git.lwn.net/linux
Pull documentation updates from Jonathan Corbet:
"After a fair amount of churn in the last couple of cycles, docs are
taking it easier this time around. Lots of fixes and some new
documentation, but nothing all that radical. Perhaps the most
interesting change for many is the scripts/sphinx-pre-install tool
from Mauro; it will tell you exactly which packages you need to
install to get a working docs toolchain on your system.
There are two little patches reaching outside of Documentation/; both
just tweak kerneldoc comments to eliminate warnings and fix some
dangling doc pointers"
* 'docs-next' of git://git.lwn.net/linux: (52 commits)
Documentation/sphinx: fix kernel-doc decode for non-utf-8 locale
genalloc: Fix an incorrect kerneldoc comment
doc: Add documentation for the genalloc subsystem
assoc_array: fix path to assoc_array documentation
kernel-doc parser mishandles declarations split into lines
docs: ReSTify table of contents in core.rst
docs: process: drop git snapshots from applying-patches.rst
Documentation:input: fix typo
swap: Remove obsolete sentence
sphinx.rst: Allow Sphinx version 1.6 at the docs
docs-rst: fix verbatim font size on tables
Documentation: stable-kernel-rules: fix broken git urls
rtmutex: update rt-mutex
rtmutex: update rt-mutex-design
docs: fix minimal sphinx version in conf.py
docs: fix nested numbering in the TOC
NVMEM documentation fix: A minor typo
docs-rst: pdf: use same vertical margin on all Sphinx versions
doc: Makefile: if sphinx is not found, run a check script
docs: Fix paths in security/keys
...
+2
-10
Documentation/Makefile
+2
-10
Documentation/Makefile
···
22
23
.DEFAULT:
24
$(warning The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed and in PATH, or set the SPHINXBUILD make variable to point to the full path of the '$(SPHINXBUILD)' executable.)
25
@echo " SKIP Sphinx $@ target."
26
27
else # HAVE_SPHINX
···
96
97
# The following targets are independent of HAVE_SPHINX, and the rules should
98
# work or silently pass without Sphinx.
99
-
100
-
# no-ops for the Sphinx toolchain
101
-
sgmldocs:
102
-
@:
103
-
psdocs:
104
-
@:
105
-
mandocs:
106
-
@:
107
-
installmandocs:
108
-
@:
109
110
cleandocs:
111
$(Q)rm -rf $(BUILDDIR)
···
22
23
.DEFAULT:
24
$(warning The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed and in PATH, or set the SPHINXBUILD make variable to point to the full path of the '$(SPHINXBUILD)' executable.)
25
+
@echo
26
+
@./scripts/sphinx-pre-install
27
@echo " SKIP Sphinx $@ target."
28
29
else # HAVE_SPHINX
···
94
95
# The following targets are independent of HAVE_SPHINX, and the rules should
96
# work or silently pass without Sphinx.
97
98
cleandocs:
99
$(Q)rm -rf $(BUILDDIR)
+1
-1
Documentation/arm/firmware.txt
+1
-1
Documentation/arm/firmware.txt
+3
-3
Documentation/conf.py
+3
-3
Documentation/conf.py
···
29
# -- General configuration ------------------------------------------------
30
31
# If your documentation needs a minimal Sphinx version, state it here.
32
-
needs_sphinx = '1.2'
33
34
# Add any Sphinx extension module names here, as strings. They can be
35
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
···
344
if major == 1 and minor <= 4:
345
latex_elements['preamble'] += '\\usepackage[margin=0.5in, top=1in, bottom=1in]{geometry}'
346
elif major == 1 and (minor > 5 or (minor == 5 and patch >= 3)):
347
-
latex_elements['sphinxsetup'] = 'hmargin=0.5in, vmargin=0.5in'
348
-
349
350
# Grouping the document tree into LaTeX files. List of tuples
351
# (source start file, target name, title,
···
29
# -- General configuration ------------------------------------------------
30
31
# If your documentation needs a minimal Sphinx version, state it here.
32
+
needs_sphinx = '1.3'
33
34
# Add any Sphinx extension module names here, as strings. They can be
35
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
···
344
if major == 1 and minor <= 4:
345
latex_elements['preamble'] += '\\usepackage[margin=0.5in, top=1in, bottom=1in]{geometry}'
346
elif major == 1 and (minor > 5 or (minor == 5 and patch >= 3)):
347
+
latex_elements['sphinxsetup'] = 'hmargin=0.5in, vmargin=1in'
348
+
latex_elements['preamble'] += '\\fvset{fontsize=auto}\n'
349
350
# Grouping the document tree into LaTeX files. List of tuples
351
# (source start file, target name, title,
+144
Documentation/core-api/genalloc.rst
+144
Documentation/core-api/genalloc.rst
···
···
1
+
The genalloc/genpool subsystem
2
+
==============================
3
+
4
+
There are a number of memory-allocation subsystems in the kernel, each
5
+
aimed at a specific need. Sometimes, however, a kernel developer needs to
6
+
implement a new allocator for a specific range of special-purpose memory;
7
+
often that memory is located on a device somewhere. The author of the
8
+
driver for that device can certainly write a little allocator to get the
9
+
job done, but that is the way to fill the kernel with dozens of poorly
10
+
tested allocators. Back in 2005, Jes Sorensen lifted one of those
11
+
allocators from the sym53c8xx_2 driver and posted_ it as a generic module
12
+
for the creation of ad hoc memory allocators. This code was merged
13
+
for the 2.6.13 release; it has been modified considerably since then.
14
+
15
+
.. _posted: https://lwn.net/Articles/125842/
16
+
17
+
Code using this allocator should include <linux/genalloc.h>. The action
18
+
begins with the creation of a pool using one of:
19
+
20
+
.. kernel-doc:: lib/genalloc.c
21
+
:functions: gen_pool_create
22
+
23
+
.. kernel-doc:: lib/genalloc.c
24
+
:functions: devm_gen_pool_create
25
+
26
+
A call to :c:func:`gen_pool_create` will create a pool. The granularity of
27
+
allocations is set with min_alloc_order; it is a log-base-2 number like
28
+
those used by the page allocator, but it refers to bytes rather than pages.
29
+
So, if min_alloc_order is passed as 3, then all allocations will be a
30
+
multiple of eight bytes. Increasing min_alloc_order decreases the memory
31
+
required to track the memory in the pool. The nid parameter specifies
32
+
which NUMA node should be used for the allocation of the housekeeping
33
+
structures; it can be -1 if the caller doesn't care.
34
+
35
+
The "managed" interface :c:func:`devm_gen_pool_create` ties the pool to a
36
+
specific device. Among other things, it will automatically clean up the
37
+
pool when the given device is destroyed.
38
+
39
+
A pool is shut down with:
40
+
41
+
.. kernel-doc:: lib/genalloc.c
42
+
:functions: gen_pool_destroy
43
+
44
+
It's worth noting that, if there are still allocations outstanding from the
45
+
given pool, this function will take the rather extreme step of invoking
46
+
BUG(), crashing the entire system. You have been warned.
47
+
48
+
A freshly created pool has no memory to allocate. It is fairly useless in
49
+
that state, so one of the first orders of business is usually to add memory
50
+
to the pool. That can be done with one of:
51
+
52
+
.. kernel-doc:: include/linux/genalloc.h
53
+
:functions: gen_pool_add
54
+
55
+
.. kernel-doc:: lib/genalloc.c
56
+
:functions: gen_pool_add_virt
57
+
58
+
A call to :c:func:`gen_pool_add` will place the size bytes of memory
59
+
starting at addr (in the kernel's virtual address space) into the given
60
+
pool, once again using nid as the node ID for ancillary memory allocations.
61
+
The :c:func:`gen_pool_add_virt` variant associates an explicit physical
62
+
address with the memory; this is only necessary if the pool will be used
63
+
for DMA allocations.
64
+
65
+
The functions for allocating memory from the pool (and putting it back)
66
+
are:
67
+
68
+
.. kernel-doc:: lib/genalloc.c
69
+
:functions: gen_pool_alloc
70
+
71
+
.. kernel-doc:: lib/genalloc.c
72
+
:functions: gen_pool_dma_alloc
73
+
74
+
.. kernel-doc:: lib/genalloc.c
75
+
:functions: gen_pool_free
76
+
77
+
As one would expect, :c:func:`gen_pool_alloc` will allocate size< bytes
78
+
from the given pool. The :c:func:`gen_pool_dma_alloc` variant allocates
79
+
memory for use with DMA operations, returning the associated physical
80
+
address in the space pointed to by dma. This will only work if the memory
81
+
was added with :c:func:`gen_pool_add_virt`. Note that this function
82
+
departs from the usual genpool pattern of using unsigned long values to
83
+
represent kernel addresses; it returns a void * instead.
84
+
85
+
That all seems relatively simple; indeed, some developers clearly found it
86
+
to be too simple. After all, the interface above provides no control over
87
+
how the allocation functions choose which specific piece of memory to
88
+
return. If that sort of control is needed, the following functions will be
89
+
of interest:
90
+
91
+
.. kernel-doc:: lib/genalloc.c
92
+
:functions: gen_pool_alloc_algo
93
+
94
+
.. kernel-doc:: lib/genalloc.c
95
+
:functions: gen_pool_set_algo
96
+
97
+
Allocations with :c:func:`gen_pool_alloc_algo` specify an algorithm to be
98
+
used to choose the memory to be allocated; the default algorithm can be set
99
+
with :c:func:`gen_pool_set_algo`. The data value is passed to the
100
+
algorithm; most ignore it, but it is occasionally needed. One can,
101
+
naturally, write a special-purpose algorithm, but there is a fair set
102
+
already available:
103
+
104
+
- gen_pool_first_fit is a simple first-fit allocator; this is the default
105
+
algorithm if none other has been specified.
106
+
107
+
- gen_pool_first_fit_align forces the allocation to have a specific
108
+
alignment (passed via data in a genpool_data_align structure).
109
+
110
+
- gen_pool_first_fit_order_align aligns the allocation to the order of the
111
+
size. A 60-byte allocation will thus be 64-byte aligned, for example.
112
+
113
+
- gen_pool_best_fit, as one would expect, is a simple best-fit allocator.
114
+
115
+
- gen_pool_fixed_alloc allocates at a specific offset (passed in a
116
+
genpool_data_fixed structure via the data parameter) within the pool.
117
+
If the indicated memory is not available the allocation fails.
118
+
119
+
There is a handful of other functions, mostly for purposes like querying
120
+
the space available in the pool or iterating through chunks of memory.
121
+
Most users, however, should not need much beyond what has been described
122
+
above. With luck, wider awareness of this module will help to prevent the
123
+
writing of special-purpose memory allocators in the future.
124
+
125
+
.. kernel-doc:: lib/genalloc.c
126
+
:functions: gen_pool_virt_to_phys
127
+
128
+
.. kernel-doc:: lib/genalloc.c
129
+
:functions: gen_pool_for_each_chunk
130
+
131
+
.. kernel-doc:: lib/genalloc.c
132
+
:functions: addr_in_gen_pool
133
+
134
+
.. kernel-doc:: lib/genalloc.c
135
+
:functions: gen_pool_avail
136
+
137
+
.. kernel-doc:: lib/genalloc.c
138
+
:functions: gen_pool_size
139
+
140
+
.. kernel-doc:: lib/genalloc.c
141
+
:functions: gen_pool_get
142
+
143
+
.. kernel-doc:: lib/genalloc.c
144
+
:functions: of_gen_pool_get
+1
Documentation/core-api/index.rst
+1
Documentation/core-api/index.rst
+4
-2
Documentation/dev-tools/gdb-kernel-debugging.rst
+4
-2
Documentation/dev-tools/gdb-kernel-debugging.rst
···
31
CONFIG_DEBUG_INFO_REDUCED off. If your architecture supports
32
CONFIG_FRAME_POINTER, keep it enabled.
33
34
-
- Install that kernel on the guest.
35
Alternatively, QEMU allows to boot the kernel directly using -kernel,
36
-append, -initrd command line switches. This is generally only useful if
37
you do not depend on modules. See QEMU documentation for more details on
38
-
this mode.
39
40
- Enable the gdb stub of QEMU/KVM, either
41
···
31
CONFIG_DEBUG_INFO_REDUCED off. If your architecture supports
32
CONFIG_FRAME_POINTER, keep it enabled.
33
34
+
- Install that kernel on the guest, turn off KASLR if necessary by adding
35
+
"nokaslr" to the kernel command line.
36
Alternatively, QEMU allows to boot the kernel directly using -kernel,
37
-append, -initrd command line switches. This is generally only useful if
38
you do not depend on modules. See QEMU documentation for more details on
39
+
this mode. In this case, you should build the kernel with
40
+
CONFIG_RANDOMIZE_BASE disabled if the architecture supports KASLR.
41
42
- Enable the gdb stub of QEMU/KVM, either
43
+10
-1
Documentation/dev-tools/kgdb.rst
+10
-1
Documentation/dev-tools/kgdb.rst
···
348
- ``echo 1 > /sys/module/debug_core/parameters/kgdbreboot``
349
- Enter the debugger on reboot notify.
350
351
+
Kernel parameter: ``nokaslr``
352
+
-----------------------------
353
+
354
+
If the architecture that you are using enable KASLR by default,
355
+
you should consider turning it off. KASLR randomizes the
356
+
virtual address where the kernel image is mapped and confuse
357
+
gdb which resolve kernel symbol address from symbol table
358
+
of vmlinux.
359
+
360
Using kdb
361
=========
362
···
358
359
1. Configure kgdboc at boot using kernel parameters::
360
361
+
console=ttyS0,115200 kgdboc=ttyS0,115200 nokaslr
362
363
OR
364
+106
-1
Documentation/doc-guide/sphinx.rst
+106
-1
Documentation/doc-guide/sphinx.rst
···
19
``Documentation``. Some of these will likely be converted to reStructuredText
20
over time, but the bulk of them will remain in plain text.
21
22
Sphinx Build
23
============
24
···
222
the C domain
223
------------
224
225
-
The `Sphinx C Domain`_ (name c) is suited for documentation of C API. E.g. a
226
function prototype:
227
228
.. code-block:: rst
···
333
334
- column 3
335
336
337
Figures & Images
338
================
···
19
``Documentation``. Some of these will likely be converted to reStructuredText
20
over time, but the bulk of them will remain in plain text.
21
22
+
.. _sphinx_install:
23
+
24
+
Sphinx Install
25
+
==============
26
+
27
+
The ReST markups currently used by the Documentation/ files are meant to be
28
+
built with ``Sphinx`` version 1.3 or upper. If you're desiring to build
29
+
PDF outputs, it is recommended to use version 1.4.6 or upper.
30
+
31
+
There's a script that checks for the Spinx requirements. Please see
32
+
:ref:`sphinx-pre-install` for further details.
33
+
34
+
Most distributions are shipped with Sphinx, but its toolchain is fragile,
35
+
and it is not uncommon that upgrading it or some other Python packages
36
+
on your machine would cause the documentation build to break.
37
+
38
+
A way to get rid of that is to use a different version than the one shipped
39
+
on your distributions. In order to do that, it is recommended to install
40
+
Sphinx inside a virtual environment, using ``virtualenv-3``
41
+
or ``virtualenv``, depending on how your distribution packaged Python 3.
42
+
43
+
.. note::
44
+
45
+
#) Sphinx versions below 1.5 don't work properly with Python's
46
+
docutils version 0.13.1 or upper. So, if you're willing to use
47
+
those versions, you should run ``pip install 'docutils==0.12'``.
48
+
49
+
#) It is recommended to use the RTD theme for html output. Depending
50
+
on the Sphinx version, it should be installed in separate,
51
+
with ``pip install sphinx_rtd_theme``.
52
+
53
+
#) Some ReST pages contain math expressions. Due to the way Sphinx work,
54
+
those expressions are written using LaTeX notation. It needs texlive
55
+
installed with amdfonts and amsmath in order to evaluate them.
56
+
57
+
In summary, if you want to install Sphinx version 1.4.9, you should do::
58
+
59
+
$ virtualenv sphinx_1.4
60
+
$ . sphinx_1.4/bin/activate
61
+
(sphinx_1.4) $ pip install -r Documentation/sphinx/requirements.txt
62
+
63
+
After running ``. sphinx_1.4/bin/activate``, the prompt will change,
64
+
in order to indicate that you're using the new environment. If you
65
+
open a new shell, you need to rerun this command to enter again at
66
+
the virtual environment before building the documentation.
67
+
68
+
Image output
69
+
------------
70
+
71
+
The kernel documentation build system contains an extension that
72
+
handles images on both GraphViz and SVG formats (see
73
+
:ref:`sphinx_kfigure`).
74
+
75
+
For it to work, you need to install both GraphViz and ImageMagick
76
+
packages. If those packages are not installed, the build system will
77
+
still build the documentation, but won't include any images at the
78
+
output.
79
+
80
+
PDF and LaTeX builds
81
+
--------------------
82
+
83
+
Such builds are currently supported only with Sphinx versions 1.4 and upper.
84
+
85
+
For PDF and LaTeX output, you'll also need ``XeLaTeX`` version 3.14159265.
86
+
87
+
Depending on the distribution, you may also need to install a series of
88
+
``texlive`` packages that provide the minimal set of functionalities
89
+
required for ``XeLaTeX`` to work.
90
+
91
+
.. _sphinx-pre-install:
92
+
93
+
Checking for Sphinx dependencies
94
+
--------------------------------
95
+
96
+
There's a script that automatically check for Sphinx dependencies. If it can
97
+
recognize your distribution, it will also give a hint about the install
98
+
command line options for your distro::
99
+
100
+
$ ./scripts/sphinx-pre-install
101
+
Checking if the needed tools for Fedora release 26 (Twenty Six) are available
102
+
Warning: better to also install "texlive-luatex85".
103
+
You should run:
104
+
105
+
sudo dnf install -y texlive-luatex85
106
+
/usr/bin/virtualenv sphinx_1.4
107
+
. sphinx_1.4/bin/activate
108
+
pip install -r Documentation/sphinx/requirements.txt
109
+
110
+
Can't build as 1 mandatory dependency is missing at ./scripts/sphinx-pre-install line 468.
111
+
112
+
By default, it checks all the requirements for both html and PDF, including
113
+
the requirements for images, math expressions and LaTeX build, and assumes
114
+
that a virtual Python environment will be used. The ones needed for html
115
+
builds are assumed to be mandatory; the others to be optional.
116
+
117
+
It supports two optional parameters:
118
+
119
+
``--no-pdf``
120
+
Disable checks for PDF;
121
+
122
+
``--no-virtualenv``
123
+
Use OS packaging for Sphinx instead of Python virtual environment.
124
+
125
+
126
Sphinx Build
127
============
128
···
118
the C domain
119
------------
120
121
+
The **Sphinx C Domain** (name c) is suited for documentation of C API. E.g. a
122
function prototype:
123
124
.. code-block:: rst
···
229
230
- column 3
231
232
+
.. _sphinx_kfigure:
233
234
Figures & Images
235
================
+1
-4
Documentation/driver-api/basics.rst
+1
-4
Documentation/driver-api/basics.rst
···
4
Driver Entry and Exit points
5
----------------------------
6
7
-
.. kernel-doc:: include/linux/init.h
8
:internal:
9
10
Driver device table
···
101
:export:
102
103
.. kernel-doc:: kernel/panic.c
104
-
:export:
105
-
106
-
.. kernel-doc:: kernel/sys.c
107
:export:
108
109
.. kernel-doc:: kernel/rcu/tree.c
-3
Documentation/driver-api/dma-buf.rst
-3
Documentation/driver-api/dma-buf.rst
-1
Documentation/driver-api/miscellaneous.rst
-1
Documentation/driver-api/miscellaneous.rst
+1
-1
Documentation/driver-api/s390-drivers.rst
+1
-1
Documentation/driver-api/s390-drivers.rst
-8
Documentation/driver-api/scsi.rst
-8
Documentation/driver-api/scsi.rst
+1
-1
Documentation/features/core/tracehook/arch-support.txt
+1
-1
Documentation/features/core/tracehook/arch-support.txt
+1
-3
Documentation/filesystems/vfs.txt
+1
-3
Documentation/filesystems/vfs.txt
···
829
swap_activate: Called when swapon is used on a file to allocate
830
space if necessary and pin the block lookup information in
831
memory. A return value of zero indicates success,
832
-
in which case this file can be used to back swapspace. The
833
-
swapspace operations will be proxied to this address space's
834
-
->swap_{out,in} methods.
835
836
swap_deactivate: Called during swapoff on files where swap_activate
837
was successful.
···
829
swap_activate: Called when swapon is used on a file to allocate
830
space if necessary and pin the block lookup information in
831
memory. A return value of zero indicates success,
832
+
in which case this file can be used to back swapspace.
833
834
swap_deactivate: Called during swapoff on files where swap_activate
835
was successful.
+1
-1
Documentation/input/input.rst
+1
-1
Documentation/input/input.rst
-1
Documentation/input/joydev/index.rst
-1
Documentation/input/joydev/index.rst
+3
-3
Documentation/kbuild/makefiles.txt
+3
-3
Documentation/kbuild/makefiles.txt
···
297
ccflags-y specifies options for compiling with $(CC).
298
299
Example:
300
-
# drivers/acpi/Makefile
301
-
ccflags-y := -Os
302
-
ccflags-$(CONFIG_ACPI_DEBUG) += -DACPI_DEBUG_OUTPUT
303
304
This variable is necessary because the top Makefile owns the
305
variable $(KBUILD_CFLAGS) and uses it for compilation flags for the
···
297
ccflags-y specifies options for compiling with $(CC).
298
299
Example:
300
+
# drivers/acpi/acpica/Makefile
301
+
ccflags-y := -Os -D_LINUX -DBUILDING_ACPICA
302
+
ccflags-$(CONFIG_ACPI_DEBUG) += -DACPI_DEBUG_OUTPUT
303
304
This variable is necessary because the top Makefile owns the
305
variable $(KBUILD_CFLAGS) and uses it for compilation flags for the
+102
-324
Documentation/locking/rt-mutex-design.txt
+102
-324
Documentation/locking/rt-mutex-design.txt
···
97
a process being blocked on the mutex, it is fine to allocate
98
the waiter on the process's stack (local variable). This
99
structure holds a pointer to the task, as well as the mutex that
100
-
the task is blocked on. It also has the plist node structures to
101
-
place the task in the waiter_list of a mutex as well as the
102
-
pi_list of a mutex owner task (described below).
103
104
waiter is sometimes used in reference to the task that is waiting
105
on a mutex. This is the same as waiter->task.
···
179
|
180
F->L5-+
181
182
183
-
Plist
184
-
-----
185
-
186
-
Before I go further and talk about how the PI chain is stored through lists
187
-
on both mutexes and processes, I'll explain the plist. This is similar to
188
-
the struct list_head functionality that is already in the kernel.
189
-
The implementation of plist is out of scope for this document, but it is
190
-
very important to understand what it does.
191
-
192
-
There are a few differences between plist and list, the most important one
193
-
being that plist is a priority sorted linked list. This means that the
194
-
priorities of the plist are sorted, such that it takes O(1) to retrieve the
195
-
highest priority item in the list. Obviously this is useful to store processes
196
-
based on their priorities.
197
-
198
-
Another difference, which is important for implementation, is that, unlike
199
-
list, the head of the list is a different element than the nodes of a list.
200
-
So the head of the list is declared as struct plist_head and nodes that will
201
-
be added to the list are declared as struct plist_node.
202
-
203
-
204
-
Mutex Waiter List
205
-----------------
206
207
-
Every mutex keeps track of all the waiters that are blocked on itself. The mutex
208
-
has a plist to store these waiters by priority. This list is protected by
209
-
a spin lock that is located in the struct of the mutex. This lock is called
210
-
wait_lock. Since the modification of the waiter list is never done in
211
-
interrupt context, the wait_lock can be taken without disabling interrupts.
212
213
214
-
Task PI List
215
------------
216
217
-
To keep track of the PI chains, each process has its own PI list. This is
218
-
a list of all top waiters of the mutexes that are owned by the process.
219
-
Note that this list only holds the top waiters and not all waiters that are
220
blocked on mutexes owned by the process.
221
222
-
The top of the task's PI list is always the highest priority task that
223
is waiting on a mutex that is owned by the task. So if the task has
224
inherited a priority, it will always be the priority of the task that is
225
-
at the top of this list.
226
227
-
This list is stored in the task structure of a process as a plist called
228
-
pi_list. This list is protected by a spin lock also in the task structure,
229
called pi_lock. This lock may also be taken in interrupt context, so when
230
locking the pi_lock, interrupts must be disabled.
231
···
293
294
The mutex structure contains a pointer to the owner of the mutex. If the
295
mutex is not owned, this owner is set to NULL. Since all architectures
296
-
have the task structure on at least a four byte alignment (and if this is
297
-
not true, the rtmutex.c code will be broken!), this allows for the two
298
-
least significant bits to be used as flags. This part is also described
299
-
in Documentation/rt-mutex.txt, but will also be briefly described here.
300
301
-
Bit 0 is used as the "Pending Owner" flag. This is described later.
302
-
Bit 1 is used as the "Has Waiters" flags. This is also described later
303
-
in more detail, but is set whenever there are waiters on a mutex.
304
-
305
306
cmpxchg Tricks
307
--------------
···
337
--------------------
338
339
The implementation of the PI code in rtmutex.c has several places that a
340
-
process must adjust its priority. With the help of the pi_list of a
341
process this is rather easy to know what needs to be adjusted.
342
343
-
The functions implementing the task adjustments are rt_mutex_adjust_prio,
344
-
__rt_mutex_adjust_prio (same as the former, but expects the task pi_lock
345
-
to already be taken), rt_mutex_getprio, and rt_mutex_setprio.
346
347
-
rt_mutex_getprio and rt_mutex_setprio are only used in __rt_mutex_adjust_prio.
348
349
-
rt_mutex_getprio returns the priority that the task should have. Either the
350
-
task's own normal priority, or if a process of a higher priority is waiting on
351
-
a mutex owned by the task, then that higher priority should be returned.
352
-
Since the pi_list of a task holds an order by priority list of all the top
353
-
waiters of all the mutexes that the task owns, rt_mutex_getprio simply needs
354
-
to compare the top pi waiter to its own normal priority, and return the higher
355
-
priority back.
356
357
-
(Note: if looking at the code, you will notice that the lower number of
358
-
prio is returned. This is because the prio field in the task structure
359
-
is an inverse order of the actual priority. So a "prio" of 5 is
360
-
of higher priority than a "prio" of 10.)
361
-
362
-
__rt_mutex_adjust_prio examines the result of rt_mutex_getprio, and if the
363
-
result does not equal the task's current priority, then rt_mutex_setprio
364
-
is called to adjust the priority of the task to the new priority.
365
-
Note that rt_mutex_setprio is defined in kernel/sched/core.c to implement the
366
-
actual change in priority.
367
-
368
-
It is interesting to note that __rt_mutex_adjust_prio can either increase
369
or decrease the priority of the task. In the case that a higher priority
370
-
process has just blocked on a mutex owned by the task, __rt_mutex_adjust_prio
371
would increase/boost the task's priority. But if a higher priority task
372
were for some reason to leave the mutex (timeout or signal), this same function
373
-
would decrease/unboost the priority of the task. That is because the pi_list
374
always contains the highest priority task that is waiting on a mutex owned
375
by the task, so we only need to compare the priority of that top pi waiter
376
to the normal priority of the given task.
···
381
382
rt_mutex_adjust_prio_chain is called with a task to be checked for PI
383
(de)boosting (the owner of a mutex that a process is blocking on), a flag to
384
-
check for deadlocking, the mutex that the task owns, and a pointer to a waiter
385
that is the process's waiter struct that is blocked on the mutex (although this
386
-
parameter may be NULL for deboosting).
387
388
For this explanation, I will not mention deadlock detection. This explanation
389
will try to stay at a high level.
···
394
395
Before this function is called, the task has already had rt_mutex_adjust_prio
396
performed on it. This means that the task is set to the priority that it
397
-
should be at, but the plist nodes of the task's waiter have not been updated
398
-
with the new priorities, and that this task may not be in the proper locations
399
-
in the pi_lists and wait_lists that the task is blocked on. This function
400
solves all that.
401
402
-
A loop is entered, where task is the owner to be checked for PI changes that
403
-
was passed by parameter (for the first iteration). The pi_lock of this task is
404
-
taken to prevent any more changes to the pi_list of the task. This also
405
-
prevents new tasks from completing the blocking on a mutex that is owned by this
406
-
task.
407
-
408
-
If the task is not blocked on a mutex then the loop is exited. We are at
409
-
the top of the PI chain.
410
-
411
-
A check is now done to see if the original waiter (the process that is blocked
412
-
on the current mutex) is the top pi waiter of the task. That is, is this
413
-
waiter on the top of the task's pi_list. If it is not, it either means that
414
-
there is another process higher in priority that is blocked on one of the
415
-
mutexes that the task owns, or that the waiter has just woken up via a signal
416
-
or timeout and has left the PI chain. In either case, the loop is exited, since
417
-
we don't need to do any more changes to the priority of the current task, or any
418
-
task that owns a mutex that this current task is waiting on. A priority chain
419
-
walk is only needed when a new top pi waiter is made to a task.
420
-
421
-
The next check sees if the task's waiter plist node has the priority equal to
422
-
the priority the task is set at. If they are equal, then we are done with
423
-
the loop. Remember that the function started with the priority of the
424
-
task adjusted, but the plist nodes that hold the task in other processes
425
-
pi_lists have not been adjusted.
426
-
427
-
Next, we look at the mutex that the task is blocked on. The mutex's wait_lock
428
-
is taken. This is done by a spin_trylock, because the locking order of the
429
-
pi_lock and wait_lock goes in the opposite direction. If we fail to grab the
430
-
lock, the pi_lock is released, and we restart the loop.
431
-
432
-
Now that we have both the pi_lock of the task as well as the wait_lock of
433
-
the mutex the task is blocked on, we update the task's waiter's plist node
434
-
that is located on the mutex's wait_list.
435
-
436
-
Now we release the pi_lock of the task.
437
-
438
-
Next the owner of the mutex has its pi_lock taken, so we can update the
439
-
task's entry in the owner's pi_list. If the task is the highest priority
440
-
process on the mutex's wait_list, then we remove the previous top waiter
441
-
from the owner's pi_list, and replace it with the task.
442
-
443
-
Note: It is possible that the task was the current top waiter on the mutex,
444
-
in which case the task is not yet on the pi_list of the waiter. This
445
-
is OK, since plist_del does nothing if the plist node is not on any
446
-
list.
447
-
448
-
If the task was not the top waiter of the mutex, but it was before we
449
-
did the priority updates, that means we are deboosting/lowering the
450
-
task. In this case, the task is removed from the pi_list of the owner,
451
-
and the new top waiter is added.
452
-
453
-
Lastly, we unlock both the pi_lock of the task, as well as the mutex's
454
-
wait_lock, and continue the loop again. On the next iteration of the
455
-
loop, the previous owner of the mutex will be the task that will be
456
-
processed.
457
-
458
-
Note: One might think that the owner of this mutex might have changed
459
-
since we just grab the mutex's wait_lock. And one could be right.
460
-
The important thing to remember is that the owner could not have
461
-
become the task that is being processed in the PI chain, since
462
-
we have taken that task's pi_lock at the beginning of the loop.
463
-
So as long as there is an owner of this mutex that is not the same
464
-
process as the tasked being worked on, we are OK.
465
-
466
-
Looking closely at the code, one might be confused. The check for the
467
-
end of the PI chain is when the task isn't blocked on anything or the
468
-
task's waiter structure "task" element is NULL. This check is
469
-
protected only by the task's pi_lock. But the code to unlock the mutex
470
-
sets the task's waiter structure "task" element to NULL with only
471
-
the protection of the mutex's wait_lock, which was not taken yet.
472
-
Isn't this a race condition if the task becomes the new owner?
473
-
474
-
The answer is No! The trick is the spin_trylock of the mutex's
475
-
wait_lock. If we fail that lock, we release the pi_lock of the
476
-
task and continue the loop, doing the end of PI chain check again.
477
-
478
-
In the code to release the lock, the wait_lock of the mutex is held
479
-
the entire time, and it is not let go when we grab the pi_lock of the
480
-
new owner of the mutex. So if the switch of a new owner were to happen
481
-
after the check for end of the PI chain and the grabbing of the
482
-
wait_lock, the unlocking code would spin on the new owner's pi_lock
483
-
but never give up the wait_lock. So the PI chain loop is guaranteed to
484
-
fail the spin_trylock on the wait_lock, release the pi_lock, and
485
-
try again.
486
-
487
-
If you don't quite understand the above, that's OK. You don't have to,
488
-
unless you really want to make a proof out of it ;)
489
-
490
-
491
-
Pending Owners and Lock stealing
492
-
--------------------------------
493
-
494
-
One of the flags in the owner field of the mutex structure is "Pending Owner".
495
-
What this means is that an owner was chosen by the process releasing the
496
-
mutex, but that owner has yet to wake up and actually take the mutex.
497
-
498
-
Why is this important? Why can't we just give the mutex to another process
499
-
and be done with it?
500
-
501
-
The PI code is to help with real-time processes, and to let the highest
502
-
priority process run as long as possible with little latencies and delays.
503
-
If a high priority process owns a mutex that a lower priority process is
504
-
blocked on, when the mutex is released it would be given to the lower priority
505
-
process. What if the higher priority process wants to take that mutex again.
506
-
The high priority process would fail to take that mutex that it just gave up
507
-
and it would need to boost the lower priority process to run with full
508
-
latency of that critical section (since the low priority process just entered
509
-
it).
510
-
511
-
There's no reason a high priority process that gives up a mutex should be
512
-
penalized if it tries to take that mutex again. If the new owner of the
513
-
mutex has not woken up yet, there's no reason that the higher priority process
514
-
could not take that mutex away.
515
-
516
-
To solve this, we introduced Pending Ownership and Lock Stealing. When a
517
-
new process is given a mutex that it was blocked on, it is only given
518
-
pending ownership. This means that it's the new owner, unless a higher
519
-
priority process comes in and tries to grab that mutex. If a higher priority
520
-
process does come along and wants that mutex, we let the higher priority
521
-
process "steal" the mutex from the pending owner (only if it is still pending)
522
-
and continue with the mutex.
523
-
524
525
Taking of a mutex (The walk through)
526
------------------------------------
···
414
fails). Only when the owner field of the mutex is NULL can the lock be
415
taken with the CMPXCHG and nothing else needs to be done.
416
417
-
If there is contention on the lock, whether it is owned or pending owner
418
-
we go about the slow path (rt_mutex_slowlock).
419
420
The slow path function is where the task's waiter structure is created on
421
the stack. This is because the waiter structure is only needed for the
422
scope of this function. The waiter structure holds the nodes to store
423
-
the task on the wait_list of the mutex, and if need be, the pi_list of
424
-
the owner.
425
426
The wait_lock of the mutex is taken since the slow path of unlocking the
427
mutex also takes this lock.
···
432
433
try_to_take_rt_mutex is used every time the task tries to grab a mutex in the
434
slow path. The first thing that is done here is an atomic setting of
435
-
the "Has Waiters" flag of the mutex's owner field. Yes, this could really
436
-
be false, because if the mutex has no owner, there are no waiters and
437
-
the current task also won't have any waiters. But we don't have the lock
438
-
yet, so we assume we are going to be a waiter. The reason for this is to
439
-
play nice for those architectures that do have CMPXCHG. By setting this flag
440
-
now, the owner of the mutex can't release the mutex without going into the
441
-
slow unlock path, and it would then need to grab the wait_lock, which this
442
-
code currently holds. So setting the "Has Waiters" flag forces the owner
443
-
to synchronize with this code.
444
445
-
Now that we know that we can't have any races with the owner releasing the
446
-
mutex, we check to see if we can take the ownership. This is done if the
447
-
mutex doesn't have a owner, or if we can steal the mutex from a pending
448
-
owner. Let's look at the situations we have here.
449
450
-
1) Has owner that is pending
451
-
----------------------------
452
453
-
The mutex has a owner, but it hasn't woken up and the mutex flag
454
-
"Pending Owner" is set. The first check is to see if the owner isn't the
455
-
current task. This is because this function is also used for the pending
456
-
owner to grab the mutex. When a pending owner wakes up, it checks to see
457
-
if it can take the mutex, and this is done if the owner is already set to
458
-
itself. If so, we succeed and leave the function, clearing the "Pending
459
-
Owner" bit.
460
-
461
-
If the pending owner is not current, we check to see if the current priority is
462
-
higher than the pending owner. If not, we fail the function and return.
463
-
464
-
There's also something special about a pending owner. That is a pending owner
465
-
is never blocked on a mutex. So there is no PI chain to worry about. It also
466
-
means that if the mutex doesn't have any waiters, there's no accounting needed
467
-
to update the pending owner's pi_list, since we only worry about processes
468
-
blocked on the current mutex.
469
-
470
-
If there are waiters on this mutex, and we just stole the ownership, we need
471
-
to take the top waiter, remove it from the pi_list of the pending owner, and
472
-
add it to the current pi_list. Note that at this moment, the pending owner
473
-
is no longer on the list of waiters. This is fine, since the pending owner
474
-
would add itself back when it realizes that it had the ownership stolen
475
-
from itself. When the pending owner tries to grab the mutex, it will fail
476
-
in try_to_take_rt_mutex if the owner field points to another process.
477
-
478
-
2) No owner
479
-
-----------
480
-
481
-
If there is no owner (or we successfully stole the lock), we set the owner
482
-
of the mutex to current, and set the flag of "Has Waiters" if the current
483
-
mutex actually has waiters, or we clear the flag if it doesn't. See, it was
484
-
OK that we set that flag early, since now it is cleared.
485
-
486
-
3) Failed to grab ownership
487
-
---------------------------
488
-
489
-
The most interesting case is when we fail to take ownership. This means that
490
-
there exists an owner, or there's a pending owner with equal or higher
491
-
priority than the current task.
492
-
493
-
We'll continue on the failed case.
494
-
495
-
If the mutex has a timeout, we set up a timer to go off to break us out
496
-
of this mutex if we failed to get it after a specified amount of time.
497
-
498
-
Now we enter a loop that will continue to try to take ownership of the mutex, or
499
-
fail from a timeout or signal.
500
-
501
-
Once again we try to take the mutex. This will usually fail the first time
502
-
in the loop, since it had just failed to get the mutex. But the second time
503
-
in the loop, this would likely succeed, since the task would likely be
504
-
the pending owner.
505
-
506
-
If the mutex is TASK_INTERRUPTIBLE a check for signals and timeout is done
507
-
here.
508
-
509
-
The waiter structure has a "task" field that points to the task that is blocked
510
-
on the mutex. This field can be NULL the first time it goes through the loop
511
-
or if the task is a pending owner and had its mutex stolen. If the "task"
512
-
field is NULL then we need to set up the accounting for it.
513
514
Task blocks on mutex
515
--------------------
516
517
The accounting of a mutex and process is done with the waiter structure of
518
the process. The "task" field is set to the process, and the "lock" field
519
-
to the mutex. The plist nodes are initialized to the processes current
520
-
priority.
521
522
Since the wait_lock was taken at the entry of the slow lock, we can safely
523
-
add the waiter to the wait_list. If the current process is the highest
524
-
priority process currently waiting on this mutex, then we remove the
525
-
previous top waiter process (if it exists) from the pi_list of the owner,
526
-
and add the current process to that list. Since the pi_list of the owner
527
has changed, we call rt_mutex_adjust_prio on the owner to see if the owner
528
should adjust its priority accordingly.
529
530
-
If the owner is also blocked on a lock, and had its pi_list changed
531
(or deadlock checking is on), we unlock the wait_lock of the mutex and go ahead
532
and run rt_mutex_adjust_prio_chain on the owner, as described earlier.
533
···
480
Waking up in the loop
481
---------------------
482
483
-
The schedule can then wake up for a few reasons.
484
-
1) we were given pending ownership of the mutex.
485
-
2) we received a signal and was TASK_INTERRUPTIBLE
486
-
3) we had a timeout and was TASK_INTERRUPTIBLE
487
488
-
In any of these cases, we continue the loop and once again try to grab the
489
-
ownership of the mutex. If we succeed, we exit the loop, otherwise we continue
490
-
and on signal and timeout, will exit the loop, or if we had the mutex stolen
491
-
we just simply add ourselves back on the lists and go back to sleep.
492
493
-
Note: For various reasons, because of timeout and signals, the steal mutex
494
-
algorithm needs to be careful. This is because the current process is
495
-
still on the wait_list. And because of dynamic changing of priorities,
496
-
especially on SCHED_OTHER tasks, the current process can be the
497
-
highest priority task on the wait_list.
498
499
-
Failed to get mutex on Timeout or Signal
500
-
----------------------------------------
501
-
502
-
If a timeout or signal occurred, the waiter's "task" field would not be
503
-
NULL and the task needs to be taken off the wait_list of the mutex and perhaps
504
-
pi_list of the owner. If this process was a high priority process, then
505
-
the rt_mutex_adjust_prio_chain needs to be executed again on the owner,
506
-
but this time it will be lowering the priorities.
507
508
509
Unlocking the Mutex
···
526
owner field is set to NULL, the wait_lock is released and nothing more is
527
needed.
528
529
-
If there are waiters, then we need to wake one up and give that waiter
530
-
pending ownership.
531
532
On the wake up code, the pi_lock of the current owner is taken. The top
533
-
waiter of the lock is found and removed from the wait_list of the mutex
534
-
as well as the pi_list of the current owner. The task field of the new
535
-
pending owner's waiter structure is set to NULL, and the owner field of the
536
-
mutex is set to the new owner with the "Pending Owner" bit set, as well
537
-
as the "Has Waiters" bit if there still are other processes blocked on the
538
-
mutex.
539
-
540
-
The pi_lock of the previous owner is released, and the new pending owner's
541
-
pi_lock is taken. Remember that this is the trick to prevent the race
542
-
condition in rt_mutex_adjust_prio_chain from adding itself as a waiter
543
-
on the mutex.
544
-
545
-
We now clear the "pi_blocked_on" field of the new pending owner, and if
546
-
the mutex still has waiters pending, we add the new top waiter to the pi_list
547
-
of the pending owner.
548
549
Finally we unlock the pi_lock of the pending owner and wake it up.
550
···
546
-------
547
548
Author: Steven Rostedt <rostedt@goodmis.org>
549
550
-
Reviewers: Ingo Molnar, Thomas Gleixner, Thomas Duetsch, and Randy Dunlap
551
552
Updates
553
-------
554
555
This document was originally written for 2.6.17-rc3-mm1
···
97
a process being blocked on the mutex, it is fine to allocate
98
the waiter on the process's stack (local variable). This
99
structure holds a pointer to the task, as well as the mutex that
100
+
the task is blocked on. It also has rbtree node structures to
101
+
place the task in the waiters rbtree of a mutex as well as the
102
+
pi_waiters rbtree of a mutex owner task (described below).
103
104
waiter is sometimes used in reference to the task that is waiting
105
on a mutex. This is the same as waiter->task.
···
179
|
180
F->L5-+
181
182
+
If process G has the highest priority in the chain, then all the tasks up
183
+
the chain (A and B in this example), must have their priorities increased
184
+
to that of G.
185
186
+
Mutex Waiters Tree
187
-----------------
188
189
+
Every mutex keeps track of all the waiters that are blocked on itself. The
190
+
mutex has a rbtree to store these waiters by priority. This tree is protected
191
+
by a spin lock that is located in the struct of the mutex. This lock is called
192
+
wait_lock.
193
194
195
+
Task PI Tree
196
------------
197
198
+
To keep track of the PI chains, each process has its own PI rbtree. This is
199
+
a tree of all top waiters of the mutexes that are owned by the process.
200
+
Note that this tree only holds the top waiters and not all waiters that are
201
blocked on mutexes owned by the process.
202
203
+
The top of the task's PI tree is always the highest priority task that
204
is waiting on a mutex that is owned by the task. So if the task has
205
inherited a priority, it will always be the priority of the task that is
206
+
at the top of this tree.
207
208
+
This tree is stored in the task structure of a process as a rbtree called
209
+
pi_waiters. It is protected by a spin lock also in the task structure,
210
called pi_lock. This lock may also be taken in interrupt context, so when
211
locking the pi_lock, interrupts must be disabled.
212
···
312
313
The mutex structure contains a pointer to the owner of the mutex. If the
314
mutex is not owned, this owner is set to NULL. Since all architectures
315
+
have the task structure on at least a two byte alignment (and if this is
316
+
not true, the rtmutex.c code will be broken!), this allows for the least
317
+
significant bit to be used as a flag. Bit 0 is used as the "Has Waiters"
318
+
flag. It's set whenever there are waiters on a mutex.
319
320
+
See Documentation/locking/rt-mutex.txt for further details.
321
322
cmpxchg Tricks
323
--------------
···
359
--------------------
360
361
The implementation of the PI code in rtmutex.c has several places that a
362
+
process must adjust its priority. With the help of the pi_waiters of a
363
process this is rather easy to know what needs to be adjusted.
364
365
+
The functions implementing the task adjustments are rt_mutex_adjust_prio
366
+
and rt_mutex_setprio. rt_mutex_setprio is only used in rt_mutex_adjust_prio.
367
368
+
rt_mutex_adjust_prio examines the priority of the task, and the highest
369
+
priority process that is waiting any of mutexes owned by the task. Since
370
+
the pi_waiters of a task holds an order by priority of all the top waiters
371
+
of all the mutexes that the task owns, we simply need to compare the top
372
+
pi waiter to its own normal/deadline priority and take the higher one.
373
+
Then rt_mutex_setprio is called to adjust the priority of the task to the
374
+
new priority. Note that rt_mutex_setprio is defined in kernel/sched/core.c
375
+
to implement the actual change in priority.
376
377
+
(Note: For the "prio" field in task_struct, the lower the number, the
378
+
higher the priority. A "prio" of 5 is of higher priority than a
379
+
"prio" of 10.)
380
381
+
It is interesting to note that rt_mutex_adjust_prio can either increase
382
or decrease the priority of the task. In the case that a higher priority
383
+
process has just blocked on a mutex owned by the task, rt_mutex_adjust_prio
384
would increase/boost the task's priority. But if a higher priority task
385
were for some reason to leave the mutex (timeout or signal), this same function
386
+
would decrease/unboost the priority of the task. That is because the pi_waiters
387
always contains the highest priority task that is waiting on a mutex owned
388
by the task, so we only need to compare the priority of that top pi waiter
389
to the normal priority of the given task.
···
412
413
rt_mutex_adjust_prio_chain is called with a task to be checked for PI
414
(de)boosting (the owner of a mutex that a process is blocking on), a flag to
415
+
check for deadlocking, the mutex that the task owns, a pointer to a waiter
416
that is the process's waiter struct that is blocked on the mutex (although this
417
+
parameter may be NULL for deboosting), a pointer to the mutex on which the task
418
+
is blocked, and a top_task as the top waiter of the mutex.
419
420
For this explanation, I will not mention deadlock detection. This explanation
421
will try to stay at a high level.
···
424
425
Before this function is called, the task has already had rt_mutex_adjust_prio
426
performed on it. This means that the task is set to the priority that it
427
+
should be at, but the rbtree nodes of the task's waiter have not been updated
428
+
with the new priorities, and this task may not be in the proper locations
429
+
in the pi_waiters and waiters trees that the task is blocked on. This function
430
solves all that.
431
432
+
The main operation of this function is summarized by Thomas Gleixner in
433
+
rtmutex.c. See the 'Chain walk basics and protection scope' comment for further
434
+
details.
435
436
Taking of a mutex (The walk through)
437
------------------------------------
···
563
fails). Only when the owner field of the mutex is NULL can the lock be
564
taken with the CMPXCHG and nothing else needs to be done.
565
566
+
If there is contention on the lock, we go about the slow path
567
+
(rt_mutex_slowlock).
568
569
The slow path function is where the task's waiter structure is created on
570
the stack. This is because the waiter structure is only needed for the
571
scope of this function. The waiter structure holds the nodes to store
572
+
the task on the waiters tree of the mutex, and if need be, the pi_waiters
573
+
tree of the owner.
574
575
The wait_lock of the mutex is taken since the slow path of unlocking the
576
mutex also takes this lock.
···
581
582
try_to_take_rt_mutex is used every time the task tries to grab a mutex in the
583
slow path. The first thing that is done here is an atomic setting of
584
+
the "Has Waiters" flag of the mutex's owner field. By setting this flag
585
+
now, the current owner of the mutex being contended for can't release the mutex
586
+
without going into the slow unlock path, and it would then need to grab the
587
+
wait_lock, which this code currently holds. So setting the "Has Waiters" flag
588
+
forces the current owner to synchronize with this code.
589
590
+
The lock is taken if the following are true:
591
+
1) The lock has no owner
592
+
2) The current task is the highest priority against all other
593
+
waiters of the lock
594
595
+
If the task succeeds to acquire the lock, then the task is set as the
596
+
owner of the lock, and if the lock still has waiters, the top_waiter
597
+
(highest priority task waiting on the lock) is added to this task's
598
+
pi_waiters tree.
599
600
+
If the lock is not taken by try_to_take_rt_mutex(), then the
601
+
task_blocks_on_rt_mutex() function is called. This will add the task to
602
+
the lock's waiter tree and propagate the pi chain of the lock as well
603
+
as the lock's owner's pi_waiters tree. This is described in the next
604
+
section.
605
606
Task blocks on mutex
607
--------------------
608
609
The accounting of a mutex and process is done with the waiter structure of
610
the process. The "task" field is set to the process, and the "lock" field
611
+
to the mutex. The rbtree node of waiter are initialized to the processes
612
+
current priority.
613
614
Since the wait_lock was taken at the entry of the slow lock, we can safely
615
+
add the waiter to the task waiter tree. If the current process is the
616
+
highest priority process currently waiting on this mutex, then we remove the
617
+
previous top waiter process (if it exists) from the pi_waiters of the owner,
618
+
and add the current process to that tree. Since the pi_waiter of the owner
619
has changed, we call rt_mutex_adjust_prio on the owner to see if the owner
620
should adjust its priority accordingly.
621
622
+
If the owner is also blocked on a lock, and had its pi_waiters changed
623
(or deadlock checking is on), we unlock the wait_lock of the mutex and go ahead
624
and run rt_mutex_adjust_prio_chain on the owner, as described earlier.
625
···
686
Waking up in the loop
687
---------------------
688
689
+
The task can then wake up for a couple of reasons:
690
+
1) The previous lock owner released the lock, and the task now is top_waiter
691
+
2) we received a signal or timeout
692
693
+
In both cases, the task will try again to acquire the lock. If it
694
+
does, then it will take itself off the waiters tree and set itself back
695
+
to the TASK_RUNNING state.
696
697
+
In first case, if the lock was acquired by another task before this task
698
+
could get the lock, then it will go back to sleep and wait to be woken again.
699
700
+
The second case is only applicable for tasks that are grabbing a mutex
701
+
that can wake up before getting the lock, either due to a signal or
702
+
a timeout (i.e. rt_mutex_timed_futex_lock()). When woken, it will try to
703
+
take the lock again, if it succeeds, then the task will return with the
704
+
lock held, otherwise it will return with -EINTR if the task was woken
705
+
by a signal, or -ETIMEDOUT if it timed out.
706
707
708
Unlocking the Mutex
···
739
owner field is set to NULL, the wait_lock is released and nothing more is
740
needed.
741
742
+
If there are waiters, then we need to wake one up.
743
744
On the wake up code, the pi_lock of the current owner is taken. The top
745
+
waiter of the lock is found and removed from the waiters tree of the mutex
746
+
as well as the pi_waiters tree of the current owner. The "Has Waiters" bit is
747
+
marked to prevent lower priority tasks from stealing the lock.
748
749
Finally we unlock the pi_lock of the pending owner and wake it up.
750
···
772
-------
773
774
Author: Steven Rostedt <rostedt@goodmis.org>
775
+
Updated: Alex Shi <alex.shi@linaro.org> - 7/6/2017
776
777
+
Original Reviewers: Ingo Molnar, Thomas Gleixner, Thomas Duetsch, and
778
+
Randy Dunlap
779
+
Update (7/6/2017) Reviewers: Steven Rostedt and Sebastian Siewior
780
781
Updates
782
-------
783
784
This document was originally written for 2.6.17-rc3-mm1
785
+
was updated on 4.12
+26
-32
Documentation/locking/rt-mutex.txt
+26
-32
Documentation/locking/rt-mutex.txt
···
28
well-designed applications to use userspace locks in critical parts of
29
an high priority thread, without losing determinism.
30
31
-
The enqueueing of the waiters into the rtmutex waiter list is done in
32
priority order. For same priorities FIFO order is chosen. For each
33
rtmutex, only the top priority waiter is enqueued into the owner's
34
-
priority waiters list. This list too queues in priority order. Whenever
35
the top priority waiter of a task changes (for example it timed out or
36
-
got a signal), the priority of the owner task is readjusted. [The
37
-
priority enqueueing is handled by "plists", see include/linux/plist.h
38
-
for more details.]
39
40
RT-mutexes are optimized for fastpath operations and have no internal
41
locking overhead when locking an uncontended mutex or unlocking a mutex
···
45
The state of the rt-mutex is tracked via the owner field of the rt-mutex
46
structure:
47
48
-
rt_mutex->owner holds the task_struct pointer of the owner. Bit 0 and 1
49
-
are used to keep track of the "owner is pending" and "rtmutex has
50
-
waiters" state.
51
52
-
owner bit1 bit0
53
-
NULL 0 0 mutex is free (fast acquire possible)
54
-
NULL 0 1 invalid state
55
-
NULL 1 0 Transitional state*
56
-
NULL 1 1 invalid state
57
-
taskpointer 0 0 mutex is held (fast release possible)
58
-
taskpointer 0 1 task is pending owner
59
-
taskpointer 1 0 mutex is held and has waiters
60
-
taskpointer 1 1 task is pending owner and mutex has waiters
61
62
-
Pending-ownership handling is a performance optimization:
63
-
pending-ownership is assigned to the first (highest priority) waiter of
64
-
the mutex, when the mutex is released. The thread is woken up and once
65
-
it starts executing it can acquire the mutex. Until the mutex is taken
66
-
by it (bit 0 is cleared) a competing higher priority thread can "steal"
67
-
the mutex which puts the woken up thread back on the waiters list.
68
69
-
The pending-ownership optimization is especially important for the
70
-
uninterrupted workflow of high-prio tasks which repeatedly
71
-
takes/releases locks that have lower-prio waiters. Without this
72
-
optimization the higher-prio thread would ping-pong to the lower-prio
73
-
task [because at unlock time we always assign a new owner].
74
75
-
(*) The "mutex has waiters" bit gets set to take the lock. If the lock
76
-
doesn't already have an owner, this bit is quickly cleared if there are
77
-
no waiters. So this is a transitional state to synchronize with looking
78
-
at the owner field of the mutex and the mutex owner releasing the lock.
···
28
well-designed applications to use userspace locks in critical parts of
29
an high priority thread, without losing determinism.
30
31
+
The enqueueing of the waiters into the rtmutex waiter tree is done in
32
priority order. For same priorities FIFO order is chosen. For each
33
rtmutex, only the top priority waiter is enqueued into the owner's
34
+
priority waiters tree. This tree too queues in priority order. Whenever
35
the top priority waiter of a task changes (for example it timed out or
36
+
got a signal), the priority of the owner task is readjusted. The
37
+
priority enqueueing is handled by "pi_waiters".
38
39
RT-mutexes are optimized for fastpath operations and have no internal
40
locking overhead when locking an uncontended mutex or unlocking a mutex
···
46
The state of the rt-mutex is tracked via the owner field of the rt-mutex
47
structure:
48
49
+
lock->owner holds the task_struct pointer of the owner. Bit 0 is used to
50
+
keep track of the "lock has waiters" state.
51
52
+
owner bit0
53
+
NULL 0 lock is free (fast acquire possible)
54
+
NULL 1 lock is free and has waiters and the top waiter
55
+
is going to take the lock*
56
+
taskpointer 0 lock is held (fast release possible)
57
+
taskpointer 1 lock is held and has waiters**
58
59
+
The fast atomic compare exchange based acquire and release is only
60
+
possible when bit 0 of lock->owner is 0.
61
62
+
(*) It also can be a transitional state when grabbing the lock
63
+
with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
64
+
we need to set the bit0 before looking at the lock, and the owner may be
65
+
NULL in this small time, hence this can be a transitional state.
66
67
+
(**) There is a small time when bit 0 is set but there are no
68
+
waiters. This can happen when grabbing the lock in the slow path.
69
+
To prevent a cmpxchg of the owner releasing the lock, we need to
70
+
set this bit before looking at the lock.
71
+
72
+
BTW, there is still technically a "Pending Owner", it's just not called
73
+
that anymore. The pending owner happens to be the top_waiter of a lock
74
+
that has no owner and has been woken up to grab the lock.
-1
Documentation/media/uapi/cec/cec-funcs.rst
-1
Documentation/media/uapi/cec/cec-funcs.rst
+8
-8
Documentation/networking/ieee802154.txt
+8
-8
Documentation/networking/ieee802154.txt
···
84
==================
85
86
The include/net/mac802154.h defines following functions:
87
-
- struct ieee802154_dev *ieee802154_alloc_device
88
-
(size_t priv_size, struct ieee802154_ops *ops):
89
-
allocation of IEEE 802.15.4 compatible device
90
91
-
- void ieee802154_free_device(struct ieee802154_dev *dev):
92
-
freeing allocated device
93
94
-
- int ieee802154_register_device(struct ieee802154_dev *dev):
95
-
register PHY in the system
96
97
-
- void ieee802154_unregister_device(struct ieee802154_dev *dev):
98
freeing registered PHY
99
100
Moreover IEEE 802.15.4 device operations structure should be filled.
···
84
==================
85
86
The include/net/mac802154.h defines following functions:
87
+
- struct ieee802154_hw *
88
+
ieee802154_alloc_hw(size_t priv_data_len, const struct ieee802154_ops *ops):
89
+
allocation of IEEE 802.15.4 compatible hardware device
90
91
+
- void ieee802154_free_hw(struct ieee802154_hw *hw):
92
+
freeing allocated hardware device
93
94
+
- int ieee802154_register_hw(struct ieee802154_hw *hw):
95
+
register PHY which is the allocated hardware device, in the system
96
97
+
- void ieee802154_unregister_hw(struct ieee802154_hw *hw):
98
freeing registered PHY
99
100
Moreover IEEE 802.15.4 device operations structure should be filled.
+1
-1
Documentation/nvmem/nvmem.txt
+1
-1
Documentation/nvmem/nvmem.txt
···
112
5. Releasing a reference to the NVMEM
113
=====================================
114
115
-
When a consumers no longer needs the NVMEM, it has to release the reference
116
to the NVMEM it has obtained using the APIs mentioned in the above section.
117
The NVMEM framework provides 2 APIs to release a reference to the NVMEM.
118
···
112
5. Releasing a reference to the NVMEM
113
=====================================
114
115
+
When a consumer no longer needs the NVMEM, it has to release the reference
116
to the NVMEM it has obtained using the APIs mentioned in the above section.
117
The NVMEM framework provides 2 APIs to release a reference to the NVMEM.
118
+1
-42
Documentation/process/applying-patches.rst
+1
-42
Documentation/process/applying-patches.rst
···
6
Original by:
7
Jesper Juhl, August 2005
8
9
-
Last update:
10
-
2016-09-14
11
-
12
.. note::
13
14
This document is obsolete. In most cases, rather than using ``patch``
···
341
342
This is a good branch to run for people who want to help out testing
343
development kernels but do not want to run some of the really experimental
344
-
stuff (such people should see the sections about -git and -mm kernels below).
345
346
The -rc patches are not incremental, they apply to a base 4.x kernel, just
347
like the 4.x.y patches described above. The kernel version before the -rcN
···
375
$ patch -p1 < ../patch-4.8-rc5 # apply new 4.8-rc5 patch
376
$ cd ..
377
$ mv linux-4.7.3 linux-4.8-rc5 # rename the kernel source dir
378
-
379
-
380
-
The -git kernels
381
-
================
382
-
383
-
These are daily snapshots of Linus' kernel tree (managed in a git
384
-
repository, hence the name).
385
-
386
-
These patches are usually released daily and represent the current state of
387
-
Linus's tree. They are more experimental than -rc kernels since they are
388
-
generated automatically without even a cursory glance to see if they are
389
-
sane.
390
-
391
-
-git patches are not incremental and apply either to a base 4.x kernel or
392
-
a base 4.x-rc kernel -- you can see which from their name.
393
-
A patch named 4.7-git1 applies to the 4.7 kernel source and a patch
394
-
named 4.8-rc3-git2 applies to the source of the 4.8-rc3 kernel.
395
-
396
-
Here are some examples of how to apply these patches::
397
-
398
-
# moving from 4.7 to 4.7-git1
399
-
400
-
$ cd ~/linux-4.7 # change to the kernel source dir
401
-
$ patch -p1 < ../patch-4.7-git1 # apply the 4.7-git1 patch
402
-
$ cd ..
403
-
$ mv linux-4.7 linux-4.7-git1 # rename the kernel source dir
404
-
405
-
# moving from 4.7-git1 to 4.8-rc2-git3
406
-
407
-
$ cd ~/linux-4.7-git1 # change to the kernel source dir
408
-
$ patch -p1 -R < ../patch-4.7-git1 # revert the 4.7-git1 patch
409
-
# we now have a 4.7 kernel
410
-
$ patch -p1 < ../patch-4.8-rc2 # apply the 4.8-rc2 patch
411
-
# the kernel is now 4.8-rc2
412
-
$ patch -p1 < ../patch-4.8-rc2-git3 # apply the 4.8-rc2-git3 patch
413
-
# the kernel is now 4.8-rc2-git3
414
-
$ cd ..
415
-
$ mv linux-4.7-git1 linux-4.8-rc2-git3 # rename source dir
416
417
418
The -mm patches and the linux-next tree
···
6
Original by:
7
Jesper Juhl, August 2005
8
9
.. note::
10
11
This document is obsolete. In most cases, rather than using ``patch``
···
344
345
This is a good branch to run for people who want to help out testing
346
development kernels but do not want to run some of the really experimental
347
+
stuff (such people should see the sections about -next and -mm kernels below).
348
349
The -rc patches are not incremental, they apply to a base 4.x kernel, just
350
like the 4.x.y patches described above. The kernel version before the -rcN
···
378
$ patch -p1 < ../patch-4.8-rc5 # apply new 4.8-rc5 patch
379
$ cd ..
380
$ mv linux-4.7.3 linux-4.8-rc5 # rename the kernel source dir
381
382
383
The -mm patches and the linux-next tree
+3
-13
Documentation/process/changes.rst
+3
-13
Documentation/process/changes.rst
···
53
iptables 1.4.2 iptables -V
54
openssl & libcrypto 1.0.0 openssl version
55
bc 1.06.95 bc --version
56
-
Sphinx\ [#f1]_ 1.2 sphinx-build --version
57
====================== =============== ========================================
58
59
.. [#f1] Sphinx is needed only to build the Kernel documentation
···
309
Sphinx
310
------
311
312
-
The ReST markups currently used by the Documentation/ files are meant to be
313
-
built with ``Sphinx`` version 1.2 or upper. If you're desiring to build
314
-
PDF outputs, it is recommended to use version 1.4.6.
315
-
316
-
.. note::
317
-
318
-
Please notice that, for PDF and LaTeX output, you'll also need ``XeLaTeX``
319
-
version 3.14159265. Depending on the distribution, you may also need to
320
-
install a series of ``texlive`` packages that provide the minimal set of
321
-
functionalities required for ``XeLaTex`` to work. For PDF output you'll also
322
-
need ``convert(1)`` from ImageMagick (https://www.imagemagick.org).
323
-
324
325
Getting updated software
326
========================
···
53
iptables 1.4.2 iptables -V
54
openssl & libcrypto 1.0.0 openssl version
55
bc 1.06.95 bc --version
56
+
Sphinx\ [#f1]_ 1.3 sphinx-build --version
57
====================== =============== ========================================
58
59
.. [#f1] Sphinx is needed only to build the Kernel documentation
···
309
Sphinx
310
------
311
312
+
Please see :ref:`sphinx_install` in ``Documentation/doc-guide/sphinx.rst``
313
+
for details about Sphinx requirements.
314
315
Getting updated software
316
========================
+2
-2
Documentation/process/stable-kernel-rules.rst
+2
-2
Documentation/process/stable-kernel-rules.rst
···
166
- The queues of patches, for both completed versions and in progress
167
versions can be found at:
168
169
-
http://git.kernel.org/?p=linux/kernel/git/stable/stable-queue.git
170
171
- The finalized and tagged releases of all stable kernels can be found
172
in separate branches per version at:
173
174
-
http://git.kernel.org/?p=linux/kernel/git/stable/linux-stable.git
175
176
177
Review committee
···
166
- The queues of patches, for both completed versions and in progress
167
versions can be found at:
168
169
+
https://git.kernel.org/pub/scm/linux/kernel/git/stable/stable-queue.git
170
171
- The finalized and tagged releases of all stable kernels can be found
172
in separate branches per version at:
173
174
+
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
175
176
177
Review committee
+1
-1
Documentation/process/submitting-patches.rst
+1
-1
Documentation/process/submitting-patches.rst
+3
-13
Documentation/security/keys/core.rst
+3
-13
Documentation/security/keys/core.rst
···
16
17
This document has the following sections:
18
19
-
- Key overview
20
-
- Key service overview
21
-
- Key access permissions
22
-
- SELinux support
23
-
- New procfs files
24
-
- Userspace system call interface
25
-
- Kernel services
26
-
- Notes on accessing payload contents
27
-
- Defining a key type
28
-
- Request-key callback service
29
-
- Garbage collection
30
31
32
Key Overview
···
433
/sbin/request-key will be invoked in an attempt to obtain a key. The
434
callout_info string will be passed as an argument to the program.
435
436
-
See also Documentation/security/keys-request-key.txt.
437
438
439
The keyctl syscall functions are:
···
963
If successful, the key will have been attached to the default keyring for
964
implicitly obtained request-key keys, as set by KEYCTL_SET_REQKEY_KEYRING.
965
966
-
See also Documentation/security/keys-request-key.txt.
967
968
969
* To search for a key, passing auxiliary data to the upcaller, call::
···
16
17
This document has the following sections:
18
19
+
.. contents:: :local:
20
21
22
Key Overview
···
443
/sbin/request-key will be invoked in an attempt to obtain a key. The
444
callout_info string will be passed as an argument to the program.
445
446
+
See also Documentation/security/keys/request-key.rst.
447
448
449
The keyctl syscall functions are:
···
973
If successful, the key will have been attached to the default keyring for
974
implicitly obtained request-key keys, as set by KEYCTL_SET_REQKEY_KEYRING.
975
976
+
See also Documentation/security/keys/request-key.rst.
977
978
979
* To search for a key, passing auxiliary data to the upcaller, call::
+1
-1
Documentation/security/keys/request-key.rst
+1
-1
Documentation/security/keys/request-key.rst
+1
-1
Documentation/security/keys/trusted-encrypted.rst
+1
-1
Documentation/security/keys/trusted-encrypted.rst
+17
Documentation/sphinx-static/theme_overrides.css
+17
Documentation/sphinx-static/theme_overrides.css
···
4
*
5
*/
6
7
+
/* Interim: Code-blocks with line nos - lines and line numbers don't line up.
8
+
* see: https://github.com/rtfd/sphinx_rtd_theme/issues/419
9
+
*/
10
+
11
+
div[class^="highlight"] pre {
12
+
line-height: normal;
13
+
}
14
+
.rst-content .highlight > pre {
15
+
line-height: normal;
16
+
}
17
+
18
@media screen {
19
20
/* content column
···
54
background-color: darkblue;
55
font-weight: bold;
56
font-family: "Courier New", Courier, monospace
57
+
}
58
+
59
+
/* fix bottom margin of lists items */
60
+
61
+
.rst-content .section ul li:last-child, .rst-content .section ul li p:last-child {
62
+
margin-bottom: 12px;
63
}
64
65
/* inline literal: drop the borderbox, padding and red color */
+3
-5
Documentation/sphinx/kerneldoc.py
+3
-5
Documentation/sphinx/kerneldoc.py
···
27
# Please make sure this works on both python2 and python3.
28
#
29
30
import os
31
import subprocess
32
import sys
···
89
try:
90
env.app.verbose('calling kernel-doc \'%s\'' % (" ".join(cmd)))
91
92
-
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
93
out, err = p.communicate()
94
95
-
# python2 needs conversion to unicode.
96
-
# python3 with universal_newlines=True returns strings.
97
-
if sys.version_info.major < 3:
98
-
out, err = unicode(out, 'utf-8'), unicode(err, 'utf-8')
99
100
if p.returncode != 0:
101
sys.stderr.write(err)
···
27
# Please make sure this works on both python2 and python3.
28
#
29
30
+
import codecs
31
import os
32
import subprocess
33
import sys
···
88
try:
89
env.app.verbose('calling kernel-doc \'%s\'' % (" ".join(cmd)))
90
91
+
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
92
out, err = p.communicate()
93
94
+
out, err = codecs.decode(out, 'utf-8'), codecs.decode(err, 'utf-8')
95
96
if p.returncode != 0:
97
sys.stderr.write(err)
+3
Documentation/sphinx/requirements.txt
+3
Documentation/sphinx/requirements.txt
-2
Documentation/translations/zh_CN/HOWTO
-2
Documentation/translations/zh_CN/HOWTO
+1
-1
Makefile
+1
-1
Makefile
···
1468
1469
# Documentation targets
1470
# ---------------------------------------------------------------------------
1471
-
DOC_TARGETS := xmldocs sgmldocs psdocs latexdocs pdfdocs htmldocs mandocs installmandocs epubdocs cleandocs linkcheckdocs
1472
PHONY += $(DOC_TARGETS)
1473
$(DOC_TARGETS): scripts_basic FORCE
1474
$(Q)$(MAKE) $(build)=Documentation $@
···
1468
1469
# Documentation targets
1470
# ---------------------------------------------------------------------------
1471
+
DOC_TARGETS := xmldocs latexdocs pdfdocs htmldocs epubdocs cleandocs linkcheckdocs
1472
PHONY += $(DOC_TARGETS)
1473
$(DOC_TARGETS): scripts_basic FORCE
1474
$(Q)$(MAKE) $(build)=Documentation $@
+3
-2
include/linux/genalloc.h
+3
-2
include/linux/genalloc.h
···
38
struct gen_pool;
39
40
/**
41
-
* Allocation callback function type definition
42
* @map: Pointer to bitmap
43
* @size: The bitmap size in bits
44
* @start: The bitnumber to start searching at
45
* @nr: The number of zeroed bits we're looking for
46
-
* @data: optional additional data used by @genpool_algo_t
47
*/
48
typedef unsigned long (*genpool_algo_t)(unsigned long *map,
49
unsigned long size,
···
38
struct gen_pool;
39
40
/**
41
+
* typedef genpool_algo_t: Allocation callback function type definition
42
* @map: Pointer to bitmap
43
* @size: The bitmap size in bits
44
* @start: The bitnumber to start searching at
45
* @nr: The number of zeroed bits we're looking for
46
+
* @data: optional additional data used by the callback
47
+
* @pool: the pool being allocated from
48
*/
49
typedef unsigned long (*genpool_algo_t)(unsigned long *map,
50
unsigned long size,
+1
-1
lib/assoc_array.c
+1
-1
lib/assoc_array.c
+4
scripts/kernel-doc
+4
scripts/kernel-doc
···
2226
if ($x =~ /enum\s+(\w+)\s*{(.*)}/) {
2227
$declaration_name = $1;
2228
my $members = $2;
2229
2230
foreach my $arg (split ',', $members) {
2231
$arg =~ s/^\s*(\w+).*/$1/;
···
2767
2768
while (1) {
2769
if ( $x =~ /([^{};]*)([{};])(.*)/ ) {
2770
$prototype .= $1 . $2;
2771
($2 eq '{') && $brcount++;
2772
($2 eq '}') && $brcount--;
···
2226
if ($x =~ /enum\s+(\w+)\s*{(.*)}/) {
2227
$declaration_name = $1;
2228
my $members = $2;
2229
+
$members =~ s/\s+$//;
2230
2231
foreach my $arg (split ',', $members) {
2232
$arg =~ s/^\s*(\w+).*/$1/;
···
2766
2767
while (1) {
2768
if ( $x =~ /([^{};]*)([{};])(.*)/ ) {
2769
+
if( length $prototype ) {
2770
+
$prototype .= " "
2771
+
}
2772
$prototype .= $1 . $2;
2773
($2 eq '{') && $brcount++;
2774
($2 eq '}') && $brcount--;
+609
scripts/sphinx-pre-install
+609
scripts/sphinx-pre-install
···
···
1
+
#!/usr/bin/perl
2
+
use strict;
3
+
4
+
# Copyright (c) 2017 Mauro Carvalho Chehab <mchehab@kernel.org>
5
+
#
6
+
# This program is free software; you can redistribute it and/or
7
+
# modify it under the terms of the GNU General Public License
8
+
# as published by the Free Software Foundation; either version 2
9
+
# of the License, or (at your option) any later version.
10
+
#
11
+
# This program is distributed in the hope that it will be useful,
12
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+
# GNU General Public License for more details.
15
+
16
+
my $virtenv_dir = "sphinx_1.4";
17
+
my $requirement_file = "Documentation/sphinx/requirements.txt";
18
+
19
+
#
20
+
# Static vars
21
+
#
22
+
23
+
my %missing;
24
+
my $system_release;
25
+
my $need = 0;
26
+
my $optional = 0;
27
+
my $need_symlink = 0;
28
+
my $need_sphinx = 0;
29
+
my $install = "";
30
+
31
+
#
32
+
# Command line arguments
33
+
#
34
+
35
+
my $pdf = 1;
36
+
my $virtualenv = 1;
37
+
38
+
#
39
+
# List of required texlive packages on Fedora and OpenSuse
40
+
#
41
+
42
+
my %texlive = (
43
+
'adjustbox.sty' => 'texlive-adjustbox',
44
+
'amsfonts.sty' => 'texlive-amsfonts',
45
+
'amsmath.sty' => 'texlive-amsmath',
46
+
'amssymb.sty' => 'texlive-amsfonts',
47
+
'amsthm.sty' => 'texlive-amscls',
48
+
'anyfontsize.sty' => 'texlive-anyfontsize',
49
+
'atbegshi.sty' => 'texlive-oberdiek',
50
+
'bm.sty' => 'texlive-tools',
51
+
'capt-of.sty' => 'texlive-capt-of',
52
+
'cmap.sty' => 'texlive-cmap',
53
+
'ecrm1000.tfm' => 'texlive-ec',
54
+
'eqparbox.sty' => 'texlive-eqparbox',
55
+
'eu1enc.def' => 'texlive-euenc',
56
+
'fancybox.sty' => 'texlive-fancybox',
57
+
'fancyvrb.sty' => 'texlive-fancyvrb',
58
+
'float.sty' => 'texlive-float',
59
+
'fncychap.sty' => 'texlive-fncychap',
60
+
'footnote.sty' => 'texlive-mdwtools',
61
+
'framed.sty' => 'texlive-framed',
62
+
'luatex85.sty' => 'texlive-luatex85',
63
+
'multirow.sty' => 'texlive-multirow',
64
+
'needspace.sty' => 'texlive-needspace',
65
+
'palatino.sty' => 'texlive-psnfss',
66
+
'parskip.sty' => 'texlive-parskip',
67
+
'polyglossia.sty' => 'texlive-polyglossia',
68
+
'tabulary.sty' => 'texlive-tabulary',
69
+
'threeparttable.sty' => 'texlive-threeparttable',
70
+
'titlesec.sty' => 'texlive-titlesec',
71
+
'ucs.sty' => 'texlive-ucs',
72
+
'upquote.sty' => 'texlive-upquote',
73
+
'wrapfig.sty' => 'texlive-wrapfig',
74
+
);
75
+
76
+
#
77
+
# Subroutines that checks if a feature exists
78
+
#
79
+
80
+
sub check_missing(%)
81
+
{
82
+
my %map = %{$_[0]};
83
+
84
+
foreach my $prog (sort keys %missing) {
85
+
my $is_optional = $missing{$prog};
86
+
87
+
if ($is_optional) {
88
+
print "Warning: better to also install \"$prog\".\n";
89
+
} else {
90
+
print "ERROR: please install \"$prog\", otherwise, build won't work.\n";
91
+
}
92
+
if (defined($map{$prog})) {
93
+
$install .= " " . $map{$prog};
94
+
} else {
95
+
$install .= " " . $prog;
96
+
}
97
+
}
98
+
99
+
$install =~ s/^\s//;
100
+
}
101
+
102
+
sub add_package($$)
103
+
{
104
+
my $package = shift;
105
+
my $is_optional = shift;
106
+
107
+
$missing{$package} = $is_optional;
108
+
if ($is_optional) {
109
+
$optional++;
110
+
} else {
111
+
$need++;
112
+
}
113
+
}
114
+
115
+
sub check_missing_file($$$)
116
+
{
117
+
my $file = shift;
118
+
my $package = shift;
119
+
my $is_optional = shift;
120
+
121
+
return if(-e $file);
122
+
123
+
add_package($package, $is_optional);
124
+
}
125
+
126
+
sub findprog($)
127
+
{
128
+
foreach(split(/:/, $ENV{PATH})) {
129
+
return "$_/$_[0]" if(-x "$_/$_[0]");
130
+
}
131
+
}
132
+
133
+
sub check_program($$)
134
+
{
135
+
my $prog = shift;
136
+
my $is_optional = shift;
137
+
138
+
return if findprog($prog);
139
+
140
+
add_package($prog, $is_optional);
141
+
}
142
+
143
+
sub check_perl_module($$)
144
+
{
145
+
my $prog = shift;
146
+
my $is_optional = shift;
147
+
148
+
my $err = system("perl -M$prog -e 1 2>/dev/null /dev/null");
149
+
return if ($err == 0);
150
+
151
+
add_package($prog, $is_optional);
152
+
}
153
+
154
+
sub check_python_module($$)
155
+
{
156
+
my $prog = shift;
157
+
my $is_optional = shift;
158
+
159
+
my $err = system("python3 -c 'import $prog' 2>/dev/null /dev/null");
160
+
return if ($err == 0);
161
+
my $err = system("python -c 'import $prog' 2>/dev/null /dev/null");
162
+
return if ($err == 0);
163
+
164
+
add_package($prog, $is_optional);
165
+
}
166
+
167
+
sub check_rpm_missing($$)
168
+
{
169
+
my @pkgs = @{$_[0]};
170
+
my $is_optional = $_[1];
171
+
172
+
foreach my $prog(@pkgs) {
173
+
my $err = system("rpm -q '$prog' 2>/dev/null >/dev/null");
174
+
add_package($prog, $is_optional) if ($err);
175
+
}
176
+
}
177
+
178
+
sub check_pacman_missing($$)
179
+
{
180
+
my @pkgs = @{$_[0]};
181
+
my $is_optional = $_[1];
182
+
183
+
foreach my $prog(@pkgs) {
184
+
my $err = system("pacman -Q '$prog' 2>/dev/null >/dev/null");
185
+
add_package($prog, $is_optional) if ($err);
186
+
}
187
+
}
188
+
189
+
sub check_missing_tex($)
190
+
{
191
+
my $is_optional = shift;
192
+
my $kpsewhich = findprog("kpsewhich");
193
+
194
+
foreach my $prog(keys %texlive) {
195
+
my $package = $texlive{$prog};
196
+
if (!$kpsewhich) {
197
+
add_package($package, $is_optional);
198
+
next;
199
+
}
200
+
my $file = qx($kpsewhich $prog);
201
+
add_package($package, $is_optional) if ($file =~ /^\s*$/);
202
+
}
203
+
}
204
+
205
+
sub check_sphinx()
206
+
{
207
+
return if findprog("sphinx-build");
208
+
209
+
if (findprog("sphinx-build-3")) {
210
+
$need_symlink = 1;
211
+
return;
212
+
}
213
+
214
+
if ($virtualenv) {
215
+
my $prog = findprog("virtualenv-3");
216
+
$prog = findprog("virtualenv-3.5") if (!$prog);
217
+
218
+
check_program("virtualenv", 0) if (!$prog);
219
+
$need_sphinx = 1;
220
+
} else {
221
+
add_package("python-sphinx", 0);
222
+
}
223
+
}
224
+
225
+
#
226
+
# Ancillary subroutines
227
+
#
228
+
229
+
sub catcheck($)
230
+
{
231
+
my $res = "";
232
+
$res = qx(cat $_[0]) if (-r $_[0]);
233
+
return $res;
234
+
}
235
+
236
+
sub which($)
237
+
{
238
+
my $file = shift;
239
+
my @path = split ":", $ENV{PATH};
240
+
241
+
foreach my $dir(@path) {
242
+
my $name = $dir.'/'.$file;
243
+
return $name if (-x $name );
244
+
}
245
+
return undef;
246
+
}
247
+
248
+
#
249
+
# Subroutines that check distro-specific hints
250
+
#
251
+
252
+
sub give_debian_hints()
253
+
{
254
+
my %map = (
255
+
"python-sphinx" => "python3-sphinx",
256
+
"sphinx_rtd_theme" => "python3-sphinx-rtd-theme",
257
+
"virtualenv" => "virtualenv",
258
+
"dot" => "graphviz",
259
+
"convert" => "imagemagick",
260
+
"Pod::Usage" => "perl-modules",
261
+
"xelatex" => "texlive-xetex",
262
+
"rsvg-convert" => "librsvg2-bin",
263
+
);
264
+
265
+
if ($pdf) {
266
+
check_missing_file("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
267
+
"fonts-dejavu", 1);
268
+
}
269
+
270
+
check_program("dvipng", 1) if ($pdf);
271
+
check_missing(\%map);
272
+
273
+
return if (!$need && !$optional);
274
+
printf("You should run:\n\n\tsudo apt-get install $install\n");
275
+
}
276
+
277
+
sub give_redhat_hints()
278
+
{
279
+
my %map = (
280
+
"python-sphinx" => "python3-sphinx",
281
+
"sphinx_rtd_theme" => "python3-sphinx_rtd_theme",
282
+
"virtualenv" => "python3-virtualenv",
283
+
"dot" => "graphviz",
284
+
"convert" => "ImageMagick",
285
+
"Pod::Usage" => "perl-Pod-Usage",
286
+
"xelatex" => "texlive-xetex-bin",
287
+
"rsvg-convert" => "librsvg2-tools",
288
+
);
289
+
290
+
my @fedora26_opt_pkgs = (
291
+
"graphviz-gd", # Fedora 26: needed for PDF support
292
+
);
293
+
294
+
my @fedora_tex_pkgs = (
295
+
"texlive-collection-fontsrecommended",
296
+
"texlive-collection-latex",
297
+
"dejavu-sans-fonts",
298
+
"dejavu-serif-fonts",
299
+
"dejavu-sans-mono-fonts",
300
+
);
301
+
302
+
#
303
+
# Checks valid for RHEL/CentOS version 7.x.
304
+
#
305
+
if (! $system_release =~ /Fedora/) {
306
+
$map{"virtualenv"} = "python-virtualenv";
307
+
}
308
+
309
+
my $release;
310
+
311
+
$release = $1 if ($system_release =~ /Fedora\s+release\s+(\d+)/);
312
+
313
+
check_rpm_missing(\@fedora26_opt_pkgs, 1) if ($pdf && $release >= 26);
314
+
check_rpm_missing(\@fedora_tex_pkgs, 1) if ($pdf);
315
+
check_missing_tex(1) if ($pdf);
316
+
check_missing(\%map);
317
+
318
+
return if (!$need && !$optional);
319
+
320
+
if ($release >= 18) {
321
+
# dnf, for Fedora 18+
322
+
printf("You should run:\n\n\tsudo dnf install -y $install\n");
323
+
} else {
324
+
# yum, for RHEL (and clones) or Fedora version < 18
325
+
printf("You should run:\n\n\tsudo yum install -y $install\n");
326
+
}
327
+
}
328
+
329
+
sub give_opensuse_hints()
330
+
{
331
+
my %map = (
332
+
"python-sphinx" => "python3-sphinx",
333
+
"sphinx_rtd_theme" => "python3-sphinx_rtd_theme",
334
+
"virtualenv" => "python3-virtualenv",
335
+
"dot" => "graphviz",
336
+
"convert" => "ImageMagick",
337
+
"Pod::Usage" => "perl-Pod-Usage",
338
+
"xelatex" => "texlive-xetex-bin",
339
+
"rsvg-convert" => "rsvg-view",
340
+
);
341
+
342
+
my @suse_tex_pkgs = (
343
+
"texlive-babel-english",
344
+
"texlive-caption",
345
+
"texlive-colortbl",
346
+
"texlive-courier",
347
+
"texlive-dvips",
348
+
"texlive-helvetic",
349
+
"texlive-makeindex",
350
+
"texlive-metafont",
351
+
"texlive-metapost",
352
+
"texlive-palatino",
353
+
"texlive-preview",
354
+
"texlive-times",
355
+
"texlive-zapfchan",
356
+
"texlive-zapfding",
357
+
);
358
+
359
+
check_rpm_missing(\@suse_tex_pkgs, 1) if ($pdf);
360
+
check_missing_tex(1) if ($pdf);
361
+
check_missing(\%map);
362
+
363
+
return if (!$need && !$optional);
364
+
printf("You should run:\n\n\tsudo zypper install --no-recommends $install\n");
365
+
}
366
+
367
+
sub give_mageia_hints()
368
+
{
369
+
my %map = (
370
+
"python-sphinx" => "python3-sphinx",
371
+
"sphinx_rtd_theme" => "python3-sphinx_rtd_theme",
372
+
"virtualenv" => "python3-virtualenv",
373
+
"dot" => "graphviz",
374
+
"convert" => "ImageMagick",
375
+
"Pod::Usage" => "perl-Pod-Usage",
376
+
"xelatex" => "texlive",
377
+
"rsvg-convert" => "librsvg2-tools",
378
+
);
379
+
380
+
my @tex_pkgs = (
381
+
"texlive-fontsextra",
382
+
);
383
+
384
+
check_rpm_missing(\@tex_pkgs, 1) if ($pdf);
385
+
check_missing(\%map);
386
+
387
+
return if (!$need && !$optional);
388
+
printf("You should run:\n\n\tsudo urpmi $install\n");
389
+
}
390
+
391
+
sub give_arch_linux_hints()
392
+
{
393
+
my %map = (
394
+
"sphinx_rtd_theme" => "python-sphinx_rtd_theme",
395
+
"virtualenv" => "python-virtualenv",
396
+
"dot" => "graphviz",
397
+
"convert" => "imagemagick",
398
+
"xelatex" => "texlive-bin",
399
+
"rsvg-convert" => "extra/librsvg",
400
+
);
401
+
402
+
my @archlinux_tex_pkgs = (
403
+
"texlive-core",
404
+
"texlive-latexextra",
405
+
"ttf-dejavu",
406
+
);
407
+
check_pacman_missing(\@archlinux_tex_pkgs, 1) if ($pdf);
408
+
check_missing(\%map);
409
+
410
+
return if (!$need && !$optional);
411
+
printf("You should run:\n\n\tsudo pacman -S $install\n");
412
+
}
413
+
414
+
sub give_gentoo_hints()
415
+
{
416
+
my %map = (
417
+
"sphinx_rtd_theme" => "dev-python/sphinx_rtd_theme",
418
+
"virtualenv" => "dev-python/virtualenv",
419
+
"dot" => "media-gfx/graphviz",
420
+
"convert" => "media-gfx/imagemagick",
421
+
"xelatex" => "dev-texlive/texlive-xetex media-fonts/dejavu",
422
+
"rsvg-convert" => "gnome-base/librsvg",
423
+
);
424
+
425
+
check_missing_file("/usr/share/fonts/dejavu/DejaVuSans.ttf",
426
+
"media-fonts/dejavu", 1) if ($pdf);
427
+
428
+
check_missing(\%map);
429
+
430
+
return if (!$need && !$optional);
431
+
432
+
printf("You should run:\n\n");
433
+
printf("\tsudo su -c 'echo \"media-gfx/imagemagick svg png\" > /etc/portage/package.use/imagemagick'\n");
434
+
printf("\tsudo su -c 'echo \"media-gfx/graphviz cairo pdf\" > /etc/portage/package.use/graphviz'\n");
435
+
printf("\tsudo emerge --ask $install\n");
436
+
437
+
}
438
+
439
+
sub check_distros()
440
+
{
441
+
# Distro-specific hints
442
+
if ($system_release =~ /Red Hat Enterprise Linux/) {
443
+
give_redhat_hints;
444
+
return;
445
+
}
446
+
if ($system_release =~ /CentOS/) {
447
+
give_redhat_hints;
448
+
return;
449
+
}
450
+
if ($system_release =~ /Scientific Linux/) {
451
+
give_redhat_hints;
452
+
return;
453
+
}
454
+
if ($system_release =~ /Oracle Linux Server/) {
455
+
give_redhat_hints;
456
+
return;
457
+
}
458
+
if ($system_release =~ /Fedora/) {
459
+
give_redhat_hints;
460
+
return;
461
+
}
462
+
if ($system_release =~ /Ubuntu/) {
463
+
give_debian_hints;
464
+
return;
465
+
}
466
+
if ($system_release =~ /Debian/) {
467
+
give_debian_hints;
468
+
return;
469
+
}
470
+
if ($system_release =~ /openSUSE/) {
471
+
give_opensuse_hints;
472
+
return;
473
+
}
474
+
if ($system_release =~ /Mageia/) {
475
+
give_mageia_hints;
476
+
return;
477
+
}
478
+
if ($system_release =~ /Arch Linux/) {
479
+
give_arch_linux_hints;
480
+
return;
481
+
}
482
+
if ($system_release =~ /Gentoo/) {
483
+
give_gentoo_hints;
484
+
return;
485
+
}
486
+
487
+
#
488
+
# Fall-back to generic hint code for other distros
489
+
# That's far from ideal, specially for LaTeX dependencies.
490
+
#
491
+
my %map = (
492
+
"sphinx-build" => "sphinx"
493
+
);
494
+
check_missing_tex(1) if ($pdf);
495
+
check_missing(\%map);
496
+
print "I don't know distro $system_release.\n";
497
+
print "So, I can't provide you a hint with the install procedure.\n";
498
+
print "There are likely missing dependencies.\n";
499
+
}
500
+
501
+
#
502
+
# Common dependencies
503
+
#
504
+
505
+
sub check_needs()
506
+
{
507
+
if ($system_release) {
508
+
print "Detected OS: $system_release.\n";
509
+
} else {
510
+
print "Unknown OS\n";
511
+
}
512
+
513
+
# RHEL 7.x and clones have Sphinx version 1.1.x and incomplete texlive
514
+
if (($system_release =~ /Red Hat Enterprise Linux/) ||
515
+
($system_release =~ /CentOS/) ||
516
+
($system_release =~ /Scientific Linux/) ||
517
+
($system_release =~ /Oracle Linux Server/)) {
518
+
$virtualenv = 1;
519
+
$pdf = 0;
520
+
521
+
printf("NOTE: On this distro, Sphinx and TexLive shipped versions are incompatible\n");
522
+
printf("with doc build. So, use Sphinx via a Python virtual environment.\n\n");
523
+
printf("This script can't install a TexLive version that would provide PDF.\n");
524
+
}
525
+
526
+
# Check for needed programs/tools
527
+
check_sphinx();
528
+
check_perl_module("Pod::Usage", 0);
529
+
check_program("make", 0);
530
+
check_program("gcc", 0);
531
+
check_python_module("sphinx_rtd_theme", 1) if (!$virtualenv);
532
+
check_program("xelatex", 1) if ($pdf);
533
+
check_program("dot", 1);
534
+
check_program("convert", 1);
535
+
check_program("rsvg-convert", 1) if ($pdf);
536
+
537
+
check_distros();
538
+
539
+
if ($need_symlink) {
540
+
printf "\tsudo ln -sf %s /usr/bin/sphinx-build\n\n",
541
+
which("sphinx-build-3");
542
+
}
543
+
if ($need_sphinx) {
544
+
my $activate = "$virtenv_dir/bin/activate";
545
+
if (-e "$ENV{'PWD'}/$activate") {
546
+
printf "\nNeed to activate virtualenv with:\n";
547
+
printf "\t. $activate\n";
548
+
} else {
549
+
my $virtualenv = findprog("virtualenv-3");
550
+
$virtualenv = findprog("virtualenv-3.5") if (!$virtualenv);
551
+
$virtualenv = findprog("virtualenv") if (!$virtualenv);
552
+
$virtualenv = "virtualenv" if (!$virtualenv);
553
+
554
+
printf "\t$virtualenv $virtenv_dir\n";
555
+
printf "\t. $activate\n";
556
+
printf "\tpip install -r $requirement_file\n";
557
+
$need++;
558
+
}
559
+
}
560
+
printf "\n";
561
+
562
+
print "All optional dependenties are met.\n" if (!$optional);
563
+
564
+
if ($need == 1) {
565
+
die "Can't build as $need mandatory dependency is missing";
566
+
} elsif ($need) {
567
+
die "Can't build as $need mandatory dependencies are missing";
568
+
}
569
+
570
+
print "Needed package dependencies are met.\n";
571
+
}
572
+
573
+
#
574
+
# Main
575
+
#
576
+
577
+
while (@ARGV) {
578
+
my $arg = shift(@ARGV);
579
+
580
+
if ($arg eq "--no-virtualenv") {
581
+
$virtualenv = 0;
582
+
} elsif ($arg eq "--no-pdf"){
583
+
$pdf = 0;
584
+
} else {
585
+
print "Usage:\n\t$0 <--no-virtualenv> <--no-pdf>\n\n";
586
+
exit -1;
587
+
}
588
+
}
589
+
590
+
#
591
+
# Determine the system type. There's no standard unique way that would
592
+
# work with all distros with a minimal package install. So, several
593
+
# methods are used here.
594
+
#
595
+
# By default, it will use lsb_release function. If not available, it will
596
+
# fail back to reading the known different places where the distro name
597
+
# is stored
598
+
#
599
+
600
+
$system_release = qx(lsb_release -d) if which("lsb_release");
601
+
$system_release =~ s/Description:\s*// if ($system_release);
602
+
$system_release = catcheck("/etc/system-release") if !$system_release;
603
+
$system_release = catcheck("/etc/redhat-release") if !$system_release;
604
+
$system_release = catcheck("/etc/lsb-release") if !$system_release;
605
+
$system_release = catcheck("/etc/gentoo-release") if !$system_release;
606
+
$system_release = catcheck("/etc/issue") if !$system_release;
607
+
$system_release =~ s/\s+$//;
608
+
609
+
check_needs;