tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2019 Intel Corporation
5 */
6
7#include <linux/wait_bit.h>
8
9#include "intel_runtime_pm.h"
10#include "intel_wakeref.h"
11
12static void rpm_get(struct intel_wakeref *wf)
13{
14 wf->wakeref = intel_runtime_pm_get(wf->rpm);
15}
16
17static void rpm_put(struct intel_wakeref *wf)
18{
19 intel_wakeref_t wakeref = fetch_and_zero(&wf->wakeref);
20
21 intel_runtime_pm_put(wf->rpm, wakeref);
22 INTEL_WAKEREF_BUG_ON(!wakeref);
23}
24
25int __intel_wakeref_get_first(struct intel_wakeref *wf)
26{
27 /*
28 * Treat get/put as different subclasses, as we may need to run
29 * the put callback from under the shrinker and do not want to
30 * cross-contanimate that callback with any extra work performed
31 * upon acquiring the wakeref.
32 */
33 mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING);
34 if (!atomic_read(&wf->count)) {
35 int err;
36
37 rpm_get(wf);
38
39 err = wf->ops->get(wf);
40 if (unlikely(err)) {
41 rpm_put(wf);
42 mutex_unlock(&wf->mutex);
43 return err;
44 }
45
46 smp_mb__before_atomic(); /* release wf->count */
47 }
48 atomic_inc(&wf->count);
49 mutex_unlock(&wf->mutex);
50
51 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
52 return 0;
53}
54
55static void ____intel_wakeref_put_last(struct intel_wakeref *wf)
56{
57 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
58 if (unlikely(!atomic_dec_and_test(&wf->count)))
59 goto unlock;
60
61 /* ops->put() must reschedule its own release on error/deferral */
62 if (likely(!wf->ops->put(wf))) {
63 rpm_put(wf);
64 wake_up_var(&wf->wakeref);
65 }
66
67unlock:
68 mutex_unlock(&wf->mutex);
69}
70
71void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags)
72{
73 INTEL_WAKEREF_BUG_ON(work_pending(&wf->work));
74
75 /* Assume we are not in process context and so cannot sleep. */
76 if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) {
77 schedule_work(&wf->work);
78 return;
79 }
80
81 ____intel_wakeref_put_last(wf);
82}
83
84static void __intel_wakeref_put_work(struct work_struct *wrk)
85{
86 struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work);
87
88 if (atomic_add_unless(&wf->count, -1, 1))
89 return;
90
91 mutex_lock(&wf->mutex);
92 ____intel_wakeref_put_last(wf);
93}
94
95void __intel_wakeref_init(struct intel_wakeref *wf,
96 struct intel_runtime_pm *rpm,
97 const struct intel_wakeref_ops *ops,
98 struct intel_wakeref_lockclass *key)
99{
100 wf->rpm = rpm;
101 wf->ops = ops;
102
103 __mutex_init(&wf->mutex, "wakeref.mutex", &key->mutex);
104 atomic_set(&wf->count, 0);
105 wf->wakeref = 0;
106
107 INIT_WORK(&wf->work, __intel_wakeref_put_work);
108 lockdep_init_map(&wf->work.lockdep_map, "wakeref.work", &key->work, 0);
109}
110
111int intel_wakeref_wait_for_idle(struct intel_wakeref *wf)
112{
113 int err;
114
115 might_sleep();
116
117 err = wait_var_event_killable(&wf->wakeref,
118 !intel_wakeref_is_active(wf));
119 if (err)
120 return err;
121
122 intel_wakeref_unlock_wait(wf);
123 return 0;
124}
125
126static void wakeref_auto_timeout(struct timer_list *t)
127{
128 struct intel_wakeref_auto *wf = from_timer(wf, t, timer);
129 intel_wakeref_t wakeref;
130 unsigned long flags;
131
132 if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags))
133 return;
134
135 wakeref = fetch_and_zero(&wf->wakeref);
136 spin_unlock_irqrestore(&wf->lock, flags);
137
138 intel_runtime_pm_put(wf->rpm, wakeref);
139}
140
141void intel_wakeref_auto_init(struct intel_wakeref_auto *wf,
142 struct intel_runtime_pm *rpm)
143{
144 spin_lock_init(&wf->lock);
145 timer_setup(&wf->timer, wakeref_auto_timeout, 0);
146 refcount_set(&wf->count, 0);
147 wf->wakeref = 0;
148 wf->rpm = rpm;
149}
150
151void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout)
152{
153 unsigned long flags;
154
155 if (!timeout) {
156 if (del_timer_sync(&wf->timer))
157 wakeref_auto_timeout(&wf->timer);
158 return;
159 }
160
161 /* Our mission is that we only extend an already active wakeref */
162 assert_rpm_wakelock_held(wf->rpm);
163
164 if (!refcount_inc_not_zero(&wf->count)) {
165 spin_lock_irqsave(&wf->lock, flags);
166 if (!refcount_inc_not_zero(&wf->count)) {
167 INTEL_WAKEREF_BUG_ON(wf->wakeref);
168 wf->wakeref = intel_runtime_pm_get_if_in_use(wf->rpm);
169 refcount_set(&wf->count, 1);
170 }
171 spin_unlock_irqrestore(&wf->lock, flags);
172 }
173
174 /*
175 * If we extend a pending timer, we will only get a single timer
176 * callback and so need to cancel the local inc by running the
177 * elided callback to keep the wf->count balanced.
178 */
179 if (mod_timer(&wf->timer, jiffies + timeout))
180 wakeref_auto_timeout(&wf->timer);
181}
182
183void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf)
184{
185 intel_wakeref_auto(wf, 0);
186 INTEL_WAKEREF_BUG_ON(wf->wakeref);
187}