GNU Linux-libre 4.19.211-gnu1
[releases.git] / fs / btrfs / locking.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2008 Oracle.  All rights reserved.
4  */
5
6 #include <linux/sched.h>
7 #include <linux/pagemap.h>
8 #include <linux/spinlock.h>
9 #include <linux/page-flags.h>
10 #include <asm/bug.h>
11 #include "ctree.h"
12 #include "extent_io.h"
13 #include "locking.h"
14
15 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb);
16
17 /*
18  * if we currently have a spinning reader or writer lock
19  * (indicated by the rw flag) this will bump the count
20  * of blocking holders and drop the spinlock.
21  */
22 void btrfs_set_lock_blocking_rw(struct extent_buffer *eb, int rw)
23 {
24         /*
25          * no lock is required.  The lock owner may change if
26          * we have a read lock, but it won't change to or away
27          * from us.  If we have the write lock, we are the owner
28          * and it'll never change.
29          */
30         if (eb->lock_nested && current->pid == eb->lock_owner)
31                 return;
32         if (rw == BTRFS_WRITE_LOCK) {
33                 if (atomic_read(&eb->blocking_writers) == 0) {
34                         WARN_ON(atomic_read(&eb->spinning_writers) != 1);
35                         atomic_dec(&eb->spinning_writers);
36                         btrfs_assert_tree_locked(eb);
37                         atomic_inc(&eb->blocking_writers);
38                         write_unlock(&eb->lock);
39                 }
40         } else if (rw == BTRFS_READ_LOCK) {
41                 btrfs_assert_tree_read_locked(eb);
42                 atomic_inc(&eb->blocking_readers);
43                 WARN_ON(atomic_read(&eb->spinning_readers) == 0);
44                 atomic_dec(&eb->spinning_readers);
45                 read_unlock(&eb->lock);
46         }
47 }
48
49 /*
50  * if we currently have a blocking lock, take the spinlock
51  * and drop our blocking count
52  */
53 void btrfs_clear_lock_blocking_rw(struct extent_buffer *eb, int rw)
54 {
55         /*
56          * no lock is required.  The lock owner may change if
57          * we have a read lock, but it won't change to or away
58          * from us.  If we have the write lock, we are the owner
59          * and it'll never change.
60          */
61         if (eb->lock_nested && current->pid == eb->lock_owner)
62                 return;
63
64         if (rw == BTRFS_WRITE_LOCK_BLOCKING) {
65                 BUG_ON(atomic_read(&eb->blocking_writers) != 1);
66                 write_lock(&eb->lock);
67                 WARN_ON(atomic_read(&eb->spinning_writers));
68                 atomic_inc(&eb->spinning_writers);
69                 /* atomic_dec_and_test implies a barrier */
70                 if (atomic_dec_and_test(&eb->blocking_writers))
71                         cond_wake_up_nomb(&eb->write_lock_wq);
72         } else if (rw == BTRFS_READ_LOCK_BLOCKING) {
73                 BUG_ON(atomic_read(&eb->blocking_readers) == 0);
74                 read_lock(&eb->lock);
75                 atomic_inc(&eb->spinning_readers);
76                 /* atomic_dec_and_test implies a barrier */
77                 if (atomic_dec_and_test(&eb->blocking_readers))
78                         cond_wake_up_nomb(&eb->read_lock_wq);
79         }
80 }
81
82 /*
83  * take a spinning read lock.  This will wait for any blocking
84  * writers
85  */
86 void btrfs_tree_read_lock(struct extent_buffer *eb)
87 {
88 again:
89         BUG_ON(!atomic_read(&eb->blocking_writers) &&
90                current->pid == eb->lock_owner);
91
92         read_lock(&eb->lock);
93         if (atomic_read(&eb->blocking_writers) &&
94             current->pid == eb->lock_owner) {
95                 /*
96                  * This extent is already write-locked by our thread. We allow
97                  * an additional read lock to be added because it's for the same
98                  * thread. btrfs_find_all_roots() depends on this as it may be
99                  * called on a partly (write-)locked tree.
100                  */
101                 BUG_ON(eb->lock_nested);
102                 eb->lock_nested = 1;
103                 read_unlock(&eb->lock);
104                 return;
105         }
106         if (atomic_read(&eb->blocking_writers)) {
107                 read_unlock(&eb->lock);
108                 wait_event(eb->write_lock_wq,
109                            atomic_read(&eb->blocking_writers) == 0);
110                 goto again;
111         }
112         atomic_inc(&eb->read_locks);
113         atomic_inc(&eb->spinning_readers);
114 }
115
116 /*
117  * take a spinning read lock.
118  * returns 1 if we get the read lock and 0 if we don't
119  * this won't wait for blocking writers
120  */
121 int btrfs_tree_read_lock_atomic(struct extent_buffer *eb)
122 {
123         if (atomic_read(&eb->blocking_writers))
124                 return 0;
125
126         read_lock(&eb->lock);
127         if (atomic_read(&eb->blocking_writers)) {
128                 read_unlock(&eb->lock);
129                 return 0;
130         }
131         atomic_inc(&eb->read_locks);
132         atomic_inc(&eb->spinning_readers);
133         return 1;
134 }
135
136 /*
137  * returns 1 if we get the read lock and 0 if we don't
138  * this won't wait for blocking writers
139  */
140 int btrfs_try_tree_read_lock(struct extent_buffer *eb)
141 {
142         if (atomic_read(&eb->blocking_writers))
143                 return 0;
144
145         if (!read_trylock(&eb->lock))
146                 return 0;
147
148         if (atomic_read(&eb->blocking_writers)) {
149                 read_unlock(&eb->lock);
150                 return 0;
151         }
152         atomic_inc(&eb->read_locks);
153         atomic_inc(&eb->spinning_readers);
154         return 1;
155 }
156
157 /*
158  * returns 1 if we get the read lock and 0 if we don't
159  * this won't wait for blocking writers or readers
160  */
161 int btrfs_try_tree_write_lock(struct extent_buffer *eb)
162 {
163         if (atomic_read(&eb->blocking_writers) ||
164             atomic_read(&eb->blocking_readers))
165                 return 0;
166
167         write_lock(&eb->lock);
168         if (atomic_read(&eb->blocking_writers) ||
169             atomic_read(&eb->blocking_readers)) {
170                 write_unlock(&eb->lock);
171                 return 0;
172         }
173         atomic_inc(&eb->write_locks);
174         atomic_inc(&eb->spinning_writers);
175         eb->lock_owner = current->pid;
176         return 1;
177 }
178
179 /*
180  * drop a spinning read lock
181  */
182 void btrfs_tree_read_unlock(struct extent_buffer *eb)
183 {
184         /*
185          * if we're nested, we have the write lock.  No new locking
186          * is needed as long as we are the lock owner.
187          * The write unlock will do a barrier for us, and the lock_nested
188          * field only matters to the lock owner.
189          */
190         if (eb->lock_nested && current->pid == eb->lock_owner) {
191                 eb->lock_nested = 0;
192                 return;
193         }
194         btrfs_assert_tree_read_locked(eb);
195         WARN_ON(atomic_read(&eb->spinning_readers) == 0);
196         atomic_dec(&eb->spinning_readers);
197         atomic_dec(&eb->read_locks);
198         read_unlock(&eb->lock);
199 }
200
201 /*
202  * drop a blocking read lock
203  */
204 void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb)
205 {
206         /*
207          * if we're nested, we have the write lock.  No new locking
208          * is needed as long as we are the lock owner.
209          * The write unlock will do a barrier for us, and the lock_nested
210          * field only matters to the lock owner.
211          */
212         if (eb->lock_nested && current->pid == eb->lock_owner) {
213                 eb->lock_nested = 0;
214                 return;
215         }
216         btrfs_assert_tree_read_locked(eb);
217         WARN_ON(atomic_read(&eb->blocking_readers) == 0);
218         /* atomic_dec_and_test implies a barrier */
219         if (atomic_dec_and_test(&eb->blocking_readers))
220                 cond_wake_up_nomb(&eb->read_lock_wq);
221         atomic_dec(&eb->read_locks);
222 }
223
224 /*
225  * take a spinning write lock.  This will wait for both
226  * blocking readers or writers
227  */
228 void btrfs_tree_lock(struct extent_buffer *eb)
229 {
230         WARN_ON(eb->lock_owner == current->pid);
231 again:
232         wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0);
233         wait_event(eb->write_lock_wq, atomic_read(&eb->blocking_writers) == 0);
234         write_lock(&eb->lock);
235         if (atomic_read(&eb->blocking_readers)) {
236                 write_unlock(&eb->lock);
237                 wait_event(eb->read_lock_wq,
238                            atomic_read(&eb->blocking_readers) == 0);
239                 goto again;
240         }
241         if (atomic_read(&eb->blocking_writers)) {
242                 write_unlock(&eb->lock);
243                 wait_event(eb->write_lock_wq,
244                            atomic_read(&eb->blocking_writers) == 0);
245                 goto again;
246         }
247         WARN_ON(atomic_read(&eb->spinning_writers));
248         atomic_inc(&eb->spinning_writers);
249         atomic_inc(&eb->write_locks);
250         eb->lock_owner = current->pid;
251 }
252
253 /*
254  * drop a spinning or a blocking write lock.
255  */
256 void btrfs_tree_unlock(struct extent_buffer *eb)
257 {
258         int blockers = atomic_read(&eb->blocking_writers);
259
260         BUG_ON(blockers > 1);
261
262         btrfs_assert_tree_locked(eb);
263         eb->lock_owner = 0;
264         atomic_dec(&eb->write_locks);
265
266         if (blockers) {
267                 WARN_ON(atomic_read(&eb->spinning_writers));
268                 atomic_dec(&eb->blocking_writers);
269                 /* Use the lighter barrier after atomic */
270                 smp_mb__after_atomic();
271                 cond_wake_up_nomb(&eb->write_lock_wq);
272         } else {
273                 WARN_ON(atomic_read(&eb->spinning_writers) != 1);
274                 atomic_dec(&eb->spinning_writers);
275                 write_unlock(&eb->lock);
276         }
277 }
278
279 void btrfs_assert_tree_locked(struct extent_buffer *eb)
280 {
281         BUG_ON(!atomic_read(&eb->write_locks));
282 }
283
284 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb)
285 {
286         BUG_ON(!atomic_read(&eb->read_locks));
287 }