GNU Linux-libre 4.19.314-gnu1
[releases.git] / net / sunrpc / xprt.c
1 /*
2  *  linux/net/sunrpc/xprt.c
3  *
4  *  This is a generic RPC call interface supporting congestion avoidance,
5  *  and asynchronous calls.
6  *
7  *  The interface works like this:
8  *
9  *  -   When a process places a call, it allocates a request slot if
10  *      one is available. Otherwise, it sleeps on the backlog queue
11  *      (xprt_reserve).
12  *  -   Next, the caller puts together the RPC message, stuffs it into
13  *      the request struct, and calls xprt_transmit().
14  *  -   xprt_transmit sends the message and installs the caller on the
15  *      transport's wait list. At the same time, if a reply is expected,
16  *      it installs a timer that is run after the packet's timeout has
17  *      expired.
18  *  -   When a packet arrives, the data_ready handler walks the list of
19  *      pending requests for that transport. If a matching XID is found, the
20  *      caller is woken up, and the timer removed.
21  *  -   When no reply arrives within the timeout interval, the timer is
22  *      fired by the kernel and runs xprt_timer(). It either adjusts the
23  *      timeout values (minor timeout) or wakes up the caller with a status
24  *      of -ETIMEDOUT.
25  *  -   When the caller receives a notification from RPC that a reply arrived,
26  *      it should release the RPC slot, and process the reply.
27  *      If the call timed out, it may choose to retry the operation by
28  *      adjusting the initial timeout value, and simply calling rpc_call
29  *      again.
30  *
31  *  Support for async RPC is done through a set of RPC-specific scheduling
32  *  primitives that `transparently' work for processes as well as async
33  *  tasks that rely on callbacks.
34  *
35  *  Copyright (C) 1995-1997, Olaf Kirch <okir@monad.swb.de>
36  *
37  *  Transport switch API copyright (C) 2005, Chuck Lever <cel@netapp.com>
38  */
39
40 #include <linux/module.h>
41
42 #include <linux/types.h>
43 #include <linux/interrupt.h>
44 #include <linux/workqueue.h>
45 #include <linux/net.h>
46 #include <linux/ktime.h>
47
48 #include <linux/sunrpc/clnt.h>
49 #include <linux/sunrpc/metrics.h>
50 #include <linux/sunrpc/bc_xprt.h>
51 #include <linux/rcupdate.h>
52
53 #include <trace/events/sunrpc.h>
54
55 #include "sunrpc.h"
56
57 /*
58  * Local variables
59  */
60
61 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
62 # define RPCDBG_FACILITY        RPCDBG_XPRT
63 #endif
64
65 /*
66  * Local functions
67  */
68 static void      xprt_init(struct rpc_xprt *xprt, struct net *net);
69 static __be32   xprt_alloc_xid(struct rpc_xprt *xprt);
70 static void     xprt_connect_status(struct rpc_task *task);
71 static int      __xprt_get_cong(struct rpc_xprt *, struct rpc_task *);
72 static void     __xprt_put_cong(struct rpc_xprt *, struct rpc_rqst *);
73 static void      xprt_destroy(struct rpc_xprt *xprt);
74
75 static DEFINE_SPINLOCK(xprt_list_lock);
76 static LIST_HEAD(xprt_list);
77
78 /**
79  * xprt_register_transport - register a transport implementation
80  * @transport: transport to register
81  *
82  * If a transport implementation is loaded as a kernel module, it can
83  * call this interface to make itself known to the RPC client.
84  *
85  * Returns:
86  * 0:           transport successfully registered
87  * -EEXIST:     transport already registered
88  * -EINVAL:     transport module being unloaded
89  */
90 int xprt_register_transport(struct xprt_class *transport)
91 {
92         struct xprt_class *t;
93         int result;
94
95         result = -EEXIST;
96         spin_lock(&xprt_list_lock);
97         list_for_each_entry(t, &xprt_list, list) {
98                 /* don't register the same transport class twice */
99                 if (t->ident == transport->ident)
100                         goto out;
101         }
102
103         list_add_tail(&transport->list, &xprt_list);
104         printk(KERN_INFO "RPC: Registered %s transport module.\n",
105                transport->name);
106         result = 0;
107
108 out:
109         spin_unlock(&xprt_list_lock);
110         return result;
111 }
112 EXPORT_SYMBOL_GPL(xprt_register_transport);
113
114 /**
115  * xprt_unregister_transport - unregister a transport implementation
116  * @transport: transport to unregister
117  *
118  * Returns:
119  * 0:           transport successfully unregistered
120  * -ENOENT:     transport never registered
121  */
122 int xprt_unregister_transport(struct xprt_class *transport)
123 {
124         struct xprt_class *t;
125         int result;
126
127         result = 0;
128         spin_lock(&xprt_list_lock);
129         list_for_each_entry(t, &xprt_list, list) {
130                 if (t == transport) {
131                         printk(KERN_INFO
132                                 "RPC: Unregistered %s transport module.\n",
133                                 transport->name);
134                         list_del_init(&transport->list);
135                         goto out;
136                 }
137         }
138         result = -ENOENT;
139
140 out:
141         spin_unlock(&xprt_list_lock);
142         return result;
143 }
144 EXPORT_SYMBOL_GPL(xprt_unregister_transport);
145
146 static void
147 xprt_class_release(const struct xprt_class *t)
148 {
149         module_put(t->owner);
150 }
151
152 static const struct xprt_class *
153 xprt_class_find_by_netid_locked(const char *netid)
154 {
155         const struct xprt_class *t;
156         unsigned int i;
157
158         list_for_each_entry(t, &xprt_list, list) {
159                 for (i = 0; t->netid[i][0] != '\0'; i++) {
160                         if (strcmp(t->netid[i], netid) != 0)
161                                 continue;
162                         if (!try_module_get(t->owner))
163                                 continue;
164                         return t;
165                 }
166         }
167         return NULL;
168 }
169
170 static const struct xprt_class *
171 xprt_class_find_by_netid(const char *netid)
172 {
173         const struct xprt_class *t;
174
175         spin_lock(&xprt_list_lock);
176         t = xprt_class_find_by_netid_locked(netid);
177         if (!t) {
178                 spin_unlock(&xprt_list_lock);
179                 request_module("rpc%s", netid);
180                 spin_lock(&xprt_list_lock);
181                 t = xprt_class_find_by_netid_locked(netid);
182         }
183         spin_unlock(&xprt_list_lock);
184         return t;
185 }
186
187 /**
188  * xprt_load_transport - load a transport implementation
189  * @netid: transport to load
190  *
191  * Returns:
192  * 0:           transport successfully loaded
193  * -ENOENT:     transport module not available
194  */
195 int xprt_load_transport(const char *netid)
196 {
197         const struct xprt_class *t;
198
199         t = xprt_class_find_by_netid(netid);
200         if (!t)
201                 return -ENOENT;
202         xprt_class_release(t);
203         return 0;
204 }
205 EXPORT_SYMBOL_GPL(xprt_load_transport);
206
207 /**
208  * xprt_reserve_xprt - serialize write access to transports
209  * @task: task that is requesting access to the transport
210  * @xprt: pointer to the target transport
211  *
212  * This prevents mixing the payload of separate requests, and prevents
213  * transport connects from colliding with writes.  No congestion control
214  * is provided.
215  */
216 int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
217 {
218         struct rpc_rqst *req = task->tk_rqstp;
219         int priority;
220
221         if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
222                 if (task == xprt->snd_task)
223                         return 1;
224                 goto out_sleep;
225         }
226         xprt->snd_task = task;
227         if (req != NULL)
228                 req->rq_ntrans++;
229
230         return 1;
231
232 out_sleep:
233         dprintk("RPC: %5u failed to lock transport %p\n",
234                         task->tk_pid, xprt);
235         task->tk_timeout = 0;
236         task->tk_status = -EAGAIN;
237         if (req == NULL)
238                 priority = RPC_PRIORITY_LOW;
239         else if (!req->rq_ntrans)
240                 priority = RPC_PRIORITY_NORMAL;
241         else
242                 priority = RPC_PRIORITY_HIGH;
243         rpc_sleep_on_priority(&xprt->sending, task, NULL, priority);
244         return 0;
245 }
246 EXPORT_SYMBOL_GPL(xprt_reserve_xprt);
247
248 static void xprt_clear_locked(struct rpc_xprt *xprt)
249 {
250         xprt->snd_task = NULL;
251         if (!test_bit(XPRT_CLOSE_WAIT, &xprt->state)) {
252                 smp_mb__before_atomic();
253                 clear_bit(XPRT_LOCKED, &xprt->state);
254                 smp_mb__after_atomic();
255         } else
256                 queue_work(xprtiod_workqueue, &xprt->task_cleanup);
257 }
258
259 /*
260  * xprt_reserve_xprt_cong - serialize write access to transports
261  * @task: task that is requesting access to the transport
262  *
263  * Same as xprt_reserve_xprt, but Van Jacobson congestion control is
264  * integrated into the decision of whether a request is allowed to be
265  * woken up and given access to the transport.
266  */
267 int xprt_reserve_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)
268 {
269         struct rpc_rqst *req = task->tk_rqstp;
270         int priority;
271
272         if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
273                 if (task == xprt->snd_task)
274                         return 1;
275                 goto out_sleep;
276         }
277         if (req == NULL) {
278                 xprt->snd_task = task;
279                 return 1;
280         }
281         if (__xprt_get_cong(xprt, task)) {
282                 xprt->snd_task = task;
283                 req->rq_ntrans++;
284                 return 1;
285         }
286         xprt_clear_locked(xprt);
287 out_sleep:
288         if (req)
289                 __xprt_put_cong(xprt, req);
290         dprintk("RPC: %5u failed to lock transport %p\n", task->tk_pid, xprt);
291         task->tk_timeout = 0;
292         task->tk_status = -EAGAIN;
293         if (req == NULL)
294                 priority = RPC_PRIORITY_LOW;
295         else if (!req->rq_ntrans)
296                 priority = RPC_PRIORITY_NORMAL;
297         else
298                 priority = RPC_PRIORITY_HIGH;
299         rpc_sleep_on_priority(&xprt->sending, task, NULL, priority);
300         return 0;
301 }
302 EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong);
303
304 static inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task)
305 {
306         int retval;
307
308         spin_lock_bh(&xprt->transport_lock);
309         retval = xprt->ops->reserve_xprt(xprt, task);
310         spin_unlock_bh(&xprt->transport_lock);
311         return retval;
312 }
313
314 static bool __xprt_lock_write_func(struct rpc_task *task, void *data)
315 {
316         struct rpc_xprt *xprt = data;
317         struct rpc_rqst *req;
318
319         req = task->tk_rqstp;
320         xprt->snd_task = task;
321         if (req)
322                 req->rq_ntrans++;
323         return true;
324 }
325
326 static void __xprt_lock_write_next(struct rpc_xprt *xprt)
327 {
328         if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
329                 return;
330
331         if (rpc_wake_up_first_on_wq(xprtiod_workqueue, &xprt->sending,
332                                 __xprt_lock_write_func, xprt))
333                 return;
334         xprt_clear_locked(xprt);
335 }
336
337 static bool __xprt_lock_write_cong_func(struct rpc_task *task, void *data)
338 {
339         struct rpc_xprt *xprt = data;
340         struct rpc_rqst *req;
341
342         req = task->tk_rqstp;
343         if (req == NULL) {
344                 xprt->snd_task = task;
345                 return true;
346         }
347         if (__xprt_get_cong(xprt, task)) {
348                 xprt->snd_task = task;
349                 req->rq_ntrans++;
350                 return true;
351         }
352         return false;
353 }
354
355 static void __xprt_lock_write_next_cong(struct rpc_xprt *xprt)
356 {
357         if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
358                 return;
359         if (RPCXPRT_CONGESTED(xprt))
360                 goto out_unlock;
361         if (rpc_wake_up_first_on_wq(xprtiod_workqueue, &xprt->sending,
362                                 __xprt_lock_write_cong_func, xprt))
363                 return;
364 out_unlock:
365         xprt_clear_locked(xprt);
366 }
367
368 static void xprt_task_clear_bytes_sent(struct rpc_task *task)
369 {
370         if (task != NULL) {
371                 struct rpc_rqst *req = task->tk_rqstp;
372                 if (req != NULL)
373                         req->rq_bytes_sent = 0;
374         }
375 }
376
377 /**
378  * xprt_release_xprt - allow other requests to use a transport
379  * @xprt: transport with other tasks potentially waiting
380  * @task: task that is releasing access to the transport
381  *
382  * Note that "task" can be NULL.  No congestion control is provided.
383  */
384 void xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
385 {
386         if (xprt->snd_task == task) {
387                 xprt_task_clear_bytes_sent(task);
388                 xprt_clear_locked(xprt);
389                 __xprt_lock_write_next(xprt);
390         }
391 }
392 EXPORT_SYMBOL_GPL(xprt_release_xprt);
393
394 /**
395  * xprt_release_xprt_cong - allow other requests to use a transport
396  * @xprt: transport with other tasks potentially waiting
397  * @task: task that is releasing access to the transport
398  *
399  * Note that "task" can be NULL.  Another task is awoken to use the
400  * transport if the transport's congestion window allows it.
401  */
402 void xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)
403 {
404         if (xprt->snd_task == task) {
405                 xprt_task_clear_bytes_sent(task);
406                 xprt_clear_locked(xprt);
407                 __xprt_lock_write_next_cong(xprt);
408         }
409 }
410 EXPORT_SYMBOL_GPL(xprt_release_xprt_cong);
411
412 static inline void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task)
413 {
414         spin_lock_bh(&xprt->transport_lock);
415         xprt->ops->release_xprt(xprt, task);
416         spin_unlock_bh(&xprt->transport_lock);
417 }
418
419 /*
420  * Van Jacobson congestion avoidance. Check if the congestion window
421  * overflowed. Put the task to sleep if this is the case.
422  */
423 static int
424 __xprt_get_cong(struct rpc_xprt *xprt, struct rpc_task *task)
425 {
426         struct rpc_rqst *req = task->tk_rqstp;
427
428         if (req->rq_cong)
429                 return 1;
430         dprintk("RPC: %5u xprt_cwnd_limited cong = %lu cwnd = %lu\n",
431                         task->tk_pid, xprt->cong, xprt->cwnd);
432         if (RPCXPRT_CONGESTED(xprt))
433                 return 0;
434         req->rq_cong = 1;
435         xprt->cong += RPC_CWNDSCALE;
436         return 1;
437 }
438
439 /*
440  * Adjust the congestion window, and wake up the next task
441  * that has been sleeping due to congestion
442  */
443 static void
444 __xprt_put_cong(struct rpc_xprt *xprt, struct rpc_rqst *req)
445 {
446         if (!req->rq_cong)
447                 return;
448         req->rq_cong = 0;
449         xprt->cong -= RPC_CWNDSCALE;
450         __xprt_lock_write_next_cong(xprt);
451 }
452
453 /**
454  * xprt_release_rqst_cong - housekeeping when request is complete
455  * @task: RPC request that recently completed
456  *
457  * Useful for transports that require congestion control.
458  */
459 void xprt_release_rqst_cong(struct rpc_task *task)
460 {
461         struct rpc_rqst *req = task->tk_rqstp;
462
463         __xprt_put_cong(req->rq_xprt, req);
464 }
465 EXPORT_SYMBOL_GPL(xprt_release_rqst_cong);
466
467 /**
468  * xprt_adjust_cwnd - adjust transport congestion window
469  * @xprt: pointer to xprt
470  * @task: recently completed RPC request used to adjust window
471  * @result: result code of completed RPC request
472  *
473  * The transport code maintains an estimate on the maximum number of out-
474  * standing RPC requests, using a smoothed version of the congestion
475  * avoidance implemented in 44BSD. This is basically the Van Jacobson
476  * congestion algorithm: If a retransmit occurs, the congestion window is
477  * halved; otherwise, it is incremented by 1/cwnd when
478  *
479  *      -       a reply is received and
480  *      -       a full number of requests are outstanding and
481  *      -       the congestion window hasn't been updated recently.
482  */
483 void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result)
484 {
485         struct rpc_rqst *req = task->tk_rqstp;
486         unsigned long cwnd = xprt->cwnd;
487
488         if (result >= 0 && cwnd <= xprt->cong) {
489                 /* The (cwnd >> 1) term makes sure
490                  * the result gets rounded properly. */
491                 cwnd += (RPC_CWNDSCALE * RPC_CWNDSCALE + (cwnd >> 1)) / cwnd;
492                 if (cwnd > RPC_MAXCWND(xprt))
493                         cwnd = RPC_MAXCWND(xprt);
494                 __xprt_lock_write_next_cong(xprt);
495         } else if (result == -ETIMEDOUT) {
496                 cwnd >>= 1;
497                 if (cwnd < RPC_CWNDSCALE)
498                         cwnd = RPC_CWNDSCALE;
499         }
500         dprintk("RPC:       cong %ld, cwnd was %ld, now %ld\n",
501                         xprt->cong, xprt->cwnd, cwnd);
502         xprt->cwnd = cwnd;
503         __xprt_put_cong(xprt, req);
504 }
505 EXPORT_SYMBOL_GPL(xprt_adjust_cwnd);
506
507 /**
508  * xprt_wake_pending_tasks - wake all tasks on a transport's pending queue
509  * @xprt: transport with waiting tasks
510  * @status: result code to plant in each task before waking it
511  *
512  */
513 void xprt_wake_pending_tasks(struct rpc_xprt *xprt, int status)
514 {
515         if (status < 0)
516                 rpc_wake_up_status(&xprt->pending, status);
517         else
518                 rpc_wake_up(&xprt->pending);
519 }
520 EXPORT_SYMBOL_GPL(xprt_wake_pending_tasks);
521
522 /**
523  * xprt_wait_for_buffer_space - wait for transport output buffer to clear
524  * @task: task to be put to sleep
525  * @action: function pointer to be executed after wait
526  *
527  * Note that we only set the timer for the case of RPC_IS_SOFT(), since
528  * we don't in general want to force a socket disconnection due to
529  * an incomplete RPC call transmission.
530  */
531 void xprt_wait_for_buffer_space(struct rpc_task *task, rpc_action action)
532 {
533         struct rpc_rqst *req = task->tk_rqstp;
534         struct rpc_xprt *xprt = req->rq_xprt;
535
536         task->tk_timeout = RPC_IS_SOFT(task) ? req->rq_timeout : 0;
537         rpc_sleep_on(&xprt->pending, task, action);
538 }
539 EXPORT_SYMBOL_GPL(xprt_wait_for_buffer_space);
540
541 /**
542  * xprt_write_space - wake the task waiting for transport output buffer space
543  * @xprt: transport with waiting tasks
544  *
545  * Can be called in a soft IRQ context, so xprt_write_space never sleeps.
546  */
547 void xprt_write_space(struct rpc_xprt *xprt)
548 {
549         spin_lock_bh(&xprt->transport_lock);
550         if (xprt->snd_task) {
551                 dprintk("RPC:       write space: waking waiting task on "
552                                 "xprt %p\n", xprt);
553                 rpc_wake_up_queued_task_on_wq(xprtiod_workqueue,
554                                 &xprt->pending, xprt->snd_task);
555         }
556         spin_unlock_bh(&xprt->transport_lock);
557 }
558 EXPORT_SYMBOL_GPL(xprt_write_space);
559
560 /**
561  * xprt_set_retrans_timeout_def - set a request's retransmit timeout
562  * @task: task whose timeout is to be set
563  *
564  * Set a request's retransmit timeout based on the transport's
565  * default timeout parameters.  Used by transports that don't adjust
566  * the retransmit timeout based on round-trip time estimation.
567  */
568 void xprt_set_retrans_timeout_def(struct rpc_task *task)
569 {
570         task->tk_timeout = task->tk_rqstp->rq_timeout;
571 }
572 EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_def);
573
574 /**
575  * xprt_set_retrans_timeout_rtt - set a request's retransmit timeout
576  * @task: task whose timeout is to be set
577  *
578  * Set a request's retransmit timeout using the RTT estimator.
579  */
580 void xprt_set_retrans_timeout_rtt(struct rpc_task *task)
581 {
582         int timer = task->tk_msg.rpc_proc->p_timer;
583         struct rpc_clnt *clnt = task->tk_client;
584         struct rpc_rtt *rtt = clnt->cl_rtt;
585         struct rpc_rqst *req = task->tk_rqstp;
586         unsigned long max_timeout = clnt->cl_timeout->to_maxval;
587
588         task->tk_timeout = rpc_calc_rto(rtt, timer);
589         task->tk_timeout <<= rpc_ntimeo(rtt, timer) + req->rq_retries;
590         if (task->tk_timeout > max_timeout || task->tk_timeout == 0)
591                 task->tk_timeout = max_timeout;
592 }
593 EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_rtt);
594
595 static void xprt_reset_majortimeo(struct rpc_rqst *req)
596 {
597         const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
598
599         req->rq_majortimeo = req->rq_timeout;
600         if (to->to_exponential)
601                 req->rq_majortimeo <<= to->to_retries;
602         else
603                 req->rq_majortimeo += to->to_increment * to->to_retries;
604         if (req->rq_majortimeo > to->to_maxval || req->rq_majortimeo == 0)
605                 req->rq_majortimeo = to->to_maxval;
606         req->rq_majortimeo += jiffies;
607 }
608
609 /**
610  * xprt_adjust_timeout - adjust timeout values for next retransmit
611  * @req: RPC request containing parameters to use for the adjustment
612  *
613  */
614 int xprt_adjust_timeout(struct rpc_rqst *req)
615 {
616         struct rpc_xprt *xprt = req->rq_xprt;
617         const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
618         int status = 0;
619
620         if (time_before(jiffies, req->rq_majortimeo)) {
621                 if (to->to_exponential)
622                         req->rq_timeout <<= 1;
623                 else
624                         req->rq_timeout += to->to_increment;
625                 if (to->to_maxval && req->rq_timeout >= to->to_maxval)
626                         req->rq_timeout = to->to_maxval;
627                 req->rq_retries++;
628         } else {
629                 req->rq_timeout = to->to_initval;
630                 req->rq_retries = 0;
631                 xprt_reset_majortimeo(req);
632                 /* Reset the RTT counters == "slow start" */
633                 spin_lock_bh(&xprt->transport_lock);
634                 rpc_init_rtt(req->rq_task->tk_client->cl_rtt, to->to_initval);
635                 spin_unlock_bh(&xprt->transport_lock);
636                 status = -ETIMEDOUT;
637         }
638
639         if (req->rq_timeout == 0) {
640                 printk(KERN_WARNING "xprt_adjust_timeout: rq_timeout = 0!\n");
641                 req->rq_timeout = 5 * HZ;
642         }
643         return status;
644 }
645
646 static void xprt_autoclose(struct work_struct *work)
647 {
648         struct rpc_xprt *xprt =
649                 container_of(work, struct rpc_xprt, task_cleanup);
650
651         clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
652         xprt->ops->close(xprt);
653         xprt_release_write(xprt, NULL);
654         wake_up_bit(&xprt->state, XPRT_LOCKED);
655 }
656
657 /**
658  * xprt_disconnect_done - mark a transport as disconnected
659  * @xprt: transport to flag for disconnect
660  *
661  */
662 void xprt_disconnect_done(struct rpc_xprt *xprt)
663 {
664         dprintk("RPC:       disconnected transport %p\n", xprt);
665         spin_lock_bh(&xprt->transport_lock);
666         xprt_clear_connected(xprt);
667         xprt_wake_pending_tasks(xprt, -EAGAIN);
668         spin_unlock_bh(&xprt->transport_lock);
669 }
670 EXPORT_SYMBOL_GPL(xprt_disconnect_done);
671
672 /**
673  * xprt_force_disconnect - force a transport to disconnect
674  * @xprt: transport to disconnect
675  *
676  */
677 void xprt_force_disconnect(struct rpc_xprt *xprt)
678 {
679         /* Don't race with the test_bit() in xprt_clear_locked() */
680         spin_lock_bh(&xprt->transport_lock);
681         set_bit(XPRT_CLOSE_WAIT, &xprt->state);
682         /* Try to schedule an autoclose RPC call */
683         if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
684                 queue_work(xprtiod_workqueue, &xprt->task_cleanup);
685         xprt_wake_pending_tasks(xprt, -EAGAIN);
686         spin_unlock_bh(&xprt->transport_lock);
687 }
688 EXPORT_SYMBOL_GPL(xprt_force_disconnect);
689
690 /**
691  * xprt_conditional_disconnect - force a transport to disconnect
692  * @xprt: transport to disconnect
693  * @cookie: 'connection cookie'
694  *
695  * This attempts to break the connection if and only if 'cookie' matches
696  * the current transport 'connection cookie'. It ensures that we don't
697  * try to break the connection more than once when we need to retransmit
698  * a batch of RPC requests.
699  *
700  */
701 void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie)
702 {
703         /* Don't race with the test_bit() in xprt_clear_locked() */
704         spin_lock_bh(&xprt->transport_lock);
705         if (cookie != xprt->connect_cookie)
706                 goto out;
707         if (test_bit(XPRT_CLOSING, &xprt->state))
708                 goto out;
709         set_bit(XPRT_CLOSE_WAIT, &xprt->state);
710         /* Try to schedule an autoclose RPC call */
711         if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
712                 queue_work(xprtiod_workqueue, &xprt->task_cleanup);
713         xprt_wake_pending_tasks(xprt, -EAGAIN);
714 out:
715         spin_unlock_bh(&xprt->transport_lock);
716 }
717
718 static bool
719 xprt_has_timer(const struct rpc_xprt *xprt)
720 {
721         return xprt->idle_timeout != 0;
722 }
723
724 static void
725 xprt_schedule_autodisconnect(struct rpc_xprt *xprt)
726         __must_hold(&xprt->transport_lock)
727 {
728         if (list_empty(&xprt->recv) && xprt_has_timer(xprt))
729                 mod_timer(&xprt->timer, xprt->last_used + xprt->idle_timeout);
730 }
731
732 static void
733 xprt_init_autodisconnect(struct timer_list *t)
734 {
735         struct rpc_xprt *xprt = from_timer(xprt, t, timer);
736
737         spin_lock(&xprt->transport_lock);
738         if (!list_empty(&xprt->recv))
739                 goto out_abort;
740         /* Reset xprt->last_used to avoid connect/autodisconnect cycling */
741         xprt->last_used = jiffies;
742         if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
743                 goto out_abort;
744         spin_unlock(&xprt->transport_lock);
745         queue_work(xprtiod_workqueue, &xprt->task_cleanup);
746         return;
747 out_abort:
748         spin_unlock(&xprt->transport_lock);
749 }
750
751 bool xprt_lock_connect(struct rpc_xprt *xprt,
752                 struct rpc_task *task,
753                 void *cookie)
754 {
755         bool ret = false;
756
757         spin_lock_bh(&xprt->transport_lock);
758         if (!test_bit(XPRT_LOCKED, &xprt->state))
759                 goto out;
760         if (xprt->snd_task != task)
761                 goto out;
762         xprt_task_clear_bytes_sent(task);
763         xprt->snd_task = cookie;
764         ret = true;
765 out:
766         spin_unlock_bh(&xprt->transport_lock);
767         return ret;
768 }
769
770 void xprt_unlock_connect(struct rpc_xprt *xprt, void *cookie)
771 {
772         spin_lock_bh(&xprt->transport_lock);
773         if (xprt->snd_task != cookie)
774                 goto out;
775         if (!test_bit(XPRT_LOCKED, &xprt->state))
776                 goto out;
777         xprt->snd_task =NULL;
778         xprt->ops->release_xprt(xprt, NULL);
779         xprt_schedule_autodisconnect(xprt);
780 out:
781         spin_unlock_bh(&xprt->transport_lock);
782         wake_up_bit(&xprt->state, XPRT_LOCKED);
783 }
784
785 /**
786  * xprt_connect - schedule a transport connect operation
787  * @task: RPC task that is requesting the connect
788  *
789  */
790 void xprt_connect(struct rpc_task *task)
791 {
792         struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt;
793
794         dprintk("RPC: %5u xprt_connect xprt %p %s connected\n", task->tk_pid,
795                         xprt, (xprt_connected(xprt) ? "is" : "is not"));
796
797         if (!xprt_bound(xprt)) {
798                 task->tk_status = -EAGAIN;
799                 return;
800         }
801         if (!xprt_lock_write(xprt, task))
802                 return;
803
804         if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state))
805                 xprt->ops->close(xprt);
806
807         if (!xprt_connected(xprt)) {
808                 task->tk_rqstp->rq_bytes_sent = 0;
809                 task->tk_timeout = task->tk_rqstp->rq_timeout;
810                 task->tk_rqstp->rq_connect_cookie = xprt->connect_cookie;
811                 rpc_sleep_on(&xprt->pending, task, xprt_connect_status);
812
813                 if (test_bit(XPRT_CLOSING, &xprt->state))
814                         return;
815                 if (xprt_test_and_set_connecting(xprt))
816                         return;
817                 /* Race breaker */
818                 if (!xprt_connected(xprt)) {
819                         xprt->stat.connect_start = jiffies;
820                         xprt->ops->connect(xprt, task);
821                 } else {
822                         xprt_clear_connecting(xprt);
823                         task->tk_status = 0;
824                         rpc_wake_up_queued_task(&xprt->pending, task);
825                 }
826         }
827         xprt_release_write(xprt, task);
828 }
829
830 static void xprt_connect_status(struct rpc_task *task)
831 {
832         switch (task->tk_status) {
833         case 0:
834                 dprintk("RPC: %5u xprt_connect_status: connection established\n",
835                                 task->tk_pid);
836                 break;
837         case -ECONNREFUSED:
838         case -ECONNRESET:
839         case -ECONNABORTED:
840         case -ENETUNREACH:
841         case -EHOSTUNREACH:
842         case -EPIPE:
843         case -EAGAIN:
844                 dprintk("RPC: %5u xprt_connect_status: retrying\n", task->tk_pid);
845                 break;
846         case -ETIMEDOUT:
847                 dprintk("RPC: %5u xprt_connect_status: connect attempt timed "
848                                 "out\n", task->tk_pid);
849                 break;
850         default:
851                 dprintk("RPC: %5u xprt_connect_status: error %d connecting to "
852                                 "server %s\n", task->tk_pid, -task->tk_status,
853                                 task->tk_rqstp->rq_xprt->servername);
854                 task->tk_status = -EIO;
855         }
856 }
857
858 /**
859  * xprt_lookup_rqst - find an RPC request corresponding to an XID
860  * @xprt: transport on which the original request was transmitted
861  * @xid: RPC XID of incoming reply
862  *
863  * Caller holds xprt->recv_lock.
864  */
865 struct rpc_rqst *xprt_lookup_rqst(struct rpc_xprt *xprt, __be32 xid)
866 {
867         struct rpc_rqst *entry;
868
869         list_for_each_entry(entry, &xprt->recv, rq_list)
870                 if (entry->rq_xid == xid) {
871                         trace_xprt_lookup_rqst(xprt, xid, 0);
872                         entry->rq_rtt = ktime_sub(ktime_get(), entry->rq_xtime);
873                         return entry;
874                 }
875
876         dprintk("RPC:       xprt_lookup_rqst did not find xid %08x\n",
877                         ntohl(xid));
878         trace_xprt_lookup_rqst(xprt, xid, -ENOENT);
879         xprt->stat.bad_xids++;
880         return NULL;
881 }
882 EXPORT_SYMBOL_GPL(xprt_lookup_rqst);
883
884 /**
885  * xprt_pin_rqst - Pin a request on the transport receive list
886  * @req: Request to pin
887  *
888  * Caller must ensure this is atomic with the call to xprt_lookup_rqst()
889  * so should be holding the xprt transport lock.
890  */
891 void xprt_pin_rqst(struct rpc_rqst *req)
892 {
893         set_bit(RPC_TASK_MSG_RECV, &req->rq_task->tk_runstate);
894 }
895 EXPORT_SYMBOL_GPL(xprt_pin_rqst);
896
897 /**
898  * xprt_unpin_rqst - Unpin a request on the transport receive list
899  * @req: Request to pin
900  *
901  * Caller should be holding the xprt transport lock.
902  */
903 void xprt_unpin_rqst(struct rpc_rqst *req)
904 {
905         struct rpc_task *task = req->rq_task;
906
907         clear_bit(RPC_TASK_MSG_RECV, &task->tk_runstate);
908         if (test_bit(RPC_TASK_MSG_RECV_WAIT, &task->tk_runstate))
909                 wake_up_bit(&task->tk_runstate, RPC_TASK_MSG_RECV);
910 }
911 EXPORT_SYMBOL_GPL(xprt_unpin_rqst);
912
913 static void xprt_wait_on_pinned_rqst(struct rpc_rqst *req)
914 __must_hold(&req->rq_xprt->recv_lock)
915 {
916         struct rpc_task *task = req->rq_task;
917
918         if (task && test_bit(RPC_TASK_MSG_RECV, &task->tk_runstate)) {
919                 spin_unlock(&req->rq_xprt->recv_lock);
920                 set_bit(RPC_TASK_MSG_RECV_WAIT, &task->tk_runstate);
921                 wait_on_bit(&task->tk_runstate, RPC_TASK_MSG_RECV,
922                                 TASK_UNINTERRUPTIBLE);
923                 clear_bit(RPC_TASK_MSG_RECV_WAIT, &task->tk_runstate);
924                 spin_lock(&req->rq_xprt->recv_lock);
925         }
926 }
927
928 /**
929  * xprt_update_rtt - Update RPC RTT statistics
930  * @task: RPC request that recently completed
931  *
932  * Caller holds xprt->recv_lock.
933  */
934 void xprt_update_rtt(struct rpc_task *task)
935 {
936         struct rpc_rqst *req = task->tk_rqstp;
937         struct rpc_rtt *rtt = task->tk_client->cl_rtt;
938         unsigned int timer = task->tk_msg.rpc_proc->p_timer;
939         long m = usecs_to_jiffies(ktime_to_us(req->rq_rtt));
940
941         if (timer) {
942                 if (req->rq_ntrans == 1)
943                         rpc_update_rtt(rtt, timer, m);
944                 rpc_set_timeo(rtt, timer, req->rq_ntrans - 1);
945         }
946 }
947 EXPORT_SYMBOL_GPL(xprt_update_rtt);
948
949 /**
950  * xprt_complete_rqst - called when reply processing is complete
951  * @task: RPC request that recently completed
952  * @copied: actual number of bytes received from the transport
953  *
954  * Caller holds xprt->recv_lock.
955  */
956 void xprt_complete_rqst(struct rpc_task *task, int copied)
957 {
958         struct rpc_rqst *req = task->tk_rqstp;
959         struct rpc_xprt *xprt = req->rq_xprt;
960
961         dprintk("RPC: %5u xid %08x complete (%d bytes received)\n",
962                         task->tk_pid, ntohl(req->rq_xid), copied);
963         trace_xprt_complete_rqst(xprt, req->rq_xid, copied);
964
965         xprt->stat.recvs++;
966
967         list_del_init(&req->rq_list);
968         req->rq_private_buf.len = copied;
969         /* Ensure all writes are done before we update */
970         /* req->rq_reply_bytes_recvd */
971         smp_wmb();
972         req->rq_reply_bytes_recvd = copied;
973         rpc_wake_up_queued_task(&xprt->pending, task);
974 }
975 EXPORT_SYMBOL_GPL(xprt_complete_rqst);
976
977 static void xprt_timer(struct rpc_task *task)
978 {
979         struct rpc_rqst *req = task->tk_rqstp;
980         struct rpc_xprt *xprt = req->rq_xprt;
981
982         if (task->tk_status != -ETIMEDOUT)
983                 return;
984
985         trace_xprt_timer(xprt, req->rq_xid, task->tk_status);
986         if (!req->rq_reply_bytes_recvd) {
987                 if (xprt->ops->timer)
988                         xprt->ops->timer(xprt, task);
989         } else
990                 task->tk_status = 0;
991 }
992
993 /**
994  * xprt_prepare_transmit - reserve the transport before sending a request
995  * @task: RPC task about to send a request
996  *
997  */
998 bool xprt_prepare_transmit(struct rpc_task *task)
999 {
1000         struct rpc_rqst *req = task->tk_rqstp;
1001         struct rpc_xprt *xprt = req->rq_xprt;
1002         bool ret = false;
1003
1004         dprintk("RPC: %5u xprt_prepare_transmit\n", task->tk_pid);
1005
1006         spin_lock_bh(&xprt->transport_lock);
1007         if (!req->rq_bytes_sent) {
1008                 if (req->rq_reply_bytes_recvd) {
1009                         task->tk_status = req->rq_reply_bytes_recvd;
1010                         goto out_unlock;
1011                 }
1012                 if ((task->tk_flags & RPC_TASK_NO_RETRANS_TIMEOUT)
1013                     && xprt_connected(xprt)
1014                     && req->rq_connect_cookie == xprt->connect_cookie) {
1015                         xprt->ops->set_retrans_timeout(task);
1016                         rpc_sleep_on(&xprt->pending, task, xprt_timer);
1017                         goto out_unlock;
1018                 }
1019         }
1020         if (!xprt->ops->reserve_xprt(xprt, task)) {
1021                 task->tk_status = -EAGAIN;
1022                 goto out_unlock;
1023         }
1024         ret = true;
1025 out_unlock:
1026         spin_unlock_bh(&xprt->transport_lock);
1027         return ret;
1028 }
1029
1030 void xprt_end_transmit(struct rpc_task *task)
1031 {
1032         xprt_release_write(task->tk_rqstp->rq_xprt, task);
1033 }
1034
1035 /**
1036  * xprt_transmit - send an RPC request on a transport
1037  * @task: controlling RPC task
1038  *
1039  * We have to copy the iovec because sendmsg fiddles with its contents.
1040  */
1041 void xprt_transmit(struct rpc_task *task)
1042 {
1043         struct rpc_rqst *req = task->tk_rqstp;
1044         struct rpc_xprt *xprt = req->rq_xprt;
1045         unsigned int connect_cookie;
1046         int status;
1047
1048         dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid, req->rq_slen);
1049
1050         if (!req->rq_reply_bytes_recvd) {
1051                 if (list_empty(&req->rq_list) && rpc_reply_expected(task)) {
1052                         /*
1053                          * Add to the list only if we're expecting a reply
1054                          */
1055                         /* Update the softirq receive buffer */
1056                         memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
1057                                         sizeof(req->rq_private_buf));
1058                         /* Add request to the receive list */
1059                         spin_lock(&xprt->recv_lock);
1060                         list_add_tail(&req->rq_list, &xprt->recv);
1061                         spin_unlock(&xprt->recv_lock);
1062                         xprt_reset_majortimeo(req);
1063                         /* Turn off autodisconnect */
1064                         del_singleshot_timer_sync(&xprt->timer);
1065                 }
1066         } else if (!req->rq_bytes_sent)
1067                 return;
1068
1069         connect_cookie = xprt->connect_cookie;
1070         status = xprt->ops->send_request(task);
1071         trace_xprt_transmit(xprt, req->rq_xid, status);
1072         if (status != 0) {
1073                 task->tk_status = status;
1074                 return;
1075         }
1076         xprt_inject_disconnect(xprt);
1077
1078         dprintk("RPC: %5u xmit complete\n", task->tk_pid);
1079         task->tk_flags |= RPC_TASK_SENT;
1080         spin_lock_bh(&xprt->transport_lock);
1081
1082         xprt->ops->set_retrans_timeout(task);
1083
1084         xprt->stat.sends++;
1085         xprt->stat.req_u += xprt->stat.sends - xprt->stat.recvs;
1086         xprt->stat.bklog_u += xprt->backlog.qlen;
1087         xprt->stat.sending_u += xprt->sending.qlen;
1088         xprt->stat.pending_u += xprt->pending.qlen;
1089         spin_unlock_bh(&xprt->transport_lock);
1090
1091         req->rq_connect_cookie = connect_cookie;
1092         if (rpc_reply_expected(task) && !READ_ONCE(req->rq_reply_bytes_recvd)) {
1093                 /*
1094                  * Sleep on the pending queue if we're expecting a reply.
1095                  * The spinlock ensures atomicity between the test of
1096                  * req->rq_reply_bytes_recvd, and the call to rpc_sleep_on().
1097                  */
1098                 spin_lock(&xprt->recv_lock);
1099                 if (!req->rq_reply_bytes_recvd) {
1100                         rpc_sleep_on(&xprt->pending, task, xprt_timer);
1101                         /*
1102                          * Send an extra queue wakeup call if the
1103                          * connection was dropped in case the call to
1104                          * rpc_sleep_on() raced.
1105                          */
1106                         if (!xprt_connected(xprt))
1107                                 xprt_wake_pending_tasks(xprt, -ENOTCONN);
1108                 }
1109                 spin_unlock(&xprt->recv_lock);
1110         }
1111 }
1112
1113 static void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task)
1114 {
1115         set_bit(XPRT_CONGESTED, &xprt->state);
1116         rpc_sleep_on(&xprt->backlog, task, NULL);
1117 }
1118
1119 static void xprt_wake_up_backlog(struct rpc_xprt *xprt)
1120 {
1121         if (rpc_wake_up_next(&xprt->backlog) == NULL)
1122                 clear_bit(XPRT_CONGESTED, &xprt->state);
1123 }
1124
1125 static bool xprt_throttle_congested(struct rpc_xprt *xprt, struct rpc_task *task)
1126 {
1127         bool ret = false;
1128
1129         if (!test_bit(XPRT_CONGESTED, &xprt->state))
1130                 goto out;
1131         spin_lock(&xprt->reserve_lock);
1132         if (test_bit(XPRT_CONGESTED, &xprt->state)) {
1133                 rpc_sleep_on(&xprt->backlog, task, NULL);
1134                 ret = true;
1135         }
1136         spin_unlock(&xprt->reserve_lock);
1137 out:
1138         return ret;
1139 }
1140
1141 static struct rpc_rqst *xprt_dynamic_alloc_slot(struct rpc_xprt *xprt)
1142 {
1143         struct rpc_rqst *req = ERR_PTR(-EAGAIN);
1144
1145         if (xprt->num_reqs >= xprt->max_reqs)
1146                 goto out;
1147         ++xprt->num_reqs;
1148         spin_unlock(&xprt->reserve_lock);
1149         req = kzalloc(sizeof(struct rpc_rqst), GFP_NOFS);
1150         spin_lock(&xprt->reserve_lock);
1151         if (req != NULL)
1152                 goto out;
1153         --xprt->num_reqs;
1154         req = ERR_PTR(-ENOMEM);
1155 out:
1156         return req;
1157 }
1158
1159 static bool xprt_dynamic_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req)
1160 {
1161         if (xprt->num_reqs > xprt->min_reqs) {
1162                 --xprt->num_reqs;
1163                 kfree(req);
1164                 return true;
1165         }
1166         return false;
1167 }
1168
1169 void xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)
1170 {
1171         struct rpc_rqst *req;
1172
1173         spin_lock(&xprt->reserve_lock);
1174         if (!list_empty(&xprt->free)) {
1175                 req = list_entry(xprt->free.next, struct rpc_rqst, rq_list);
1176                 list_del(&req->rq_list);
1177                 goto out_init_req;
1178         }
1179         req = xprt_dynamic_alloc_slot(xprt);
1180         if (!IS_ERR(req))
1181                 goto out_init_req;
1182         switch (PTR_ERR(req)) {
1183         case -ENOMEM:
1184                 dprintk("RPC:       dynamic allocation of request slot "
1185                                 "failed! Retrying\n");
1186                 task->tk_status = -ENOMEM;
1187                 break;
1188         case -EAGAIN:
1189                 xprt_add_backlog(xprt, task);
1190                 dprintk("RPC:       waiting for request slot\n");
1191                 /* fall through */
1192         default:
1193                 task->tk_status = -EAGAIN;
1194         }
1195         spin_unlock(&xprt->reserve_lock);
1196         return;
1197 out_init_req:
1198         xprt->stat.max_slots = max_t(unsigned int, xprt->stat.max_slots,
1199                                      xprt->num_reqs);
1200         spin_unlock(&xprt->reserve_lock);
1201
1202         task->tk_status = 0;
1203         task->tk_rqstp = req;
1204 }
1205 EXPORT_SYMBOL_GPL(xprt_alloc_slot);
1206
1207 void xprt_lock_and_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)
1208 {
1209         /* Note: grabbing the xprt_lock_write() ensures that we throttle
1210          * new slot allocation if the transport is congested (i.e. when
1211          * reconnecting a stream transport or when out of socket write
1212          * buffer space).
1213          */
1214         if (xprt_lock_write(xprt, task)) {
1215                 xprt_alloc_slot(xprt, task);
1216                 xprt_release_write(xprt, task);
1217         }
1218 }
1219 EXPORT_SYMBOL_GPL(xprt_lock_and_alloc_slot);
1220
1221 void xprt_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req)
1222 {
1223         spin_lock(&xprt->reserve_lock);
1224         if (!xprt_dynamic_free_slot(xprt, req)) {
1225                 memset(req, 0, sizeof(*req));   /* mark unused */
1226                 list_add(&req->rq_list, &xprt->free);
1227         }
1228         xprt_wake_up_backlog(xprt);
1229         spin_unlock(&xprt->reserve_lock);
1230 }
1231 EXPORT_SYMBOL_GPL(xprt_free_slot);
1232
1233 static void xprt_free_all_slots(struct rpc_xprt *xprt)
1234 {
1235         struct rpc_rqst *req;
1236         while (!list_empty(&xprt->free)) {
1237                 req = list_first_entry(&xprt->free, struct rpc_rqst, rq_list);
1238                 list_del(&req->rq_list);
1239                 kfree(req);
1240         }
1241 }
1242
1243 struct rpc_xprt *xprt_alloc(struct net *net, size_t size,
1244                 unsigned int num_prealloc,
1245                 unsigned int max_alloc)
1246 {
1247         struct rpc_xprt *xprt;
1248         struct rpc_rqst *req;
1249         int i;
1250
1251         xprt = kzalloc(size, GFP_KERNEL);
1252         if (xprt == NULL)
1253                 goto out;
1254
1255         xprt_init(xprt, net);
1256
1257         for (i = 0; i < num_prealloc; i++) {
1258                 req = kzalloc(sizeof(struct rpc_rqst), GFP_KERNEL);
1259                 if (!req)
1260                         goto out_free;
1261                 list_add(&req->rq_list, &xprt->free);
1262         }
1263         if (max_alloc > num_prealloc)
1264                 xprt->max_reqs = max_alloc;
1265         else
1266                 xprt->max_reqs = num_prealloc;
1267         xprt->min_reqs = num_prealloc;
1268         xprt->num_reqs = num_prealloc;
1269
1270         return xprt;
1271
1272 out_free:
1273         xprt_free(xprt);
1274 out:
1275         return NULL;
1276 }
1277 EXPORT_SYMBOL_GPL(xprt_alloc);
1278
1279 void xprt_free(struct rpc_xprt *xprt)
1280 {
1281         put_net(xprt->xprt_net);
1282         xprt_free_all_slots(xprt);
1283         kfree_rcu(xprt, rcu);
1284 }
1285 EXPORT_SYMBOL_GPL(xprt_free);
1286
1287 static __be32
1288 xprt_alloc_xid(struct rpc_xprt *xprt)
1289 {
1290         __be32 xid;
1291
1292         spin_lock(&xprt->reserve_lock);
1293         xid = (__force __be32)xprt->xid++;
1294         spin_unlock(&xprt->reserve_lock);
1295         return xid;
1296 }
1297
1298 static void
1299 xprt_init_xid(struct rpc_xprt *xprt)
1300 {
1301         xprt->xid = prandom_u32();
1302 }
1303
1304 static void
1305 xprt_request_init(struct rpc_task *task)
1306 {
1307         struct rpc_xprt *xprt = task->tk_xprt;
1308         struct rpc_rqst *req = task->tk_rqstp;
1309
1310         INIT_LIST_HEAD(&req->rq_list);
1311         req->rq_timeout = task->tk_client->cl_timeout->to_initval;
1312         req->rq_task    = task;
1313         req->rq_xprt    = xprt;
1314         req->rq_buffer  = NULL;
1315         req->rq_xid     = xprt_alloc_xid(xprt);
1316         req->rq_connect_cookie = xprt->connect_cookie - 1;
1317         req->rq_bytes_sent = 0;
1318         req->rq_snd_buf.len = 0;
1319         req->rq_snd_buf.buflen = 0;
1320         req->rq_rcv_buf.len = 0;
1321         req->rq_rcv_buf.buflen = 0;
1322         req->rq_release_snd_buf = NULL;
1323         xprt_reset_majortimeo(req);
1324         dprintk("RPC: %5u reserved req %p xid %08x\n", task->tk_pid,
1325                         req, ntohl(req->rq_xid));
1326 }
1327
1328 static void
1329 xprt_do_reserve(struct rpc_xprt *xprt, struct rpc_task *task)
1330 {
1331         xprt->ops->alloc_slot(xprt, task);
1332         if (task->tk_rqstp != NULL)
1333                 xprt_request_init(task);
1334 }
1335
1336 /**
1337  * xprt_reserve - allocate an RPC request slot
1338  * @task: RPC task requesting a slot allocation
1339  *
1340  * If the transport is marked as being congested, or if no more
1341  * slots are available, place the task on the transport's
1342  * backlog queue.
1343  */
1344 void xprt_reserve(struct rpc_task *task)
1345 {
1346         struct rpc_xprt *xprt = task->tk_xprt;
1347
1348         task->tk_status = 0;
1349         if (task->tk_rqstp != NULL)
1350                 return;
1351
1352         task->tk_timeout = 0;
1353         task->tk_status = -EAGAIN;
1354         if (!xprt_throttle_congested(xprt, task))
1355                 xprt_do_reserve(xprt, task);
1356 }
1357
1358 /**
1359  * xprt_retry_reserve - allocate an RPC request slot
1360  * @task: RPC task requesting a slot allocation
1361  *
1362  * If no more slots are available, place the task on the transport's
1363  * backlog queue.
1364  * Note that the only difference with xprt_reserve is that we now
1365  * ignore the value of the XPRT_CONGESTED flag.
1366  */
1367 void xprt_retry_reserve(struct rpc_task *task)
1368 {
1369         struct rpc_xprt *xprt = task->tk_xprt;
1370
1371         task->tk_status = 0;
1372         if (task->tk_rqstp != NULL)
1373                 return;
1374
1375         task->tk_timeout = 0;
1376         task->tk_status = -EAGAIN;
1377         xprt_do_reserve(xprt, task);
1378 }
1379
1380 /**
1381  * xprt_release - release an RPC request slot
1382  * @task: task which is finished with the slot
1383  *
1384  */
1385 void xprt_release(struct rpc_task *task)
1386 {
1387         struct rpc_xprt *xprt;
1388         struct rpc_rqst *req = task->tk_rqstp;
1389
1390         if (req == NULL) {
1391                 if (task->tk_client) {
1392                         xprt = task->tk_xprt;
1393                         if (xprt->snd_task == task)
1394                                 xprt_release_write(xprt, task);
1395                 }
1396                 return;
1397         }
1398
1399         xprt = req->rq_xprt;
1400         if (task->tk_ops->rpc_count_stats != NULL)
1401                 task->tk_ops->rpc_count_stats(task, task->tk_calldata);
1402         else if (task->tk_client)
1403                 rpc_count_iostats(task, task->tk_client->cl_metrics);
1404         spin_lock(&xprt->recv_lock);
1405         if (!list_empty(&req->rq_list)) {
1406                 list_del_init(&req->rq_list);
1407                 xprt_wait_on_pinned_rqst(req);
1408         }
1409         spin_unlock(&xprt->recv_lock);
1410         spin_lock_bh(&xprt->transport_lock);
1411         xprt->ops->release_xprt(xprt, task);
1412         if (xprt->ops->release_request)
1413                 xprt->ops->release_request(task);
1414         xprt->last_used = jiffies;
1415         xprt_schedule_autodisconnect(xprt);
1416         spin_unlock_bh(&xprt->transport_lock);
1417         if (req->rq_buffer)
1418                 xprt->ops->buf_free(task);
1419         xprt_inject_disconnect(xprt);
1420         if (req->rq_cred != NULL)
1421                 put_rpccred(req->rq_cred);
1422         task->tk_rqstp = NULL;
1423         if (req->rq_release_snd_buf)
1424                 req->rq_release_snd_buf(req);
1425
1426         dprintk("RPC: %5u release request %p\n", task->tk_pid, req);
1427         if (likely(!bc_prealloc(req)))
1428                 xprt->ops->free_slot(xprt, req);
1429         else
1430                 xprt_free_bc_request(req);
1431 }
1432
1433 static void xprt_init(struct rpc_xprt *xprt, struct net *net)
1434 {
1435         kref_init(&xprt->kref);
1436
1437         spin_lock_init(&xprt->transport_lock);
1438         spin_lock_init(&xprt->reserve_lock);
1439         spin_lock_init(&xprt->recv_lock);
1440
1441         INIT_LIST_HEAD(&xprt->free);
1442         INIT_LIST_HEAD(&xprt->recv);
1443 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
1444         spin_lock_init(&xprt->bc_pa_lock);
1445         INIT_LIST_HEAD(&xprt->bc_pa_list);
1446 #endif /* CONFIG_SUNRPC_BACKCHANNEL */
1447         INIT_LIST_HEAD(&xprt->xprt_switch);
1448
1449         xprt->last_used = jiffies;
1450         xprt->cwnd = RPC_INITCWND;
1451         xprt->bind_index = 0;
1452
1453         rpc_init_wait_queue(&xprt->binding, "xprt_binding");
1454         rpc_init_wait_queue(&xprt->pending, "xprt_pending");
1455         rpc_init_priority_wait_queue(&xprt->sending, "xprt_sending");
1456         rpc_init_priority_wait_queue(&xprt->backlog, "xprt_backlog");
1457
1458         xprt_init_xid(xprt);
1459
1460         xprt->xprt_net = get_net(net);
1461 }
1462
1463 /**
1464  * xprt_create_transport - create an RPC transport
1465  * @args: rpc transport creation arguments
1466  *
1467  */
1468 struct rpc_xprt *xprt_create_transport(struct xprt_create *args)
1469 {
1470         struct rpc_xprt *xprt;
1471         struct xprt_class *t;
1472
1473         spin_lock(&xprt_list_lock);
1474         list_for_each_entry(t, &xprt_list, list) {
1475                 if (t->ident == args->ident) {
1476                         spin_unlock(&xprt_list_lock);
1477                         goto found;
1478                 }
1479         }
1480         spin_unlock(&xprt_list_lock);
1481         dprintk("RPC: transport (%d) not supported\n", args->ident);
1482         return ERR_PTR(-EIO);
1483
1484 found:
1485         xprt = t->setup(args);
1486         if (IS_ERR(xprt)) {
1487                 dprintk("RPC:       xprt_create_transport: failed, %ld\n",
1488                                 -PTR_ERR(xprt));
1489                 goto out;
1490         }
1491         if (args->flags & XPRT_CREATE_NO_IDLE_TIMEOUT)
1492                 xprt->idle_timeout = 0;
1493         INIT_WORK(&xprt->task_cleanup, xprt_autoclose);
1494         if (xprt_has_timer(xprt))
1495                 timer_setup(&xprt->timer, xprt_init_autodisconnect, 0);
1496         else
1497                 timer_setup(&xprt->timer, NULL, 0);
1498
1499         if (strlen(args->servername) > RPC_MAXNETNAMELEN) {
1500                 xprt_destroy(xprt);
1501                 return ERR_PTR(-EINVAL);
1502         }
1503         xprt->servername = kstrdup(args->servername, GFP_KERNEL);
1504         if (xprt->servername == NULL) {
1505                 xprt_destroy(xprt);
1506                 return ERR_PTR(-ENOMEM);
1507         }
1508
1509         rpc_xprt_debugfs_register(xprt);
1510
1511         dprintk("RPC:       created transport %p with %u slots\n", xprt,
1512                         xprt->max_reqs);
1513 out:
1514         return xprt;
1515 }
1516
1517 static void xprt_destroy_cb(struct work_struct *work)
1518 {
1519         struct rpc_xprt *xprt =
1520                 container_of(work, struct rpc_xprt, task_cleanup);
1521
1522         rpc_xprt_debugfs_unregister(xprt);
1523         rpc_destroy_wait_queue(&xprt->binding);
1524         rpc_destroy_wait_queue(&xprt->pending);
1525         rpc_destroy_wait_queue(&xprt->sending);
1526         rpc_destroy_wait_queue(&xprt->backlog);
1527         kfree(xprt->servername);
1528         /*
1529          * Tear down transport state and free the rpc_xprt
1530          */
1531         xprt->ops->destroy(xprt);
1532 }
1533
1534 /**
1535  * xprt_destroy - destroy an RPC transport, killing off all requests.
1536  * @xprt: transport to destroy
1537  *
1538  */
1539 static void xprt_destroy(struct rpc_xprt *xprt)
1540 {
1541         dprintk("RPC:       destroying transport %p\n", xprt);
1542
1543         /*
1544          * Exclude transport connect/disconnect handlers and autoclose
1545          */
1546         wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_UNINTERRUPTIBLE);
1547
1548         /*
1549          * xprt_schedule_autodisconnect() can run after XPRT_LOCKED
1550          * is cleared.  We use ->transport_lock to ensure the mod_timer()
1551          * can only run *before* del_time_sync(), never after.
1552          */
1553         spin_lock_bh(&xprt->transport_lock);
1554         del_timer_sync(&xprt->timer);
1555         spin_unlock_bh(&xprt->transport_lock);
1556
1557         /*
1558          * Destroy sockets etc from the system workqueue so they can
1559          * safely flush receive work running on rpciod.
1560          */
1561         INIT_WORK(&xprt->task_cleanup, xprt_destroy_cb);
1562         schedule_work(&xprt->task_cleanup);
1563 }
1564
1565 static void xprt_destroy_kref(struct kref *kref)
1566 {
1567         xprt_destroy(container_of(kref, struct rpc_xprt, kref));
1568 }
1569
1570 /**
1571  * xprt_get - return a reference to an RPC transport.
1572  * @xprt: pointer to the transport
1573  *
1574  */
1575 struct rpc_xprt *xprt_get(struct rpc_xprt *xprt)
1576 {
1577         if (xprt != NULL && kref_get_unless_zero(&xprt->kref))
1578                 return xprt;
1579         return NULL;
1580 }
1581 EXPORT_SYMBOL_GPL(xprt_get);
1582
1583 /**
1584  * xprt_put - release a reference to an RPC transport.
1585  * @xprt: pointer to the transport
1586  *
1587  */
1588 void xprt_put(struct rpc_xprt *xprt)
1589 {
1590         if (xprt != NULL)
1591                 kref_put(&xprt->kref, xprt_destroy_kref);
1592 }
1593 EXPORT_SYMBOL_GPL(xprt_put);