]>
Commit | Line | Data |
---|---|---|
d7e09d03 PT |
1 | /* |
2 | * GPL HEADER START | |
3 | * | |
4 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 only, | |
8 | * as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | * General Public License version 2 for more details (a copy is included | |
14 | * in the LICENSE file that accompanied this code). | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * version 2 along with this program; If not, see | |
18 | * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf | |
19 | * | |
20 | * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, | |
21 | * CA 95054 USA or visit www.sun.com if you need additional information or | |
22 | * have any questions. | |
23 | * | |
24 | * GPL HEADER END | |
25 | */ | |
26 | /* | |
27 | * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. | |
28 | * Use is subject to license terms. | |
29 | * | |
30 | * Copyright (c) 2012, Intel Corporation. | |
31 | */ | |
32 | /* | |
33 | * This file is part of Lustre, http://www.lustre.org/ | |
34 | * Lustre is a trademark of Sun Microsystems, Inc. | |
35 | * | |
36 | * lnet/lnet/peer.c | |
37 | */ | |
38 | ||
39 | #define DEBUG_SUBSYSTEM S_LNET | |
40 | ||
9fdaf8c0 | 41 | #include "../../include/linux/lnet/lib-lnet.h" |
d7e09d03 PT |
42 | |
43 | int | |
44 | lnet_peer_tables_create(void) | |
45 | { | |
7e7ab095 MS |
46 | struct lnet_peer_table *ptable; |
47 | struct list_head *hash; | |
48 | int i; | |
49 | int j; | |
d7e09d03 PT |
50 | |
51 | the_lnet.ln_peer_tables = cfs_percpt_alloc(lnet_cpt_table(), | |
52 | sizeof(*ptable)); | |
06ace26e | 53 | if (!the_lnet.ln_peer_tables) { |
d7e09d03 PT |
54 | CERROR("Failed to allocate cpu-partition peer tables\n"); |
55 | return -ENOMEM; | |
56 | } | |
57 | ||
58 | cfs_percpt_for_each(ptable, i, the_lnet.ln_peer_tables) { | |
59 | INIT_LIST_HEAD(&ptable->pt_deathrow); | |
60 | ||
61 | LIBCFS_CPT_ALLOC(hash, lnet_cpt_table(), i, | |
62 | LNET_PEER_HASH_SIZE * sizeof(*hash)); | |
06ace26e | 63 | if (!hash) { |
d7e09d03 PT |
64 | CERROR("Failed to create peer hash table\n"); |
65 | lnet_peer_tables_destroy(); | |
66 | return -ENOMEM; | |
67 | } | |
68 | ||
69 | for (j = 0; j < LNET_PEER_HASH_SIZE; j++) | |
70 | INIT_LIST_HEAD(&hash[j]); | |
71 | ptable->pt_hash = hash; /* sign of initialization */ | |
72 | } | |
73 | ||
74 | return 0; | |
75 | } | |
76 | ||
77 | void | |
78 | lnet_peer_tables_destroy(void) | |
79 | { | |
7e7ab095 MS |
80 | struct lnet_peer_table *ptable; |
81 | struct list_head *hash; | |
82 | int i; | |
83 | int j; | |
d7e09d03 | 84 | |
06ace26e | 85 | if (!the_lnet.ln_peer_tables) |
d7e09d03 PT |
86 | return; |
87 | ||
88 | cfs_percpt_for_each(ptable, i, the_lnet.ln_peer_tables) { | |
89 | hash = ptable->pt_hash; | |
06ace26e | 90 | if (!hash) /* not initialized */ |
d7e09d03 PT |
91 | break; |
92 | ||
93 | LASSERT(list_empty(&ptable->pt_deathrow)); | |
94 | ||
95 | ptable->pt_hash = NULL; | |
96 | for (j = 0; j < LNET_PEER_HASH_SIZE; j++) | |
97 | LASSERT(list_empty(&hash[j])); | |
98 | ||
99 | LIBCFS_FREE(hash, LNET_PEER_HASH_SIZE * sizeof(*hash)); | |
100 | } | |
101 | ||
102 | cfs_percpt_free(the_lnet.ln_peer_tables); | |
103 | the_lnet.ln_peer_tables = NULL; | |
104 | } | |
105 | ||
106 | void | |
107 | lnet_peer_tables_cleanup(void) | |
108 | { | |
7e7ab095 MS |
109 | struct lnet_peer_table *ptable; |
110 | int i; | |
111 | int j; | |
d7e09d03 PT |
112 | |
113 | LASSERT(the_lnet.ln_shutdown); /* i.e. no new peers */ | |
114 | ||
115 | cfs_percpt_for_each(ptable, i, the_lnet.ln_peer_tables) { | |
116 | lnet_net_lock(i); | |
117 | ||
118 | for (j = 0; j < LNET_PEER_HASH_SIZE; j++) { | |
119 | struct list_head *peers = &ptable->pt_hash[j]; | |
120 | ||
121 | while (!list_empty(peers)) { | |
122 | lnet_peer_t *lp = list_entry(peers->next, | |
123 | lnet_peer_t, | |
124 | lp_hashlist); | |
125 | list_del_init(&lp->lp_hashlist); | |
126 | /* lose hash table's ref */ | |
127 | lnet_peer_decref_locked(lp); | |
128 | } | |
129 | } | |
130 | ||
131 | lnet_net_unlock(i); | |
132 | } | |
133 | ||
134 | cfs_percpt_for_each(ptable, i, the_lnet.ln_peer_tables) { | |
f9541f82 | 135 | LIST_HEAD(deathrow); |
7e7ab095 | 136 | lnet_peer_t *lp; |
d7e09d03 PT |
137 | |
138 | lnet_net_lock(i); | |
139 | ||
5fd88337 | 140 | for (j = 3; ptable->pt_number; j++) { |
d7e09d03 PT |
141 | lnet_net_unlock(i); |
142 | ||
5fd88337 | 143 | if (!(j & (j - 1))) { |
d7e09d03 PT |
144 | CDEBUG(D_WARNING, |
145 | "Waiting for %d peers on peer table\n", | |
146 | ptable->pt_number); | |
147 | } | |
d3caf4d5 PT |
148 | set_current_state(TASK_UNINTERRUPTIBLE); |
149 | schedule_timeout(cfs_time_seconds(1) / 2); | |
d7e09d03 PT |
150 | lnet_net_lock(i); |
151 | } | |
152 | list_splice_init(&ptable->pt_deathrow, &deathrow); | |
153 | ||
154 | lnet_net_unlock(i); | |
155 | ||
156 | while (!list_empty(&deathrow)) { | |
157 | lp = list_entry(deathrow.next, | |
c314c319 | 158 | lnet_peer_t, lp_hashlist); |
d7e09d03 PT |
159 | list_del(&lp->lp_hashlist); |
160 | LIBCFS_FREE(lp, sizeof(*lp)); | |
161 | } | |
162 | } | |
163 | } | |
164 | ||
165 | void | |
166 | lnet_destroy_peer_locked(lnet_peer_t *lp) | |
167 | { | |
168 | struct lnet_peer_table *ptable; | |
169 | ||
5fd88337 JS |
170 | LASSERT(!lp->lp_refcount); |
171 | LASSERT(!lp->lp_rtr_refcount); | |
d7e09d03 PT |
172 | LASSERT(list_empty(&lp->lp_txq)); |
173 | LASSERT(list_empty(&lp->lp_hashlist)); | |
5fd88337 | 174 | LASSERT(!lp->lp_txqnob); |
d7e09d03 PT |
175 | |
176 | ptable = the_lnet.ln_peer_tables[lp->lp_cpt]; | |
177 | LASSERT(ptable->pt_number > 0); | |
178 | ptable->pt_number--; | |
179 | ||
180 | lnet_ni_decref_locked(lp->lp_ni, lp->lp_cpt); | |
181 | lp->lp_ni = NULL; | |
182 | ||
183 | list_add(&lp->lp_hashlist, &ptable->pt_deathrow); | |
184 | } | |
185 | ||
186 | lnet_peer_t * | |
187 | lnet_find_peer_locked(struct lnet_peer_table *ptable, lnet_nid_t nid) | |
188 | { | |
7e7ab095 MS |
189 | struct list_head *peers; |
190 | lnet_peer_t *lp; | |
d7e09d03 PT |
191 | |
192 | LASSERT(!the_lnet.ln_shutdown); | |
193 | ||
194 | peers = &ptable->pt_hash[lnet_nid2peerhash(nid)]; | |
195 | list_for_each_entry(lp, peers, lp_hashlist) { | |
196 | if (lp->lp_nid == nid) { | |
197 | lnet_peer_addref_locked(lp); | |
198 | return lp; | |
199 | } | |
200 | } | |
201 | ||
202 | return NULL; | |
203 | } | |
204 | ||
205 | int | |
206 | lnet_nid2peer_locked(lnet_peer_t **lpp, lnet_nid_t nid, int cpt) | |
207 | { | |
7e7ab095 MS |
208 | struct lnet_peer_table *ptable; |
209 | lnet_peer_t *lp = NULL; | |
210 | lnet_peer_t *lp2; | |
211 | int cpt2; | |
212 | int rc = 0; | |
d7e09d03 PT |
213 | |
214 | *lpp = NULL; | |
215 | if (the_lnet.ln_shutdown) /* it's shutting down */ | |
216 | return -ESHUTDOWN; | |
217 | ||
218 | /* cpt can be LNET_LOCK_EX if it's called from router functions */ | |
219 | cpt2 = cpt != LNET_LOCK_EX ? cpt : lnet_cpt_of_nid_locked(nid); | |
220 | ||
221 | ptable = the_lnet.ln_peer_tables[cpt2]; | |
222 | lp = lnet_find_peer_locked(ptable, nid); | |
06ace26e | 223 | if (lp) { |
d7e09d03 PT |
224 | *lpp = lp; |
225 | return 0; | |
226 | } | |
227 | ||
228 | if (!list_empty(&ptable->pt_deathrow)) { | |
229 | lp = list_entry(ptable->pt_deathrow.next, | |
c314c319 | 230 | lnet_peer_t, lp_hashlist); |
d7e09d03 PT |
231 | list_del(&lp->lp_hashlist); |
232 | } | |
233 | ||
234 | /* | |
235 | * take extra refcount in case another thread has shutdown LNet | |
236 | * and destroyed locks and peer-table before I finish the allocation | |
237 | */ | |
238 | ptable->pt_number++; | |
239 | lnet_net_unlock(cpt); | |
240 | ||
06ace26e | 241 | if (lp) |
d7e09d03 PT |
242 | memset(lp, 0, sizeof(*lp)); |
243 | else | |
244 | LIBCFS_CPT_ALLOC(lp, lnet_cpt_table(), cpt2, sizeof(*lp)); | |
245 | ||
06ace26e | 246 | if (!lp) { |
d7e09d03 PT |
247 | rc = -ENOMEM; |
248 | lnet_net_lock(cpt); | |
249 | goto out; | |
250 | } | |
251 | ||
252 | INIT_LIST_HEAD(&lp->lp_txq); | |
253 | INIT_LIST_HEAD(&lp->lp_rtrq); | |
254 | INIT_LIST_HEAD(&lp->lp_routes); | |
255 | ||
256 | lp->lp_notify = 0; | |
257 | lp->lp_notifylnd = 0; | |
258 | lp->lp_notifying = 0; | |
259 | lp->lp_alive_count = 0; | |
260 | lp->lp_timestamp = 0; | |
261 | lp->lp_alive = !lnet_peers_start_down(); /* 1 bit!! */ | |
262 | lp->lp_last_alive = cfs_time_current(); /* assumes alive */ | |
263 | lp->lp_last_query = 0; /* haven't asked NI yet */ | |
264 | lp->lp_ping_timestamp = 0; | |
265 | lp->lp_ping_feats = LNET_PING_FEAT_INVAL; | |
266 | lp->lp_nid = nid; | |
267 | lp->lp_cpt = cpt2; | |
268 | lp->lp_refcount = 2; /* 1 for caller; 1 for hash */ | |
269 | lp->lp_rtr_refcount = 0; | |
270 | ||
271 | lnet_net_lock(cpt); | |
272 | ||
273 | if (the_lnet.ln_shutdown) { | |
274 | rc = -ESHUTDOWN; | |
275 | goto out; | |
276 | } | |
277 | ||
278 | lp2 = lnet_find_peer_locked(ptable, nid); | |
06ace26e | 279 | if (lp2) { |
d7e09d03 PT |
280 | *lpp = lp2; |
281 | goto out; | |
282 | } | |
283 | ||
284 | lp->lp_ni = lnet_net2ni_locked(LNET_NIDNET(nid), cpt2); | |
06ace26e | 285 | if (!lp->lp_ni) { |
d7e09d03 PT |
286 | rc = -EHOSTUNREACH; |
287 | goto out; | |
288 | } | |
289 | ||
d3d3d37a JS |
290 | lp->lp_txcredits = lp->lp_ni->ni_peertxcredits; |
291 | lp->lp_mintxcredits = lp->lp_ni->ni_peertxcredits; | |
292 | lp->lp_rtrcredits = lnet_peer_buffer_credits(lp->lp_ni); | |
d7e09d03 PT |
293 | lp->lp_minrtrcredits = lnet_peer_buffer_credits(lp->lp_ni); |
294 | ||
295 | list_add_tail(&lp->lp_hashlist, | |
c314c319 | 296 | &ptable->pt_hash[lnet_nid2peerhash(nid)]); |
d7e09d03 PT |
297 | ptable->pt_version++; |
298 | *lpp = lp; | |
299 | ||
300 | return 0; | |
301 | out: | |
06ace26e | 302 | if (lp) |
d7e09d03 PT |
303 | list_add(&lp->lp_hashlist, &ptable->pt_deathrow); |
304 | ptable->pt_number--; | |
305 | return rc; | |
306 | } | |
307 | ||
308 | void | |
309 | lnet_debug_peer(lnet_nid_t nid) | |
310 | { | |
7e7ab095 MS |
311 | char *aliveness = "NA"; |
312 | lnet_peer_t *lp; | |
313 | int rc; | |
314 | int cpt; | |
d7e09d03 PT |
315 | |
316 | cpt = lnet_cpt_of_nid(nid); | |
317 | lnet_net_lock(cpt); | |
318 | ||
319 | rc = lnet_nid2peer_locked(&lp, nid, cpt); | |
5fd88337 | 320 | if (rc) { |
d7e09d03 PT |
321 | lnet_net_unlock(cpt); |
322 | CDEBUG(D_WARNING, "No peer %s\n", libcfs_nid2str(nid)); | |
323 | return; | |
324 | } | |
325 | ||
326 | if (lnet_isrouter(lp) || lnet_peer_aliveness_enabled(lp)) | |
327 | aliveness = lp->lp_alive ? "up" : "down"; | |
328 | ||
329 | CDEBUG(D_WARNING, "%-24s %4d %5s %5d %5d %5d %5d %5d %ld\n", | |
330 | libcfs_nid2str(lp->lp_nid), lp->lp_refcount, | |
331 | aliveness, lp->lp_ni->ni_peertxcredits, | |
332 | lp->lp_rtrcredits, lp->lp_minrtrcredits, | |
333 | lp->lp_txcredits, lp->lp_mintxcredits, lp->lp_txqnob); | |
334 | ||
335 | lnet_peer_decref_locked(lp); | |
336 | ||
337 | lnet_net_unlock(cpt); | |
338 | } |