]> git.proxmox.com Git - mirror_frr.git/blame - nhrpd/vici.c
*: Convert thread_add_XXX functions to event_add_XXX
[mirror_frr.git] / nhrpd / vici.c
CommitLineData
acddc0ed 1// SPDX-License-Identifier: GPL-2.0-or-later
2fb975da
TT
2/* strongSwan VICI protocol implementation for NHRP
3 * Copyright (c) 2014-2015 Timo Teräs
2fb975da
TT
4 */
5
b45ac5f5
DL
6#ifdef HAVE_CONFIG_H
7#include "config.h"
8#endif
9
2fb975da
TT
10#include <string.h>
11#include <sys/socket.h>
12#include <sys/un.h>
13
cb37cb33 14#include "event.h"
2fb975da
TT
15#include "zbuf.h"
16#include "log.h"
aed07011 17#include "lib_errors.h"
2fb975da 18
aed07011 19#include "nhrpd.h"
2fb975da 20#include "vici.h"
aed07011 21#include "nhrp_errors.h"
2fb975da
TT
22
23#define ERRNO_IO_RETRY(EN) (((EN) == EAGAIN) || ((EN) == EWOULDBLOCK) || ((EN) == EINTR))
24
25struct blob {
26 char *ptr;
27 int len;
28};
29
30static int blob_equal(const struct blob *b, const char *str)
31{
996c9314
LB
32 if (!b || b->len != (int)strlen(str))
33 return 0;
2fb975da
TT
34 return memcmp(b->ptr, str, b->len) == 0;
35}
36
37static int blob2buf(const struct blob *b, char *buf, size_t n)
38{
996c9314
LB
39 if (!b || b->len >= (int)n)
40 return 0;
2fb975da
TT
41 memcpy(buf, b->ptr, b->len);
42 buf[b->len] = 0;
43 return 1;
44}
45
46struct vici_conn {
e6685141 47 struct event *t_reconnect, *t_read, *t_write;
2fb975da
TT
48 struct zbuf ibuf;
49 struct zbuf_queue obuf;
50 int fd;
51 uint8_t ibuf_data[VICI_MAX_MSGLEN];
52};
53
54struct vici_message_ctx {
55 const char *sections[8];
56 int nsections;
57};
58
e6685141 59static void vici_reconnect(struct event *t);
2fb975da
TT
60static void vici_submit_request(struct vici_conn *vici, const char *name, ...);
61
62static void vici_zbuf_puts(struct zbuf *obuf, const char *str)
63{
64 size_t len = strlen(str);
65 zbuf_put8(obuf, len);
66 zbuf_put(obuf, str, len);
67}
68
69static void vici_connection_error(struct vici_conn *vici)
70{
71 nhrp_vc_reset();
72
73 THREAD_OFF(vici->t_read);
74 THREAD_OFF(vici->t_write);
75 zbuf_reset(&vici->ibuf);
76 zbufq_reset(&vici->obuf);
77
78 close(vici->fd);
79 vici->fd = -1;
907a2395 80 event_add_timer(master, vici_reconnect, vici, 2, &vici->t_reconnect);
2fb975da
TT
81}
82
996c9314
LB
83static void vici_parse_message(struct vici_conn *vici, struct zbuf *msg,
84 void (*parser)(struct vici_message_ctx *ctx,
85 enum vici_type_t msgtype,
86 const struct blob *key,
87 const struct blob *val),
88 struct vici_message_ctx *ctx)
2fb975da
TT
89{
90 uint8_t *type;
996c9314
LB
91 struct blob key = {0};
92 struct blob val = {0};
2fb975da
TT
93
94 while ((type = zbuf_may_pull(msg, uint8_t)) != NULL) {
95 switch (*type) {
96 case VICI_SECTION_START:
97 key.len = zbuf_get8(msg);
98 key.ptr = zbuf_pulln(msg, key.len);
996c9314
LB
99 debugf(NHRP_DEBUG_VICI, "VICI: Section start '%.*s'",
100 key.len, key.ptr);
2fb975da
TT
101 parser(ctx, *type, &key, NULL);
102 ctx->nsections++;
103 break;
104 case VICI_SECTION_END:
105 debugf(NHRP_DEBUG_VICI, "VICI: Section end");
106 parser(ctx, *type, NULL, NULL);
107 ctx->nsections--;
108 break;
109 case VICI_KEY_VALUE:
110 key.len = zbuf_get8(msg);
111 key.ptr = zbuf_pulln(msg, key.len);
112 val.len = zbuf_get_be16(msg);
113 val.ptr = zbuf_pulln(msg, val.len);
996c9314
LB
114 debugf(NHRP_DEBUG_VICI, "VICI: Key '%.*s'='%.*s'",
115 key.len, key.ptr, val.len, val.ptr);
2fb975da
TT
116 parser(ctx, *type, &key, &val);
117 break;
118 case VICI_LIST_START:
119 key.len = zbuf_get8(msg);
120 key.ptr = zbuf_pulln(msg, key.len);
996c9314
LB
121 debugf(NHRP_DEBUG_VICI, "VICI: List start '%.*s'",
122 key.len, key.ptr);
2fb975da
TT
123 break;
124 case VICI_LIST_ITEM:
125 val.len = zbuf_get_be16(msg);
126 val.ptr = zbuf_pulln(msg, val.len);
996c9314
LB
127 debugf(NHRP_DEBUG_VICI, "VICI: List item: '%.*s'",
128 val.len, val.ptr);
2fb975da
TT
129 parser(ctx, *type, &key, &val);
130 break;
131 case VICI_LIST_END:
132 debugf(NHRP_DEBUG_VICI, "VICI: List end");
133 break;
2fb975da
TT
134 }
135 }
136}
137
138struct handle_sa_ctx {
139 struct vici_message_ctx msgctx;
140 int event;
141 int child_ok;
142 int kill_ikesa;
143 uint32_t child_uniqueid, ike_uniqueid;
144 struct {
145 union sockunion host;
146 struct blob id, cert;
147 } local, remote;
148};
149
996c9314
LB
150static void parse_sa_message(struct vici_message_ctx *ctx,
151 enum vici_type_t msgtype, const struct blob *key,
152 const struct blob *val)
2fb975da 153{
996c9314
LB
154 struct handle_sa_ctx *sactx =
155 container_of(ctx, struct handle_sa_ctx, msgctx);
2fb975da
TT
156 struct nhrp_vc *vc;
157 char buf[512];
158
159 switch (msgtype) {
160 case VICI_SECTION_START:
161 if (ctx->nsections == 3) {
162 /* Begin of child-sa section, reset child vars */
163 sactx->child_uniqueid = 0;
164 sactx->child_ok = 0;
165 }
166 break;
167 case VICI_SECTION_END:
168 if (ctx->nsections == 3) {
169 /* End of child-sa section, update nhrp_vc */
170 int up = sactx->child_ok || sactx->event == 1;
171 if (up) {
996c9314
LB
172 vc = nhrp_vc_get(&sactx->local.host,
173 &sactx->remote.host, up);
2fb975da 174 if (vc) {
996c9314
LB
175 blob2buf(&sactx->local.id, vc->local.id,
176 sizeof(vc->local.id));
177 if (blob2buf(&sactx->local.cert,
178 (char *)vc->local.cert,
179 sizeof(vc->local.cert)))
180 vc->local.certlen =
181 sactx->local.cert.len;
182 blob2buf(&sactx->remote.id,
183 vc->remote.id,
184 sizeof(vc->remote.id));
185 if (blob2buf(&sactx->remote.cert,
186 (char *)vc->remote.cert,
187 sizeof(vc->remote.cert)))
188 vc->remote.certlen =
189 sactx->remote.cert.len;
190 sactx->kill_ikesa |=
191 nhrp_vc_ipsec_updown(
192 sactx->child_uniqueid,
193 vc);
4cbaf956 194 vc->ike_uniqueid = sactx->ike_uniqueid;
2fb975da
TT
195 }
196 } else {
197 nhrp_vc_ipsec_updown(sactx->child_uniqueid, 0);
198 }
199 }
200 break;
d0038397
DS
201 case VICI_START:
202 case VICI_KEY_VALUE:
203 case VICI_LIST_START:
204 case VICI_LIST_ITEM:
205 case VICI_LIST_END:
206 case VICI_END:
fa3bf3a2 207 if (!key || !key->ptr)
6b07f6e1
JB
208 break;
209
2fb975da
TT
210 switch (key->ptr[0]) {
211 case 'l':
996c9314
LB
212 if (blob_equal(key, "local-host")
213 && ctx->nsections == 1) {
2fb975da 214 if (blob2buf(val, buf, sizeof(buf)))
996c9314
LB
215 if (str2sockunion(buf,
216 &sactx->local.host)
217 < 0)
1c50c1c0
QY
218 flog_err(
219 EC_NHRP_SWAN,
220 "VICI: bad strongSwan local-host: %s",
221 buf);
996c9314
LB
222 } else if (blob_equal(key, "local-id")
223 && ctx->nsections == 1) {
2fb975da 224 sactx->local.id = *val;
996c9314
LB
225 } else if (blob_equal(key, "local-cert-data")
226 && ctx->nsections == 1) {
2fb975da
TT
227 sactx->local.cert = *val;
228 }
229 break;
230 case 'r':
996c9314
LB
231 if (blob_equal(key, "remote-host")
232 && ctx->nsections == 1) {
2fb975da 233 if (blob2buf(val, buf, sizeof(buf)))
996c9314
LB
234 if (str2sockunion(buf,
235 &sactx->remote.host)
236 < 0)
1c50c1c0
QY
237 flog_err(
238 EC_NHRP_SWAN,
239 "VICI: bad strongSwan remote-host: %s",
240 buf);
996c9314
LB
241 } else if (blob_equal(key, "remote-id")
242 && ctx->nsections == 1) {
2fb975da 243 sactx->remote.id = *val;
996c9314
LB
244 } else if (blob_equal(key, "remote-cert-data")
245 && ctx->nsections == 1) {
2fb975da
TT
246 sactx->remote.cert = *val;
247 }
248 break;
249 case 'u':
996c9314
LB
250 if (blob_equal(key, "uniqueid")
251 && blob2buf(val, buf, sizeof(buf))) {
2fb975da 252 if (ctx->nsections == 3)
996c9314
LB
253 sactx->child_uniqueid =
254 strtoul(buf, NULL, 0);
2fb975da 255 else if (ctx->nsections == 1)
996c9314
LB
256 sactx->ike_uniqueid =
257 strtoul(buf, NULL, 0);
2fb975da
TT
258 }
259 break;
260 case 's':
261 if (blob_equal(key, "state") && ctx->nsections == 3) {
262 sactx->child_ok =
996c9314
LB
263 (sactx->event == 0
264 && (blob_equal(val, "INSTALLED")
265 || blob_equal(val, "REKEYED")));
2fb975da
TT
266 }
267 break;
268 }
269 break;
270 }
271}
272
996c9314
LB
273static void parse_cmd_response(struct vici_message_ctx *ctx,
274 enum vici_type_t msgtype, const struct blob *key,
275 const struct blob *val)
d139786a
TT
276{
277 char buf[512];
278
279 switch (msgtype) {
280 case VICI_KEY_VALUE:
996c9314
LB
281 if (blob_equal(key, "errmsg")
282 && blob2buf(val, buf, sizeof(buf)))
2b84a521 283 flog_err(EC_NHRP_SWAN, "VICI: strongSwan: %s", buf);
d139786a 284 break;
d0038397
DS
285 case VICI_START:
286 case VICI_SECTION_START:
287 case VICI_SECTION_END:
288 case VICI_LIST_START:
289 case VICI_LIST_ITEM:
290 case VICI_LIST_END:
291 case VICI_END:
d139786a
TT
292 break;
293 }
294}
295
2fb975da
TT
296static void vici_recv_sa(struct vici_conn *vici, struct zbuf *msg, int event)
297{
298 char buf[32];
299 struct handle_sa_ctx ctx = {
300 .event = event,
0a939f4f 301 .msgctx.nsections = 0
2fb975da
TT
302 };
303
304 vici_parse_message(vici, msg, parse_sa_message, &ctx.msgctx);
305
306 if (ctx.kill_ikesa && ctx.ike_uniqueid) {
996c9314
LB
307 debugf(NHRP_DEBUG_COMMON, "VICI: Deleting IKE_SA %u",
308 ctx.ike_uniqueid);
0d6f7fd6 309 snprintf(buf, sizeof(buf), "%u", ctx.ike_uniqueid);
996c9314
LB
310 vici_submit_request(vici, "terminate", VICI_KEY_VALUE, "ike-id",
311 strlen(buf), buf, VICI_END);
2fb975da
TT
312 }
313}
314
315static void vici_recv_message(struct vici_conn *vici, struct zbuf *msg)
316{
317 uint32_t msglen;
318 uint8_t msgtype;
319 struct blob name;
7ea5df54 320 struct vici_message_ctx ctx = { .nsections = 0 };
2fb975da
TT
321
322 msglen = zbuf_get_be32(msg);
323 msgtype = zbuf_get8(msg);
324 debugf(NHRP_DEBUG_VICI, "VICI: Message %d, %d bytes", msgtype, msglen);
325
326 switch (msgtype) {
327 case VICI_EVENT:
328 name.len = zbuf_get8(msg);
329 name.ptr = zbuf_pulln(msg, name.len);
330
996c9314
LB
331 debugf(NHRP_DEBUG_VICI, "VICI: Event '%.*s'", name.len,
332 name.ptr);
333 if (blob_equal(&name, "list-sa")
334 || blob_equal(&name, "child-updown")
335 || blob_equal(&name, "child-rekey"))
2fb975da 336 vici_recv_sa(vici, msg, 0);
996c9314
LB
337 else if (blob_equal(&name, "child-state-installed")
338 || blob_equal(&name, "child-state-rekeyed"))
2fb975da
TT
339 vici_recv_sa(vici, msg, 1);
340 else if (blob_equal(&name, "child-state-destroying"))
341 vici_recv_sa(vici, msg, 2);
342 break;
d139786a 343 case VICI_CMD_RESPONSE:
6c8ca260 344 vici_parse_message(vici, msg, parse_cmd_response, &ctx);
d139786a 345 break;
2fb975da 346 case VICI_EVENT_UNKNOWN:
d139786a 347 case VICI_CMD_UNKNOWN:
1c50c1c0
QY
348 flog_err(
349 EC_NHRP_SWAN,
996c9314 350 "VICI: StrongSwan does not support mandatory events (unpatched?)");
2fb975da
TT
351 break;
352 case VICI_EVENT_CONFIRM:
2fb975da
TT
353 break;
354 default:
355 zlog_notice("VICI: Unrecognized message type %d", msgtype);
356 break;
357 }
358}
359
e6685141 360static void vici_read(struct event *t)
2fb975da
TT
361{
362 struct vici_conn *vici = THREAD_ARG(t);
363 struct zbuf *ibuf = &vici->ibuf;
364 struct zbuf pktbuf;
365
996c9314 366 if (zbuf_read(ibuf, vici->fd, (size_t)-1) < 0) {
2fb975da 367 vici_connection_error(vici);
cc9f21da 368 return;
2fb975da
TT
369 }
370
371 /* Process all messages in buffer */
372 do {
373 uint32_t *hdrlen = zbuf_may_pull(ibuf, uint32_t);
374 if (!hdrlen)
375 break;
376 if (!zbuf_may_pulln(ibuf, ntohl(*hdrlen))) {
377 zbuf_reset_head(ibuf, hdrlen);
378 break;
379 }
380
381 /* Handle packet */
996c9314
LB
382 zbuf_init(&pktbuf, hdrlen, htonl(*hdrlen) + 4,
383 htonl(*hdrlen) + 4);
2fb975da
TT
384 vici_recv_message(vici, &pktbuf);
385 } while (1);
386
907a2395 387 event_add_read(master, vici_read, vici, vici->fd, &vici->t_read);
2fb975da
TT
388}
389
e6685141 390static void vici_write(struct event *t)
2fb975da
TT
391{
392 struct vici_conn *vici = THREAD_ARG(t);
393 int r;
394
2fb975da
TT
395 r = zbufq_write(&vici->obuf, vici->fd);
396 if (r > 0) {
907a2395
DS
397 event_add_write(master, vici_write, vici, vici->fd,
398 &vici->t_write);
2fb975da
TT
399 } else if (r < 0) {
400 vici_connection_error(vici);
401 }
2fb975da
TT
402}
403
404static void vici_submit(struct vici_conn *vici, struct zbuf *obuf)
405{
406 if (vici->fd < 0) {
407 zbuf_free(obuf);
408 return;
409 }
410
411 zbufq_queue(&vici->obuf, obuf);
907a2395 412 event_add_write(master, vici_write, vici, vici->fd, &vici->t_write);
2fb975da
TT
413}
414
415static void vici_submit_request(struct vici_conn *vici, const char *name, ...)
416{
417 struct zbuf *obuf;
418 uint32_t *hdrlen;
419 va_list va;
420 size_t len;
421 int type;
422
423 obuf = zbuf_alloc(256);
996c9314
LB
424 if (!obuf)
425 return;
2fb975da
TT
426
427 hdrlen = zbuf_push(obuf, uint32_t);
428 zbuf_put8(obuf, VICI_CMD_REQUEST);
429 vici_zbuf_puts(obuf, name);
430
431 va_start(va, name);
432 for (type = va_arg(va, int); type != VICI_END; type = va_arg(va, int)) {
433 zbuf_put8(obuf, type);
434 switch (type) {
435 case VICI_KEY_VALUE:
436 vici_zbuf_puts(obuf, va_arg(va, const char *));
437 len = va_arg(va, size_t);
438 zbuf_put_be16(obuf, len);
439 zbuf_put(obuf, va_arg(va, void *), len);
440 break;
2fb975da
TT
441 default:
442 break;
443 }
444 }
445 va_end(va);
446 *hdrlen = htonl(zbuf_used(obuf) - 4);
447 vici_submit(vici, obuf);
448}
449
450static void vici_register_event(struct vici_conn *vici, const char *name)
451{
452 struct zbuf *obuf;
453 uint32_t *hdrlen;
454 uint8_t namelen;
455
456 namelen = strlen(name);
457 obuf = zbuf_alloc(4 + 1 + 1 + namelen);
996c9314
LB
458 if (!obuf)
459 return;
2fb975da
TT
460
461 hdrlen = zbuf_push(obuf, uint32_t);
462 zbuf_put8(obuf, VICI_EVENT_REGISTER);
463 zbuf_put8(obuf, namelen);
464 zbuf_put(obuf, name, namelen);
465 *hdrlen = htonl(zbuf_used(obuf) - 4);
466
467 vici_submit(vici, obuf);
468}
469
40307370
PG
470static bool vici_charon_filepath_done;
471static bool vici_charon_not_found;
472
473static char *vici_get_charon_filepath(void)
474{
475 static char buff[1200];
476 FILE *fp;
477 char *ptr;
478 char line[1024];
479
480 if (vici_charon_filepath_done)
481 return (char *)buff;
482 fp = popen("ipsec --piddir", "r");
483 if (!fp) {
484 if (!vici_charon_not_found) {
485 flog_err(EC_NHRP_SWAN,
486 "VICI: Failed to retrieve charon file path");
487 vici_charon_not_found = true;
488 }
489 return NULL;
490 }
491 /* last line of output is used to get vici path */
492 while (fgets(line, sizeof(line), fp) != NULL) {
493 ptr = strchr(line, '\n');
494 if (ptr)
495 *ptr = '\0';
496 snprintf(buff, sizeof(buff), "%s/charon.vici", line);
497 }
498 pclose(fp);
499 vici_charon_filepath_done = true;
500 return buff;
501}
502
e6685141 503static void vici_reconnect(struct event *t)
2fb975da
TT
504{
505 struct vici_conn *vici = THREAD_ARG(t);
506 int fd;
40307370 507 char *file_path;
2fb975da 508
996c9314 509 if (vici->fd >= 0)
cc9f21da 510 return;
2fb975da 511
354196c0 512 fd = sock_open_unix(VICI_SOCKET);
40307370
PG
513 if (fd < 0) {
514 file_path = vici_get_charon_filepath();
515 if (file_path)
516 fd = sock_open_unix(file_path);
517 }
2fb975da 518 if (fd < 0) {
996c9314 519 debugf(NHRP_DEBUG_VICI,
15569c58
DA
520 "%s: failure connecting VICI socket: %s", __func__,
521 strerror(errno));
907a2395
DS
522 event_add_timer(master, vici_reconnect, vici, 2,
523 &vici->t_reconnect);
cc9f21da 524 return;
2fb975da
TT
525 }
526
527 debugf(NHRP_DEBUG_COMMON, "VICI: Connected");
528 vici->fd = fd;
907a2395 529 event_add_read(master, vici_read, vici, vici->fd, &vici->t_read);
2fb975da
TT
530
531 /* Send event subscribtions */
996c9314
LB
532 // vici_register_event(vici, "child-updown");
533 // vici_register_event(vici, "child-rekey");
2fb975da
TT
534 vici_register_event(vici, "child-state-installed");
535 vici_register_event(vici, "child-state-rekeyed");
536 vici_register_event(vici, "child-state-destroying");
537 vici_register_event(vici, "list-sa");
538 vici_submit_request(vici, "list-sas", VICI_END);
2fb975da
TT
539}
540
541static struct vici_conn vici_connection;
542
543void vici_init(void)
544{
545 struct vici_conn *vici = &vici_connection;
546
547 vici->fd = -1;
548 zbuf_init(&vici->ibuf, vici->ibuf_data, sizeof(vici->ibuf_data), 0);
549 zbufq_init(&vici->obuf);
907a2395
DS
550 event_add_timer_msec(master, vici_reconnect, vici, 10,
551 &vici->t_reconnect);
2fb975da
TT
552}
553
554void vici_terminate(void)
555{
556}
557
083bbfae
GG
558void vici_terminate_vc_by_profile_name(char *profile_name)
559{
560 struct vici_conn *vici = &vici_connection;
74e5ba3a 561
58ef1668 562 debugf(NHRP_DEBUG_VICI, "Terminate profile = %s", profile_name);
083bbfae
GG
563 vici_submit_request(vici, "terminate", VICI_KEY_VALUE, "ike",
564 strlen(profile_name), profile_name, VICI_END);
565}
566
567void vici_terminate_vc_by_ike_id(unsigned int ike_id)
4cbaf956
GG
568{
569 struct vici_conn *vici = &vici_connection;
74e5ba3a
RD
570 char ike_id_str[10];
571
4cbaf956 572 snprintf(ike_id_str, sizeof(ike_id_str), "%d", ike_id);
58ef1668 573 debugf(NHRP_DEBUG_VICI, "Terminate ike_id_str = %s", ike_id_str);
4cbaf956
GG
574 vici_submit_request(vici, "terminate", VICI_KEY_VALUE, "ike-id",
575 strlen(ike_id_str), ike_id_str, VICI_END);
576}
577
996c9314
LB
578void vici_request_vc(const char *profile, union sockunion *src,
579 union sockunion *dst, int prio)
2fb975da
TT
580{
581 struct vici_conn *vici = &vici_connection;
582 char buf[2][SU_ADDRSTRLEN];
583
0d6f7fd6
DA
584 sockunion2str(src, buf[0], sizeof(buf[0]));
585 sockunion2str(dst, buf[1], sizeof(buf[1]));
2fb975da 586
996c9314
LB
587 vici_submit_request(vici, "initiate", VICI_KEY_VALUE, "child",
588 strlen(profile), profile, VICI_KEY_VALUE, "timeout",
589 (size_t)2, "-1", VICI_KEY_VALUE, "async", (size_t)1,
590 "1", VICI_KEY_VALUE, "init-limits", (size_t)1,
591 prio ? "0" : "1", VICI_KEY_VALUE, "my-host",
592 strlen(buf[0]), buf[0], VICI_KEY_VALUE,
593 "other-host", strlen(buf[1]), buf[1], VICI_END);
2fb975da
TT
594}
595
596int sock_open_unix(const char *path)
597{
598 int ret, fd;
599 struct sockaddr_un addr;
600
601 fd = socket(AF_UNIX, SOCK_STREAM, 0);
602 if (fd < 0)
603 return -1;
604
6006b807 605 memset(&addr, 0, sizeof(addr));
2fb975da 606 addr.sun_family = AF_UNIX;
fa3bf3a2 607 strlcpy(addr.sun_path, path, sizeof(addr.sun_path));
2fb975da 608
996c9314
LB
609 ret = connect(fd, (struct sockaddr *)&addr,
610 sizeof(addr.sun_family) + strlen(addr.sun_path));
2fb975da
TT
611 if (ret < 0) {
612 close(fd);
613 return -1;
614 }
615
6c8ca260
JB
616 ret = fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
617 if (ret < 0) {
618 close(fd);
619 return -1;
620 }
2fb975da
TT
621
622 return fd;
623}