1 // Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
3 // Use, modification and distribution is subject to the Boost Software
4 // License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
5 // http://www.boost.org/LICENSE_1_0.txt)
7 /** @file nonblocking.hpp
9 * This header defines operations for completing non-blocking
10 * communication requests.
12 #ifndef BOOST_MPI_NONBLOCKING_HPP
13 #define BOOST_MPI_NONBLOCKING_HPP
15 #include <boost/mpi/config.hpp>
17 #include <iterator> // for std::iterator_traits
18 #include <boost/optional.hpp>
19 #include <utility> // for std::pair
20 #include <algorithm> // for iter_swap, reverse
21 #include <boost/static_assert.hpp>
22 #include <boost/mpi/request.hpp>
23 #include <boost/mpi/status.hpp>
24 #include <boost/mpi/exception.hpp>
26 namespace boost { namespace mpi {
29 * @brief Wait until any non-blocking request has completed.
31 * This routine takes in a set of requests stored in the iterator
32 * range @c [first,last) and waits until any of these requests has
33 * been completed. It provides functionality equivalent to
36 * @param first The iterator that denotes the beginning of the
37 * sequence of request objects.
39 * @param last The iterator that denotes the end of the sequence of
40 * request objects. This may not be equal to @c first.
42 * @returns A pair containing the status object that corresponds to
43 * the completed operation and the iterator referencing the completed
46 template<typename ForwardIterator>
47 std::pair<status, ForwardIterator>
48 wait_any(ForwardIterator first, ForwardIterator last)
52 BOOST_ASSERT(first != last);
54 typedef typename std::iterator_traits<ForwardIterator>::difference_type
57 bool all_trivial_requests = true;
58 difference_type n = 0;
59 ForwardIterator current = first;
61 // Check if we have found a completed request. If so, return it.
62 if (current->m_requests[0] != MPI_REQUEST_NULL &&
63 (current->m_requests[1] != MPI_REQUEST_NULL ||
64 current->m_handler)) {
65 if (optional<status> result = current->test())
66 return std::make_pair(*result, current);
69 // Check if this request (and all others before it) are "trivial"
70 // requests, e.g., they can be represented with a single
72 all_trivial_requests =
74 && !current->m_handler
75 && current->m_requests[1] == MPI_REQUEST_NULL;
77 // Move to the next request.
79 if (++current == last) {
80 // We have reached the end of the list. If all requests thus far
81 // have been trivial, we can call MPI_Waitany directly, because
82 // it may be more efficient than our busy-wait semantics.
83 if (all_trivial_requests) {
84 std::vector<MPI_Request> requests;
86 for (current = first; current != last; ++current)
87 requests.push_back(current->m_requests[0]);
89 // Let MPI wait until one of these operations completes.
92 BOOST_MPI_CHECK_RESULT(MPI_Waitany,
93 (n, &requests[0], &index, &stat.m_status));
95 // We don't have a notion of empty requests or status objects,
96 // so this is an error.
97 if (index == MPI_UNDEFINED)
98 boost::throw_exception(exception("MPI_Waitany", MPI_ERR_REQUEST));
100 // Find the iterator corresponding to the completed request.
102 advance(current, index);
103 current->m_requests[0] = requests[index];
104 return std::make_pair(stat, current);
107 // There are some nontrivial requests, so we must continue our
108 // busy waiting loop.
111 all_trivial_requests = true;
115 // We cannot ever get here
120 * @brief Test whether any non-blocking request has completed.
122 * This routine takes in a set of requests stored in the iterator
123 * range @c [first,last) and tests whether any of these requests has
124 * been completed. This routine is similar to @c wait_any, but will
125 * not block waiting for requests to completed. It provides
126 * functionality equivalent to @c MPI_Testany.
128 * @param first The iterator that denotes the beginning of the
129 * sequence of request objects.
131 * @param last The iterator that denotes the end of the sequence of
134 * @returns If any outstanding requests have completed, a pair
135 * containing the status object that corresponds to the completed
136 * operation and the iterator referencing the completed
137 * request. Otherwise, an empty @c optional<>.
139 template<typename ForwardIterator>
140 optional<std::pair<status, ForwardIterator> >
141 test_any(ForwardIterator first, ForwardIterator last)
143 while (first != last) {
144 // Check if we have found a completed request. If so, return it.
145 if (optional<status> result = first->test()) {
146 return std::make_pair(*result, first);
152 return optional<std::pair<status, ForwardIterator> >();
156 * @brief Wait until all non-blocking requests have completed.
158 * This routine takes in a set of requests stored in the iterator
159 * range @c [first,last) and waits until all of these requests have
160 * been completed. It provides functionality equivalent to
163 * @param first The iterator that denotes the beginning of the
164 * sequence of request objects.
166 * @param last The iterator that denotes the end of the sequence of
169 * @param out If provided, an output iterator through which the
170 * status of each request will be emitted. The @c status objects are
171 * emitted in the same order as the requests are retrieved from
174 * @returns If an @p out parameter was provided, the value @c out
175 * after all of the @c status objects have been emitted.
177 template<typename ForwardIterator, typename OutputIterator>
179 wait_all(ForwardIterator first, ForwardIterator last, OutputIterator out)
181 typedef typename std::iterator_traits<ForwardIterator>::difference_type
186 difference_type num_outstanding_requests = distance(first, last);
188 std::vector<status> results(num_outstanding_requests);
189 std::vector<bool> completed(num_outstanding_requests);
191 while (num_outstanding_requests > 0) {
192 bool all_trivial_requests = true;
193 difference_type idx = 0;
194 for (ForwardIterator current = first; current != last; ++current, ++idx) {
195 if (!completed[idx]) {
196 if (optional<status> stat = current->test()) {
197 // This outstanding request has been completed. We're done.
198 results[idx] = *stat;
199 completed[idx] = true;
200 --num_outstanding_requests;
201 all_trivial_requests = false;
203 // Check if this request (and all others before it) are "trivial"
204 // requests, e.g., they can be represented with a single
206 all_trivial_requests =
208 && !current->m_handler
209 && current->m_requests[1] == MPI_REQUEST_NULL;
214 // If we have yet to fulfill any requests and all of the requests
215 // are trivial (i.e., require only a single MPI_Request to be
216 // fulfilled), call MPI_Waitall directly.
217 if (all_trivial_requests
218 && num_outstanding_requests == (difference_type)results.size()) {
219 std::vector<MPI_Request> requests;
220 requests.reserve(num_outstanding_requests);
221 for (ForwardIterator current = first; current != last; ++current)
222 requests.push_back(current->m_requests[0]);
224 // Let MPI wait until all of these operations completes.
225 std::vector<MPI_Status> stats(num_outstanding_requests);
226 BOOST_MPI_CHECK_RESULT(MPI_Waitall,
227 (num_outstanding_requests, &requests[0],
230 for (std::vector<MPI_Status>::iterator i = stats.begin();
231 i != stats.end(); ++i, ++out) {
240 all_trivial_requests = false;
243 return std::copy(results.begin(), results.end(), out);
249 template<typename ForwardIterator>
251 wait_all(ForwardIterator first, ForwardIterator last)
253 typedef typename std::iterator_traits<ForwardIterator>::difference_type
258 difference_type num_outstanding_requests = distance(first, last);
260 std::vector<bool> completed(num_outstanding_requests);
262 while (num_outstanding_requests > 0) {
263 bool all_trivial_requests = true;
265 difference_type idx = 0;
266 for (ForwardIterator current = first; current != last; ++current, ++idx) {
267 if (!completed[idx]) {
268 if (optional<status> stat = current->test()) {
269 // This outstanding request has been completed.
270 completed[idx] = true;
271 --num_outstanding_requests;
272 all_trivial_requests = false;
274 // Check if this request (and all others before it) are "trivial"
275 // requests, e.g., they can be represented with a single
277 all_trivial_requests =
279 && !current->m_handler
280 && current->m_requests[1] == MPI_REQUEST_NULL;
285 // If we have yet to fulfill any requests and all of the requests
286 // are trivial (i.e., require only a single MPI_Request to be
287 // fulfilled), call MPI_Waitall directly.
288 if (all_trivial_requests
289 && num_outstanding_requests == (difference_type)completed.size()) {
290 std::vector<MPI_Request> requests;
291 requests.reserve(num_outstanding_requests);
292 for (ForwardIterator current = first; current != last; ++current)
293 requests.push_back(current->m_requests[0]);
295 // Let MPI wait until all of these operations completes.
296 BOOST_MPI_CHECK_RESULT(MPI_Waitall,
297 (num_outstanding_requests, &requests[0],
298 MPI_STATUSES_IGNORE));
301 num_outstanding_requests = 0;
307 * @brief Tests whether all non-blocking requests have completed.
309 * This routine takes in a set of requests stored in the iterator
310 * range @c [first,last) and determines whether all of these requests
311 * have been completed. However, due to limitations of the underlying
312 * MPI implementation, if any of the requests refers to a
313 * non-blocking send or receive of a serialized data type, @c
314 * test_all will always return the equivalent of @c false (i.e., the
315 * requests cannot all be finished at this time). This routine
316 * performs the same functionality as @c wait_all, except that this
317 * routine will not block. This routine provides functionality
318 * equivalent to @c MPI_Testall.
320 * @param first The iterator that denotes the beginning of the
321 * sequence of request objects.
323 * @param last The iterator that denotes the end of the sequence of
326 * @param out If provided and all requests hav been completed, an
327 * output iterator through which the status of each request will be
328 * emitted. The @c status objects are emitted in the same order as
329 * the requests are retrieved from @c [first,last).
331 * @returns If an @p out parameter was provided, the value @c out
332 * after all of the @c status objects have been emitted (if all
333 * requests were completed) or an empty @c optional<>. If no @p out
334 * parameter was provided, returns @c true if all requests have
335 * completed or @c false otherwise.
337 template<typename ForwardIterator, typename OutputIterator>
338 optional<OutputIterator>
339 test_all(ForwardIterator first, ForwardIterator last, OutputIterator out)
341 std::vector<MPI_Request> requests;
342 for (; first != last; ++first) {
343 // If we have a non-trivial request, then no requests can be
345 if (first->m_handler || first->m_requests[1] != MPI_REQUEST_NULL)
346 return optional<OutputIterator>();
348 requests.push_back(first->m_requests[0]);
352 int n = requests.size();
353 std::vector<MPI_Status> stats(n);
354 BOOST_MPI_CHECK_RESULT(MPI_Testall, (n, &requests[0], &flag, &stats[0]));
356 for (int i = 0; i < n; ++i, ++out) {
358 stat.m_status = stats[i];
363 return optional<OutputIterator>();
370 template<typename ForwardIterator>
372 test_all(ForwardIterator first, ForwardIterator last)
374 std::vector<MPI_Request> requests;
375 for (; first != last; ++first) {
376 // If we have a non-trivial request, then no requests can be
378 if (first->m_handler || first->m_requests[1] != MPI_REQUEST_NULL)
381 requests.push_back(first->m_requests[0]);
385 int n = requests.size();
386 BOOST_MPI_CHECK_RESULT(MPI_Testall,
387 (n, &requests[0], &flag, MPI_STATUSES_IGNORE));
392 * @brief Wait until some non-blocking requests have completed.
394 * This routine takes in a set of requests stored in the iterator
395 * range @c [first,last) and waits until at least one of the requests
396 * has completed. It then completes all of the requests it can,
397 * partitioning the input sequence into pending requests followed by
398 * completed requests. If an output iterator is provided, @c status
399 * objects will be emitted for each of the completed requests. This
400 * routine provides functionality equivalent to @c MPI_Waitsome.
402 * @param first The iterator that denotes the beginning of the
403 * sequence of request objects.
405 * @param last The iterator that denotes the end of the sequence of
406 * request objects. This may not be equal to @c first.
408 * @param out If provided, the @c status objects corresponding to
409 * completed requests will be emitted through this output iterator.
411 * @returns If the @p out parameter was provided, a pair containing
412 * the output iterator @p out after all of the @c status objects have
413 * been written through it and an iterator referencing the first
414 * completed request. If no @p out parameter was provided, only the
415 * iterator referencing the first completed request will be emitted.
417 template<typename BidirectionalIterator, typename OutputIterator>
418 std::pair<OutputIterator, BidirectionalIterator>
419 wait_some(BidirectionalIterator first, BidirectionalIterator last,
425 return std::make_pair(out, first);
427 typedef typename std::iterator_traits<BidirectionalIterator>::difference_type
430 bool all_trivial_requests = true;
431 difference_type n = 0;
432 BidirectionalIterator current = first;
433 BidirectionalIterator start_of_completed = last;
435 // Check if we have found a completed request.
436 if (optional<status> result = current->test()) {
437 using std::iter_swap;
439 // Emit the resulting status object
442 // We're expanding the set of completed requests
443 --start_of_completed;
445 if (current == start_of_completed) {
446 // If we have hit the end of the list of pending
447 // requests. Finish up by fixing the order of the completed
448 // set to match the order in which we emitted status objects,
450 std::reverse(start_of_completed, last);
451 return std::make_pair(out, start_of_completed);
454 // Swap the request we just completed with the last request that
455 // has not yet been tested.
456 iter_swap(current, start_of_completed);
461 // Check if this request (and all others before it) are "trivial"
462 // requests, e.g., they can be represented with a single
464 all_trivial_requests =
466 && !current->m_handler
467 && current->m_requests[1] == MPI_REQUEST_NULL;
469 // Move to the next request.
471 if (++current == start_of_completed) {
472 if (start_of_completed != last) {
473 // We have satisfied some requests. Make the order of the
474 // completed requests match that of the status objects we've
475 // already emitted and we're done.
476 std::reverse(start_of_completed, last);
477 return std::make_pair(out, start_of_completed);
480 // We have reached the end of the list. If all requests thus far
481 // have been trivial, we can call MPI_Waitsome directly, because
482 // it may be more efficient than our busy-wait semantics.
483 if (all_trivial_requests) {
484 std::vector<MPI_Request> requests;
485 std::vector<int> indices(n);
486 std::vector<MPI_Status> stats(n);
488 for (current = first; current != last; ++current)
489 requests.push_back(current->m_requests[0]);
491 // Let MPI wait until some of these operations complete.
493 BOOST_MPI_CHECK_RESULT(MPI_Waitsome,
494 (n, &requests[0], &num_completed, &indices[0],
497 // Translate the index-based result of MPI_Waitsome into a
498 // partitioning on the requests.
499 int current_offset = 0;
501 for (int index = 0; index < num_completed; ++index, ++out) {
502 using std::iter_swap;
504 // Move "current" to the request object at this index
505 advance(current, indices[index] - current_offset);
506 current_offset = indices[index];
508 // Emit the status object
510 stat.m_status = stats[index];
513 // Finish up the request and swap it into the "completed
514 // requests" partition.
515 current->m_requests[0] = requests[indices[index]];
516 --start_of_completed;
517 iter_swap(current, start_of_completed);
520 // We have satisfied some requests. Make the order of the
521 // completed requests match that of the status objects we've
522 // already emitted and we're done.
523 std::reverse(start_of_completed, last);
524 return std::make_pair(out, start_of_completed);
527 // There are some nontrivial requests, so we must continue our
528 // busy waiting loop.
534 // We cannot ever get here
541 template<typename BidirectionalIterator>
542 BidirectionalIterator
543 wait_some(BidirectionalIterator first, BidirectionalIterator last)
550 typedef typename std::iterator_traits<BidirectionalIterator>::difference_type
553 bool all_trivial_requests = true;
554 difference_type n = 0;
555 BidirectionalIterator current = first;
556 BidirectionalIterator start_of_completed = last;
558 // Check if we have found a completed request.
559 if (optional<status> result = current->test()) {
560 using std::iter_swap;
562 // We're expanding the set of completed requests
563 --start_of_completed;
565 // If we have hit the end of the list of pending requests, we're
567 if (current == start_of_completed)
568 return start_of_completed;
570 // Swap the request we just completed with the last request that
571 // has not yet been tested.
572 iter_swap(current, start_of_completed);
577 // Check if this request (and all others before it) are "trivial"
578 // requests, e.g., they can be represented with a single
580 all_trivial_requests =
582 && !current->m_handler
583 && current->m_requests[1] == MPI_REQUEST_NULL;
585 // Move to the next request.
587 if (++current == start_of_completed) {
588 // If we have satisfied some requests, we're done.
589 if (start_of_completed != last)
590 return start_of_completed;
592 // We have reached the end of the list. If all requests thus far
593 // have been trivial, we can call MPI_Waitsome directly, because
594 // it may be more efficient than our busy-wait semantics.
595 if (all_trivial_requests) {
596 std::vector<MPI_Request> requests;
597 std::vector<int> indices(n);
599 for (current = first; current != last; ++current)
600 requests.push_back(current->m_requests[0]);
602 // Let MPI wait until some of these operations complete.
604 BOOST_MPI_CHECK_RESULT(MPI_Waitsome,
605 (n, &requests[0], &num_completed, &indices[0],
606 MPI_STATUSES_IGNORE));
608 // Translate the index-based result of MPI_Waitsome into a
609 // partitioning on the requests.
610 int current_offset = 0;
612 for (int index = 0; index < num_completed; ++index) {
613 using std::iter_swap;
615 // Move "current" to the request object at this index
616 advance(current, indices[index] - current_offset);
617 current_offset = indices[index];
619 // Finish up the request and swap it into the "completed
620 // requests" partition.
621 current->m_requests[0] = requests[indices[index]];
622 --start_of_completed;
623 iter_swap(current, start_of_completed);
626 // We have satisfied some requests, so we are done.
627 return start_of_completed;
630 // There are some nontrivial requests, so we must continue our
631 // busy waiting loop.
637 // We cannot ever get here
642 * @brief Test whether some non-blocking requests have completed.
644 * This routine takes in a set of requests stored in the iterator
645 * range @c [first,last) and tests to see if any of the requests has
646 * completed. It completes all of the requests it can, partitioning
647 * the input sequence into pending requests followed by completed
648 * requests. If an output iterator is provided, @c status objects
649 * will be emitted for each of the completed requests. This routine
650 * is similar to @c wait_some, but does not wait until any requests
651 * have completed. This routine provides functionality equivalent to
654 * @param first The iterator that denotes the beginning of the
655 * sequence of request objects.
657 * @param last The iterator that denotes the end of the sequence of
658 * request objects. This may not be equal to @c first.
660 * @param out If provided, the @c status objects corresponding to
661 * completed requests will be emitted through this output iterator.
663 * @returns If the @p out parameter was provided, a pair containing
664 * the output iterator @p out after all of the @c status objects have
665 * been written through it and an iterator referencing the first
666 * completed request. If no @p out parameter was provided, only the
667 * iterator referencing the first completed request will be emitted.
669 template<typename BidirectionalIterator, typename OutputIterator>
670 std::pair<OutputIterator, BidirectionalIterator>
671 test_some(BidirectionalIterator first, BidirectionalIterator last,
674 BidirectionalIterator current = first;
675 BidirectionalIterator start_of_completed = last;
676 while (current != start_of_completed) {
677 // Check if we have found a completed request.
678 if (optional<status> result = current->test()) {
679 using std::iter_swap;
681 // Emit the resulting status object
684 // We're expanding the set of completed requests
685 --start_of_completed;
687 // Swap the request we just completed with the last request that
688 // has not yet been tested.
689 iter_swap(current, start_of_completed);
694 // Move to the next request.
698 // Finish up by fixing the order of the completed set to match the
699 // order in which we emitted status objects, then return.
700 std::reverse(start_of_completed, last);
701 return std::make_pair(out, start_of_completed);
707 template<typename BidirectionalIterator>
708 BidirectionalIterator
709 test_some(BidirectionalIterator first, BidirectionalIterator last)
711 BidirectionalIterator current = first;
712 BidirectionalIterator start_of_completed = last;
713 while (current != start_of_completed) {
714 // Check if we have found a completed request.
715 if (optional<status> result = current->test()) {
716 using std::iter_swap;
718 // We're expanding the set of completed requests
719 --start_of_completed;
721 // Swap the request we just completed with the last request that
722 // has not yet been tested.
723 iter_swap(current, start_of_completed);
728 // Move to the next request.
732 return start_of_completed;
735 } } // end namespace boost::mpi
738 #endif // BOOST_MPI_NONBLOCKING_HPP