Add support for MPI intercommunicators

[SVN r37882]

build/Jamfile.v2

@@ -28,6 +28,7 @@ lib boost_mpi
     exception.cpp
     graph_topology.cpp
     group.cpp
+    intercommunicator.cpp
     mpi_datatype_cache.cpp
     mpi_datatype_oarchive.cpp
     packed_iarchive.cpp

doc/Jamfile.v2

@@ -21,6 +21,7 @@ doxygen mpi_autodoc
     ../../../boost/mpi/exception.hpp
     ../../../boost/mpi/graph_topology.hpp
     ../../../boost/mpi/group.hpp
+    ../../../boost/mpi/intercommunicator.hpp
     ../../../boost/mpi/nonblocking.hpp
     ../../../boost/mpi/operations.hpp
     ../../../boost/mpi/packed_iarchive.hpp

doc/mpi.qbk

@@ -1451,15 +1451,15 @@ Boost.MPI currently provides no support for inter-communicators.
   [[C Function] [Boost.MPI Equivalent]]
 
   [[[@http://www.mpi-forum.org/docs/mpi-11-html/node112.html#Node112
-  `MPI_Comm_test_inter`]] [unsupported]]
+  `MPI_Comm_test_inter`]] [use [memberref boost::mpi::communicator::as_intercommunicator communicator::as_intercommunicator]]]
   [[[@http://www.mpi-forum.org/docs/mpi-11-html/node112.html#Node112
-  `MPI_Comm_remote_size`]] [unsupported]]
+  `MPI_Comm_remote_size`]] [[memberref boost::mpi::intercommunicator::remote_size] intercommunicator::remote_size]]
   [[[@http://www.mpi-forum.org/docs/mpi-11-html/node112.html#Node112
-  `MPI_Comm_remote_group`]] [unsupported]]
+  `MPI_Comm_remote_group`]] [[memberref boost::mpi::intercommunicator::remote_group intercommunicator::remote_group]]]
   [[[@http://www.mpi-forum.org/docs/mpi-11-html/node113.html#Node113
-  `MPI_Intercomm_create`]] [unsupported]]
+  `MPI_Intercomm_create`]] [[classref boost::mpi::intercommunicator intercommunicator] constructor]]
   [[[@http://www.mpi-forum.org/docs/mpi-11-html/node113.html#Node113
-  `MPI_Intercomm_merge`]] [unsupported]]
+  `MPI_Intercomm_merge`]] [[memberref boost::mpi::intercommunicator::merge intercommunicator::merge]]]
 ]
 
 Boost.MPI currently provides no support for attribute caching.

include/boost/mpi.hpp

@@ -24,6 +24,7 @@
 #include <boost/mpi/environment.hpp>
 #include <boost/mpi/graph_topology.hpp>
 #include <boost/mpi/group.hpp>
+#include <boost/mpi/intercommunicator.hpp>
 #include <boost/mpi/nonblocking.hpp>
 #include <boost/mpi/operations.hpp>
 #include <boost/mpi/skeleton_and_content.hpp>

include/boost/mpi/communicator.hpp

@@ -89,11 +89,19 @@ enum comm_create_kind { comm_duplicate, comm_take_ownership, comm_attach };
 /**
  * INTERNAL ONLY
  * 
- * Forward-declaration of @c group needed for the @c group
- * constructor.
+ * Forward declaration of @c group needed for the @c group
+ * constructor and accessor.
  */
 class group;
 
+/**
+ * INTERNAL ONLY
+ *
+ * Forward declaration of @c intercommunicator needed for the "cast"
+ * from a communicator to an intercommunicator.
+ */
+class intercommunicator;
+
 /**
  * @brief A communicator that permits communication and
  * synchronization among a set of processes.
@@ -126,9 +134,13 @@ class communicator
    * ignored. Otherwise, the @p kind parameters determines how the
    * Boost.MPI communicator will be related to @p comm:
    *
-   *   - If @p kind is @c comm_duplicate, duplicate @c comm to create a
-   *   new communicator. This new communicator will be freed when the
-   *   Boost.MPI communicator (and all copies of it) is destroyed.
+   *   - If @p kind is @c comm_duplicate, duplicate @c comm to create
+   *   a new communicator. This new communicator will be freed when
+   *   the Boost.MPI communicator (and all copies of it) is destroyed.
+   *   This option is only permitted if @p comm is a valid MPI
+   *   intracommunicator or if the underlying MPI implementation
+   *   supports MPI 2.0 (which supports duplication of
+   *   intercommunicators).
    *
    *   - If @p kind is @c comm_take_ownership, take ownership of @c
    *   comm. It will be freed automatically when all of the Boost.MPI
@@ -156,7 +168,7 @@ class communicator
    * @param subgroup A subgroup of the MPI communicator, @p comm, for
    * which we will construct a new communicator.
    */
-  communicator(const communicator& comm, const group& subgroup);
+  communicator(const communicator& comm, const boost::mpi::group& subgroup);
 
   /**
    * @brief Determine the rank of the executing process in a
@@ -178,6 +190,13 @@ class communicator
    */
   int size() const;
 
+  /**
+   * This routine constructs a new group whose members are the
+   * processes within this communicator. Equivalent to
+   * calling @c MPI_Comm_group.
+   */
+  boost::mpi::group group() const;
+
   // ----------------------------------------------------------------
   // Point-to-point communication
   // ----------------------------------------------------------------
@@ -766,6 +785,16 @@ class communicator
    */
   communicator split(int color, int key) const;
 
+  /**
+   * Determine if the communicator is in fact an intercommunicator
+   * and, if so, return that intercommunicator.
+   *
+   * @returns an @c optional containing the intercommunicator, if this
+   * communicator is in fact an intercommunicator. Otherwise, returns
+   * an empty @c optional.
+   */
+  optional<intercommunicator> as_intercommunicator() const;
+
   /**
    * Determines whether this communicator has a Cartesian topology.
    */
@@ -876,7 +905,7 @@ class communicator
     }
   };
 
- private:
+  
   /**
    * INTERNAL ONLY
    *

include/boost/mpi/group.hpp

@@ -22,14 +22,6 @@
 
 namespace boost { namespace mpi {
 
-/**
- * INTERNAL ONLY
- * 
- * Forward-declaration of @c communicator needed for the @c
- * communicator constructor.
- */
-class communicator;
-
 /**
  * @brief A @c group is a representation of a subset of the processes
  * within a @c communicator.
@@ -68,18 +60,6 @@ public:
    */
   group(const MPI_Group& in_group, bool adopt);
 
-  /**
-   *
-   * @brief Constructs a group from a communicator.
-   *
-   * This routine constructs a new group whose members are the
-   * processes within the given communicator, @p comm. Equivalent to
-   * calling @c MPI_Comm_group.
-   *
-   * @param comm The communicator whose group we are constructing.
-   */
-  group(const communicator& comm);
-
   /**
    * @brief Determine the rank of the calling process in the group.
    * 
@@ -197,7 +177,7 @@ protected:
   /**
    * INTERNAL ONLY
    *
-   * Function object that frees an MPI communicator and deletes the
+   * Function object that frees an MPI group and deletes the
    * memory associated with it. Intended to be used as a deleter with
    * shared_ptr.
    */

include/boost/mpi/intercommunicator.hpp

@@ -0,0 +1,145 @@
+// Copyright (C) 2007 The Trustees of Indiana University.
+
+// Authors: Douglas Gregor
+//          Andrew Lumsdaine
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+/** @file intercommunicator.hpp
+ *
+ *  This header defines the @c intercommunicator class, which permits
+ *  communication between different process groups.
+ */
+#ifndef BOOST_MPI_INTERCOMMUNICATOR_HPP
+#define BOOST_MPI_INTERCOMMUNICATOR_HPP
+
+#include <boost/mpi/communicator.hpp>
+
+namespace boost { namespace mpi {
+
+/**
+ * INTERNAL ONLY
+ *
+ * Forward declaration of the MPI "group" representation, for use in
+ * the description of the @c intercommunicator class.
+ */ 
+class group;
+   
+class intercommunicator : public communicator
+{
+private:
+  friend class communicator;
+
+  /**
+   * INTERNAL ONLY
+   *
+   * Construct an intercommunicator given a shared pointer to the
+   * underlying MPI_Comm. This operation is used for "casting" from a
+   * communicator to an intercommunicator.
+   */
+  explicit intercommunicator(const shared_ptr<MPI_Comm>& comm_ptr)
+  {
+    this->comm_ptr = comm_ptr;
+  }
+
+public:
+  /**
+   * Build a new Boost.MPI intercommunicator based on the MPI
+   * intercommunicator @p comm.
+   *
+   * @p comm may be any valid MPI intercommunicator. If @p comm is
+   * MPI_COMM_NULL, an empty communicator (that cannot be used for
+   * communication) is created and the @p kind parameter is
+   * ignored. Otherwise, the @p kind parameter determines how the
+   * Boost.MPI communicator will be related to @p comm:
+   *
+   *   - If @p kind is @c comm_duplicate, duplicate @c comm to create
+   *   a new communicator. This new communicator will be freed when
+   *   the Boost.MPI communicator (and all copies of it) is
+   *   destroyed. This option is only permitted if the underlying MPI
+   *   implementation supports MPI 2.0; duplication of
+   *   intercommunicators is not available in MPI 1.x.
+   *
+   *   - If @p kind is @c comm_take_ownership, take ownership of @c
+   *   comm. It will be freed automatically when all of the Boost.MPI
+   *   communicators go out of scope.
+   *
+   *   - If @p kind is @c comm_attach, this Boost.MPI communicator
+   *   will reference the existing MPI communicator @p comm but will
+   *   not free @p comm when the Boost.MPI communicator goes out of
+   *   scope. This option should only be used when the communicator is
+   *   managed by the user.
+   */
+  intercommunicator(const MPI_Comm& comm, comm_create_kind kind)
+    : communicator(comm, kind) { }
+
+  /**
+   * Constructs a new intercommunicator whose local group is @p local
+   * and whose remote group is @p peer. The intercommunicator can then
+   * be used to communicate between processes in the two groups. This
+   * constructor is equivalent to a call to @c MPI_Intercomm_create.
+   *
+   * @param local The intracommunicator containing all of the
+   * processes that will go into the local group.
+   *
+   * @param local_leader The rank within the @p local
+   * intracommunicator that will serve as its leader.
+   *
+   * @param peer The intracommunicator containing all of the processes
+   * that will go into the remote group.
+   *
+   * @param remote_leader The rank within the @p peer group that will
+   * serve as its leader.
+   */
+  intercommunicator(const communicator& local, int local_leader,
+                    const communicator& peer, int remote_leader);
+
+  /**
+   * Returns the size of the local group, i.e., the number of local
+   * processes that are part of the group.
+   */
+  int local_size() const { return this->size(); }
+
+  /**
+   * Returns the local group, containing all of the local processes in
+   * this intercommunicator.
+   */
+  boost::mpi::group local_group() const;
+
+  /**
+   * Returns the rank of this process within the local group.
+   */
+  int local_rank() const { return this->rank(); }
+
+  /**
+   * Returns the size of the remote group, i.e., the number of
+   * processes that are part of the remote group.
+   */
+  int remote_size() const;
+
+  /**
+   * Returns the remote group, containing all of the remote processes
+   * in this intercommunicator.
+   */
+  boost::mpi::group remote_group() const;
+
+  /**
+   * Merge the local and remote groups in this intercommunicator into
+   * a new intracommunicator containing the union of the processes in
+   * both groups. This method is equivalent to @c MPI_Intercomm_merge.
+   *  
+   * @param high Whether the processes in this group should have the
+   * higher rank numbers than the processes in the other group. Each
+   * of the processes within a particular group shall have the same
+   * "high" value.
+   *
+   * @returns the new, merged intracommunicator
+   */
+  communicator merge(bool high) const;
+};
+
+} } // end namespace boost::mpi
+
+#endif // BOOST_MPI_INTERCOMMUNICATOR_HPP

src/communicator.cpp

@@ -5,6 +5,7 @@
 // http://www.boost.org/LICENSE_1_0.txt)
 #include <boost/mpi/communicator.hpp>
 #include <boost/mpi/group.hpp>
+#include <boost/mpi/intercommunicator.hpp>
 #include <boost/mpi/skeleton_and_content.hpp>
 #include <boost/mpi/detail/point_to_point.hpp>
 
@@ -55,7 +56,8 @@ communicator::communicator(const MPI_Comm& comm, comm_create_kind kind)
   }
 }
 
-communicator::communicator(const communicator& comm, const group& subgroup)
+communicator::communicator(const communicator& comm, 
+                           const boost::mpi::group& subgroup)
 {
   MPI_Comm newcomm;
   BOOST_MPI_CHECK_RESULT(MPI_Comm_create, 
@@ -77,6 +79,13 @@ int communicator::rank() const
   return rank_;
 }
 
+boost::mpi::group communicator::group() const
+{
+  MPI_Group gr;
+  BOOST_MPI_CHECK_RESULT(MPI_Comm_group, ((MPI_Comm)*this, &gr));
+  return boost::mpi::group(gr, /*adopt=*/true);
+}
+
 void communicator::send(int dest, int tag) const
 {
   BOOST_MPI_CHECK_RESULT(MPI_Send,
@@ -141,6 +150,16 @@ communicator communicator::split(int color, int key) const
   return communicator(newcomm, comm_take_ownership);
 }
 
+optional<intercommunicator> communicator::as_intercommunicator() const
+{
+  int flag;
+  BOOST_MPI_CHECK_RESULT(MPI_Comm_test_inter, ((MPI_Comm)*this, &flag));
+  if (flag)
+    return intercommunicator(comm_ptr);
+  else
+    return optional<intercommunicator>();
+}
+
 bool communicator::has_cartesian_topology() const
 {
   int status;

src/group.cpp

@@ -19,15 +19,6 @@ group::group(const MPI_Group& in_group, bool adopt)
   }
 }
 
-group::group(const communicator& comm)
-{
-  MPI_Group gr;
-  BOOST_MPI_CHECK_RESULT(MPI_Comm_group, ((MPI_Comm)comm, &gr));
-
-  if (gr != MPI_GROUP_EMPTY)
-    group_ptr.reset(new MPI_Group(gr), group_free());
-}
-
 optional<int> group::rank() const
 {
   if (!group_ptr)

src/intercommunicator.cpp

@@ -0,0 +1,54 @@
+// Copyright (C) 2007 Trustees of Indiana University
+
+// Authors: Douglas Gregor
+//          Andrew Lumsdaine
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+#include <boost/mpi/intercommunicator.hpp>
+#include <boost/mpi/environment.hpp>
+#include <boost/mpi/group.hpp>
+
+namespace boost { namespace mpi {
+
+intercommunicator::intercommunicator(const communicator& local, 
+                                     int local_leader,
+                                     const communicator& peer, 
+                                     int remote_leader)
+{
+  MPI_Comm comm;
+  BOOST_MPI_CHECK_RESULT(MPI_Intercomm_create,
+                         ((MPI_Comm)local, local_leader,
+                          (MPI_Comm)peer, remote_leader,
+                          environment::collectives_tag(), &comm));
+  comm_ptr.reset(new MPI_Comm(comm), comm_free());
+}
+
+boost::mpi::group intercommunicator::local_group() const
+{
+  return this->group();
+}
+
+int intercommunicator::remote_size() const
+{
+  int size;
+  BOOST_MPI_CHECK_RESULT(MPI_Comm_remote_size, ((MPI_Comm)*this, &size));
+  return size;
+}
+
+boost::mpi::group intercommunicator::remote_group() const
+{
+  MPI_Group gr;
+  BOOST_MPI_CHECK_RESULT(MPI_Comm_remote_group, ((MPI_Comm)*this, &gr));
+  return boost::mpi::group(gr, /*adopt=*/true);
+}
+
+communicator intercommunicator::merge(bool high) const
+{
+  MPI_Comm comm;
+  BOOST_MPI_CHECK_RESULT(MPI_Intercomm_merge, ((MPI_Comm)*this, high, &comm));
+  return communicator(comm, comm_take_ownership);
+}
+
+} } // end namespace boost::mpi