diff --git a/include/boost/mpi/cartesian_communicator.hpp b/include/boost/mpi/cartesian_communicator.hpp index 0ded619..0535073 100644 --- a/include/boost/mpi/cartesian_communicator.hpp +++ b/include/boost/mpi/cartesian_communicator.hpp @@ -23,6 +23,7 @@ #include #include #include +#include // Headers required to implement cartesian topologies #include @@ -66,6 +67,9 @@ operator!=(cartesian_dimension const& d1, cartesian_dimension const& d2) { return !(d1 == d2); } +/** + * @brief Pretty printing of a cartesian dimension (size, periodic) + */ std::ostream& operator<<(std::ostream& out, cartesian_dimension const& d); /** @@ -107,6 +111,11 @@ class BOOST_MPI_DECL cartesian_topology void split(std::vector& dims, std::vector& periodics) const; }; +/** + * @brief Pretty printing of a cartesian topology + */ +std::ostream& operator<<(std::ostream& out, cartesian_topology const& t); + /** * @brief An MPI communicator with a cartesian topology. * @@ -215,6 +224,12 @@ public: * @param coords the coordinates. the size must match the communicator's topology. */ int rank(const std::vector& coords) const; + /** + * Return the rank of the source and targetdestination process through a shift. + * @param dim the dimension in which the shift takes place. 0 <= dim <= ndim(). + * @param disp the shift displacement, can be positive (upward) or negative (downward). + */ + std::pair shifted_ranks(int dim, int disp) const; /** * Provides the coordinates of a process with the given rank. * @param rk the rank in this communicator. diff --git a/src/cartesian_communicator.cpp b/src/cartesian_communicator.cpp index d019128..e5c877a 100644 --- a/src/cartesian_communicator.cpp +++ b/src/cartesian_communicator.cpp @@ -24,6 +24,20 @@ operator<<(std::ostream& out, cartesian_dimension const& d) { return out; } +std::ostream& +operator<<(std::ostream& out, cartesian_topology const& topo) { + out << '{'; + int const sz = topo.size(); + for (int i = 0; i < sz; ++i) { + out << topo[i]; + if ( i < (sz-1) ) { + out << ','; + } + } + out << '}'; + return out; +} + cartesian_communicator::cartesian_communicator(const communicator& comm, const cartesian_topology& topology, bool reorder ) @@ -31,7 +45,8 @@ cartesian_communicator::cartesian_communicator(const communicator& comm, { std::vector dims(topology.size()); std::vector periodic(topology.size()); - for(int i = 0; i < topology.size(); ++i) { + int tsz = topology.size(); + for(int i = 0; i < tsz; ++i) { dims[i] = topology[i].size; periodic[i] = topology[i].periodic; } @@ -46,6 +61,8 @@ cartesian_communicator::cartesian_communicator(const communicator& comm, int(reorder), &newcomm)); if(newcomm != MPI_COMM_NULL) { comm_ptr.reset(new MPI_Comm(newcomm), comm_free()); + } else { + comm_ptr.reset(MPI_COMM_NULL); } } @@ -53,10 +70,11 @@ cartesian_communicator::cartesian_communicator(const cartesian_communicator& com const std::vector& keep ) : communicator() { - int max_dims = comm.ndims(); - BOOST_ASSERT(keep.size() <= max_dims); + int const max_dims = comm.ndims(); + int const nbkept = keep.size(); + BOOST_ASSERT(nbkept <= max_dims); std::vector bitset(max_dims, int(false)); - for(int i = 0; i < keep.size(); ++i) { + for(int i = 0; i < nbkept; ++i) { BOOST_ASSERT(keep[i] < max_dims); bitset[keep[i]] = true; } @@ -80,13 +98,23 @@ cartesian_communicator::ndims() const { int cartesian_communicator::rank(const std::vector& coords ) const { int r = -1; - BOOST_ASSERT(coords.size() == ndims()); + BOOST_ASSERT(int(coords.size()) == ndims()); BOOST_MPI_CHECK_RESULT(MPI_Cart_rank, (MPI_Comm(*this), const_cast&>(coords).data(), &r)); return r; } - + +std::pair +cartesian_communicator::shifted_ranks(int dim, int disp) const { + std::pair r(-1,-1); + assert((0 <= dim && dim < ndims()) || (std::abort(), false)); + BOOST_ASSERT(0 <= dim && dim < ndims()); + BOOST_MPI_CHECK_RESULT(MPI_Cart_shift, + (MPI_Comm(*this), dim, disp, &(r.first), &(r.second))); + return r; +} + std::vector& cartesian_communicator::coords(int rk, std::vector& cbuf) const { cbuf.resize(ndims()); diff --git a/test/cartesian_topology_test.cpp b/test/cartesian_topology_test.cpp index ba52b8d..d67f40f 100644 --- a/test/cartesian_topology_test.cpp +++ b/test/cartesian_topology_test.cpp @@ -59,6 +59,34 @@ void test_coordinates_consistency( mpi::cartesian_communicator const& cc, } } +void test_shifted_coords( mpi::cartesian_communicator const& cc, int pos, mpi::cartesian_dimension desc, int dim ) +{ + for (int i = -(desc.size); i < desc.size; ++i) { + std::pair rks = cc.shifted_ranks(pos, i); + int src = cc.coords(rks.first)[dim]; + int dst = cc.coords(rks.second)[dim]; + if (pos == (dim/2)) { + std::ostringstream out; + out << "Rank " << cc.rank() << ", dim. " << dim << ", pos " << pos << ", in " << desc << ' '; + out << "shifted pos: " << src << ", " << dst << '\n'; + std::cout << out.str(); + } + } +} + +void test_shifted_coords( mpi::cartesian_communicator const& cc, std::vector const& coords, mpi::cartesian_topology const& topo ) +{ + if (cc.rank() == 0) { + std::cout << "Testing shifts with topology " << topo << '\n'; + } + for(int i = 0; i < cc.ndims(); ++i) { + if (cc.rank() == 0) { + std::cout << " for dimension " << i << ' ' << topo[i] << '\n'; + } + test_shifted_coords( cc, coords[i], topo[i], i ); + } +} + void test_topology_consistency( mpi::cartesian_communicator const& cc) { mpi::cartesian_topology itopo(cc.ndims()); @@ -78,23 +106,11 @@ void test_topology_consistency( mpi::cartesian_communicator const& cc) test_coordinates_consistency( cc, coords ); } -int test_main(int argc, char* argv[]) +void test_cartesian_topology( mpi::communicator const& world, mpi::cartesian_topology const& topo) { - mpi::environment env(argc, argv); - - mpi::communicator world; - mpi::cartesian_topology topo(3); - - if (world.size() == 24) { - topo[0].size = 2; topo[1].size = 3; topo[2].size = 4; - } else { - topo[0].size = 0; topo[1].size = 3; topo[2].size = 0; - } - topo[0].periodic = true; topo[1].periodic = false; topo[2].periodic = true; - mpi::cartesian_communicator cc(world, topo, true); BOOST_CHECK(cc.has_cartesian_topology()); - BOOST_CHECK(cc.ndims() == 3); + BOOST_CHECK(cc.ndims() == int(topo.size())); for( int r = 0; r < cc.size(); ++r) { cc.barrier(); if (r == cc.rank()) { @@ -107,10 +123,31 @@ int test_main(int argc, char* argv[]) } } test_topology_consistency(cc); - std::vector sub02; - sub02.push_back(0); - sub02.push_back(2); - mpi::cartesian_communicator cc02(cc, sub02); - test_topology_consistency(cc02); + std::vector even; + for(int i = 0; i < cc.ndims(); i += 2) { + even.push_back(i); + } + mpi::cartesian_communicator cce(cc, even); + test_topology_consistency(cce); + test_shifted_coords( cce, cce.coords(cce.rank()), topo ); +} + +int test_main(int argc, char* argv[]) +{ + mpi::environment env(argc, argv); + + mpi::communicator world; + int const ndim = world.size() >= 24 ? 3 : 2; + mpi::cartesian_topology topo(ndim); + typedef mpi::cartesian_dimension cd; + if (topo.size() == 3) { + topo[0] = cd(2,true); topo[1] = cd(3,false); topo[2] = cd(4, true); + } else if (world.size() >= 6) { + topo[0] = cd(0,true); topo[1] = cd(3, false); + } else { + topo[0] = cd(0,true); topo[1] = cd(0, false); + } + test_cartesian_topology( world, topo); + return 0; }