Tpetra_DistObject_decl.hpp

Go to the documentation of this file.

// @HEADER
// ***********************************************************************
//
//          Tpetra: Templated Linear Algebra Services Package
//                 Copyright (2008) Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
//
// ************************************************************************
// @HEADER
 
#ifndef TPETRA_DISTOBJECT_DECL_HPP
#define TPETRA_DISTOBJECT_DECL_HPP
 
 
#include "Tpetra_Map.hpp"
#include "Tpetra_Import.hpp"
#include "Tpetra_Export.hpp"
#include "Tpetra_SrcDistObject.hpp"
#include "Tpetra_DistObject_fwd.hpp"
#include "Kokkos_ArithTraits.hpp"
#include <type_traits>
 
// #ifndef HAVE_TPETRA_TRANSFER_TIMERS
// #  define HAVE_TPETRA_TRANSFER_TIMERS 1
// #endif // HAVE_TPETRA_TRANSFER_TIMERS
 
#ifdef HAVE_TPETRA_TRANSFER_TIMERS
#  undef HAVE_TPETRA_TRANSFER_TIMERS
#endif // HAVE_TPETRA_TRANSFER_TIMERS
 
namespace KokkosClassic {
  enum ReadWriteOption {
    ReadWrite = 0, 
    WriteOnly = 1 
  };
} // namespace KokkosClassic
 
namespace Tpetra {
 
  template<class DistObjectType>
  void
  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
                               const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
                                                            typename DistObjectType::global_ordinal_type,
                                                            typename DistObjectType::node_type> >& newMap);
 
  template<class DistObjectType>
  void
  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input);
 
  template <class Packet,
            class LocalOrdinal,
            class GlobalOrdinal,
            class Node>
  class DistObject :
    virtual public SrcDistObject,
    virtual public Teuchos::Describable
  {
  public:
 
 
    using packet_type = typename ::Kokkos::Details::ArithTraits<Packet>::val_type;
    using local_ordinal_type = LocalOrdinal;
    using global_ordinal_type = GlobalOrdinal;
    using node_type = Node;
 
    using device_type = typename Node::device_type;
    using execution_space = typename device_type::execution_space;
 
    using map_type = Map<local_ordinal_type, global_ordinal_type, node_type>;
 
 
 
    explicit DistObject (const Teuchos::RCP<const map_type>& map);
 
    DistObject (const DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node>&) = default;
 
    DistObject& operator= (const DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node>&) = default;
 
    DistObject (DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node>&&) = default;
 
    DistObject& operator= (DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node>&&) = default;
 
    virtual ~DistObject () = default;
 
 
 
    void
    doImport (const SrcDistObject& source,
              const Import<LocalOrdinal, GlobalOrdinal, Node>& importer,
              const CombineMode CM,
              const bool restrictedMode = false);
 
    void
    doExport (const SrcDistObject& source,
              const Export<LocalOrdinal, GlobalOrdinal, Node>& exporter,
              const CombineMode CM,
              const bool restrictedMode = false);
 
    void
    doImport (const SrcDistObject& source,
              const Export<LocalOrdinal, GlobalOrdinal, Node>& exporter,
              const CombineMode CM,
              const bool restrictedMode = false);
 
    void
    doExport (const SrcDistObject& source,
              const Import<LocalOrdinal, GlobalOrdinal, Node>& importer,
              const CombineMode CM,
              const bool restrictedMode = false);
 
 
 
    bool isDistributed () const;
 
    virtual Teuchos::RCP<const map_type> getMap () const { return map_; }
 
 
 
    void print (std::ostream& os) const;
 
 
 
    virtual std::string description () const;
 
    virtual void
    describe (Teuchos::FancyOStream &out,
              const Teuchos::EVerbosityLevel verbLevel =
              Teuchos::Describable::verbLevel_default) const;
 
 
 
    virtual void
    removeEmptyProcessesInPlace (const Teuchos::RCP<const map_type>& newMap);
 
 
  protected:
    enum ReverseOption {
      DoForward, //*!< Perform the transfer in forward mode.
      DoReverse  //*!< Perform the transfer in reverse mode.
    };
 
    virtual size_t constantNumberOfPackets () const;
 
    virtual void
    doTransfer (const SrcDistObject& src,
                const ::Tpetra::Details::Transfer<local_ordinal_type, global_ordinal_type, node_type>& transfer,
                const char modeString[],
                const ReverseOption revOp,
                const CombineMode CM,
                const bool restrictedMode);
 
    virtual bool
    reallocArraysForNumPacketsPerLid (const size_t numExportLIDs,
                                      const size_t numImportLIDs);
 
#ifdef TPETRA_ENABLE_DEPRECATED_CODE
    virtual void TPETRA_DEPRECATED
    doTransferOld (const SrcDistObject& src,
                   CombineMode CM,
                   size_t numSameIDs,
                   const Teuchos::ArrayView<const local_ordinal_type> &permuteToLIDs,
                   const Teuchos::ArrayView<const local_ordinal_type> &permuteFromLIDs,
                   const Teuchos::ArrayView<const local_ordinal_type> &remoteLIDs,
                   const Teuchos::ArrayView<const local_ordinal_type> &exportLIDs,
                   Distributor &distor,
                   ReverseOption revOp,
                   const bool restrictedMode);
#endif // TPETRA_ENABLE_DEPRECATED_CODE
 
    using buffer_memory_space =
      ::Tpetra::Details::DefaultTypes::comm_buffer_memory_space<
        typename device_type::execution_space>;
 
  public:
    using buffer_device_type =
      Kokkos::Device<typename device_type::execution_space,
                     buffer_memory_space>;
  protected:
    virtual void
    doTransferNew (const SrcDistObject& src,
                   const CombineMode CM,
                   const size_t numSameIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                     buffer_device_type>& permuteToLIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                     buffer_device_type>& permuteFromLIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                     buffer_device_type>& remoteLIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                     buffer_device_type>& exportLIDs,
                   Distributor& distor,
                   const ReverseOption revOp,
                   const bool commOnHost,
                   const bool restrictedMode);
 
 
 
    virtual bool
    checkSizes (const SrcDistObject& source) = 0;
 
#ifdef TPETRA_ENABLE_DEPRECATED_CODE
    virtual void
    copyAndPermuteNew (const SrcDistObject& source,
                       const size_t numSameIDs,
                       const Kokkos::DualView<const local_ordinal_type*,
                         buffer_device_type>& permuteToLIDs,
                       const Kokkos::DualView<const local_ordinal_type*,
                         buffer_device_type>& permuteFromLIDs);
#else // TPETRA_ENABLE_DEPRECATED_CODE
    virtual void
    copyAndPermute (const SrcDistObject& source,
                    const size_t numSameIDs,
                    const Kokkos::DualView<const local_ordinal_type*,
                      buffer_device_type>& permuteToLIDs,
                    const Kokkos::DualView<const local_ordinal_type*,
                      buffer_device_type>& permuteFromLIDs);
#endif // TPETRA_ENABLE_DEPRECATED_CODE
 
#ifdef TPETRA_ENABLE_DEPRECATED_CODE
    virtual void
    packAndPrepareNew (const SrcDistObject& source,
                       const Kokkos::DualView<const local_ordinal_type*,
                         buffer_device_type>& exportLIDs,
                       Kokkos::DualView<packet_type*,
                         buffer_device_type>& exports,
                       Kokkos::DualView<size_t*,
                         buffer_device_type> numPacketsPerLID,
                       size_t& constantNumPackets,
                       Distributor& distor);
#else // TPETRA_ENABLE_DEPRECATED_CODE
    virtual void
    packAndPrepare (const SrcDistObject& source,
                    const Kokkos::DualView<const local_ordinal_type*,
                      buffer_device_type>& exportLIDs,
                    Kokkos::DualView<packet_type*,
                      buffer_device_type>& exports,
                    Kokkos::DualView<size_t*,
                      buffer_device_type> numPacketsPerLID,
                    size_t& constantNumPackets,
                    Distributor& distor);
#endif // TPETRA_ENABLE_DEPRECATED_CODE
 
#ifdef TPETRA_ENABLE_DEPRECATED_CODE
    virtual void
    unpackAndCombineNew (const Kokkos::DualView<const local_ordinal_type*,
                           buffer_device_type>& importLIDs,
                         Kokkos::DualView<packet_type*,
                           buffer_device_type> imports,
                         Kokkos::DualView<size_t*,
                           buffer_device_type> numPacketsPerLID,
                         const size_t constantNumPackets,
                         Distributor& distor,
                         const CombineMode combineMode);
#else // TPETRA_ENABLE_DEPRECATED_CODE
    virtual void
    unpackAndCombine (const Kokkos::DualView<const local_ordinal_type*,
                        buffer_device_type>& importLIDs,
                      Kokkos::DualView<packet_type*,
                        buffer_device_type> imports,
                      Kokkos::DualView<size_t*,
                        buffer_device_type> numPacketsPerLID,
                      const size_t constantNumPackets,
                      Distributor& distor,
                      const CombineMode combineMode);
#endif // TPETRA_ENABLE_DEPRECATED_CODE
 
#ifdef TPETRA_ENABLE_DEPRECATED_CODE
    virtual bool TPETRA_DEPRECATED useNewInterface () { return true; }
 
    virtual void TPETRA_DEPRECATED
    copyAndPermute (const SrcDistObject& source,
                    const size_t numSameIDs,
                    const Teuchos::ArrayView<const local_ordinal_type>& permuteToLIDs,
                    const Teuchos::ArrayView<const local_ordinal_type>& permuteFromLIDs);
 
    virtual void TPETRA_DEPRECATED
    packAndPrepare (const SrcDistObject& source,
                    const Teuchos::ArrayView<const local_ordinal_type>& exportLIDs,
                    Teuchos::Array<packet_type>& exports,
                    const Teuchos::ArrayView<size_t>& numPacketsPerLID,
                    size_t& constantNumPackets,
                    Distributor& distor);
 
    virtual void TPETRA_DEPRECATED
    unpackAndCombine (const Teuchos::ArrayView<const local_ordinal_type>& importLIDs,
                      const Teuchos::ArrayView<const packet_type>& imports,
                      const Teuchos::ArrayView<size_t>& numPacketsPerLID,
                      const size_t constantNumPackets,
                      Distributor& distor,
                      const CombineMode combineMode);
 
    virtual void TPETRA_DEPRECATED
    createViews () const;
 
    virtual void TPETRA_DEPRECATED
    createViewsNonConst (KokkosClassic::ReadWriteOption rwo);
 
    virtual void TPETRA_DEPRECATED releaseViews () const;
#endif // TPETRA_ENABLE_DEPRECATED_CODE
 
    Teuchos::RCP<const map_type> map_;
 
  protected:
    Kokkos::DualView<packet_type*, buffer_device_type> imports_;
 
    bool
    reallocImportsIfNeeded (const size_t newSize,
                            const bool verbose,
                            const std::string* prefix);
 
    Kokkos::DualView<size_t*, buffer_device_type> numImportPacketsPerLID_;
 
    Kokkos::DualView<packet_type*, buffer_device_type> exports_;
 
    Kokkos::DualView<size_t*, buffer_device_type> numExportPacketsPerLID_;
 
  private:
    using this_type = DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node>;
 
#ifdef HAVE_TPETRA_TRANSFER_TIMERS
    Teuchos::RCP<Teuchos::Time> doXferTimer_;
    Teuchos::RCP<Teuchos::Time> copyAndPermuteTimer_;
    Teuchos::RCP<Teuchos::Time> packAndPrepareTimer_;
    Teuchos::RCP<Teuchos::Time> doPostsAndWaitsTimer_;
    Teuchos::RCP<Teuchos::Time> unpackAndCombineTimer_;
#endif // HAVE_TPETRA_TRANSFER_TIMERS
  }; // class DistObject
} // namespace Tpetra
 
#endif // TPETRA_DISTOBJECT_DECL_HPP