Tpetra_DistObject_decl.hpp

Go to the documentation of this file.

// @HEADER
// ***********************************************************************
//
//          Tpetra: Templated Linear Algebra Services Package
//                 Copyright (2008) Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
//
// ************************************************************************
// @HEADER

#ifndef TPETRA_DISTOBJECT_DECL_HPP
#define TPETRA_DISTOBJECT_DECL_HPP


#include "Tpetra_Map.hpp"
#include "Tpetra_Import.hpp"
#include "Tpetra_Export.hpp"
#include "Tpetra_SrcDistObject.hpp"
#include "Tpetra_DistObject_fwd.hpp"
#include "Kokkos_ArithTraits.hpp"
#include <type_traits>

// #ifndef HAVE_TPETRA_TRANSFER_TIMERS
// #  define HAVE_TPETRA_TRANSFER_TIMERS 1
// #endif // HAVE_TPETRA_TRANSFER_TIMERS

#ifdef HAVE_TPETRA_TRANSFER_TIMERS
#  undef HAVE_TPETRA_TRANSFER_TIMERS
#endif // HAVE_TPETRA_TRANSFER_TIMERS

namespace KokkosClassic {
  enum ReadWriteOption {
    ReadWrite = 0, 
    WriteOnly = 1 
  };
} // namespace KokkosClassic

namespace Tpetra {

  template<class DistObjectType>
  void
  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input,
                               const Teuchos::RCP<const Map<typename DistObjectType::local_ordinal_type,
                                                            typename DistObjectType::global_ordinal_type,
                                                            typename DistObjectType::node_type> >& newMap);

  template<class DistObjectType>
  void
  removeEmptyProcessesInPlace (Teuchos::RCP<DistObjectType>& input);

namespace Classes {

  template <class Packet,
            class LocalOrdinal = ::Tpetra::Details::DefaultTypes::local_ordinal_type,
            class GlobalOrdinal = ::Tpetra::Details::DefaultTypes::global_ordinal_type,
            class Node = ::Tpetra::Details::DefaultTypes::node_type>
  class DistObject :
    virtual public SrcDistObject,
    virtual public Teuchos::Describable
  {
  public:


    typedef typename ::Kokkos::Details::ArithTraits<Packet>::val_type packet_type;
    typedef LocalOrdinal local_ordinal_type;
    typedef GlobalOrdinal global_ordinal_type;
    typedef Node node_type;

    typedef typename Node::device_type device_type;
    typedef typename device_type::execution_space execution_space;

  private:
    typedef typename Kokkos::View<packet_type*, device_type>::size_type view_size_type;
    typedef DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node> this_type;

  public:
    typedef Map<local_ordinal_type, global_ordinal_type, node_type> map_type;



    explicit DistObject (const Teuchos::RCP<const map_type>& map);

    DistObject (const DistObject<Packet, LocalOrdinal, GlobalOrdinal, Node>& rhs);

    virtual ~DistObject ();



    void
    doImport (const SrcDistObject& source,
              const Import<LocalOrdinal, GlobalOrdinal, Node>& importer,
              CombineMode CM);

    void
    doExport (const SrcDistObject& source,
              const Export<LocalOrdinal, GlobalOrdinal, Node>& exporter,
              CombineMode CM);

    void
    doImport (const SrcDistObject& source,
              const Export<LocalOrdinal, GlobalOrdinal, Node>& exporter,
              CombineMode CM);

    void
    doExport (const SrcDistObject& source,
              const Import<LocalOrdinal, GlobalOrdinal, Node>& importer,
              CombineMode CM);



    bool isDistributed () const;

    virtual Teuchos::RCP<const map_type> getMap () const { return map_; }



    void print (std::ostream &os) const;



    virtual std::string description () const;

    virtual void
    describe (Teuchos::FancyOStream &out,
              const Teuchos::EVerbosityLevel verbLevel =
              Teuchos::Describable::verbLevel_default) const;



    virtual void
    removeEmptyProcessesInPlace (const Teuchos::RCP<const map_type>& newMap);


  protected:
    enum ReverseOption {
      DoForward, //*!< Perform the transfer in forward mode.
      DoReverse  //*!< Perform the transfer in reverse mode.
    };

    virtual size_t constantNumberOfPackets () const;

    virtual void
    doTransfer (const SrcDistObject& src,
                const ::Tpetra::Details::Transfer<local_ordinal_type, global_ordinal_type, node_type>& transfer,
                const char modeString[],
                const ReverseOption revOp,
                const CombineMode CM);

    virtual bool
    reallocArraysForNumPacketsPerLid (const size_t numExportLIDs,
                                      const size_t numImportLIDs);

    virtual void
    doTransferOld (const SrcDistObject& src,
                   CombineMode CM,
                   size_t numSameIDs,
                   const Teuchos::ArrayView<const local_ordinal_type> &permuteToLIDs,
                   const Teuchos::ArrayView<const local_ordinal_type> &permuteFromLIDs,
                   const Teuchos::ArrayView<const local_ordinal_type> &remoteLIDs,
                   const Teuchos::ArrayView<const local_ordinal_type> &exportLIDs,
                   Distributor &distor,
                   ReverseOption revOp);

#ifdef KOKKOS_ENABLE_CUDA
    typedef typename std::conditional<
      std::is_same<typename device_type::execution_space, Kokkos::Cuda>::value,
      Kokkos::CudaSpace,
      typename device_type::memory_space>::type buffer_memory_space;
#else
    typedef typename device_type::memory_space buffer_memory_space;
#endif // KOKKOS_ENABLE_CUDA

  public:

    typedef Kokkos::Device<
      typename device_type::execution_space,
      buffer_memory_space> buffer_device_type;

  protected:

    virtual void
    doTransferNew (const SrcDistObject& src,
                   const CombineMode CM,
                   const size_t numSameIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                     device_type>& permuteToLIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                     device_type>& permuteFromLIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                     device_type>& remoteLIDs,
                   const Kokkos::DualView<const local_ordinal_type*,
                     device_type>& exportLIDs,
                   Distributor& distor,
                   const ReverseOption revOp,
                   const bool commOnHost);



    virtual bool
    checkSizes (const SrcDistObject& source) = 0;

    virtual bool useNewInterface () { return false; }

    virtual void
    copyAndPermute (const SrcDistObject& source,
                    size_t numSameIDs,
                    const Teuchos::ArrayView<const local_ordinal_type>& permuteToLIDs,
                    const Teuchos::ArrayView<const local_ordinal_type>& permuteFromLIDs)
    {}
    virtual void
    copyAndPermuteNew (const SrcDistObject& source,
                       const size_t numSameIDs,
                       const Kokkos::DualView<const local_ordinal_type*, device_type>& permuteToLIDs,
                       const Kokkos::DualView<const local_ordinal_type*, device_type>& permuteFromLIDs)
    {}

    virtual void
    packAndPrepare (const SrcDistObject& source,
                    const Teuchos::ArrayView<const local_ordinal_type>& exportLIDs,
                    Teuchos::Array<packet_type>& exports,
                    const Teuchos::ArrayView<size_t>& numPacketsPerLID,
                    size_t& constantNumPackets,
                    Distributor &distor)
    {}

    virtual void
    packAndPrepareNew (const SrcDistObject& source,
                       const Kokkos::DualView<const local_ordinal_type*, device_type>& exportLIDs,
                       Kokkos::DualView<packet_type*, buffer_device_type>& exports,
                       const Kokkos::DualView<size_t*, buffer_device_type>& numPacketsPerLID,
                       size_t& constantNumPackets,
                       Distributor& distor)
    {}

    virtual void
    unpackAndCombine (const Teuchos::ArrayView<const local_ordinal_type>& importLIDs,
                      const Teuchos::ArrayView<const packet_type>& imports,
                      const Teuchos::ArrayView<size_t>& numPacketsPerLID,
                      size_t constantNumPackets,
                      Distributor &distor,
                      CombineMode CM)
    {}

    virtual void
    unpackAndCombineNew (const Kokkos::DualView<const local_ordinal_type*, device_type>& importLIDs,
                         const Kokkos::DualView<const packet_type*, buffer_device_type>& imports,
                         const Kokkos::DualView<const size_t*, buffer_device_type>& numPacketsPerLID,
                         const size_t constantNumPackets,
                         Distributor& distor,
                         const CombineMode CM)
    {}

    virtual void createViews () const;

    virtual void createViewsNonConst (KokkosClassic::ReadWriteOption rwo);

    virtual void releaseViews () const;

    Teuchos::RCP<const map_type> map_;

  protected:
    Kokkos::DualView<packet_type*, buffer_device_type> imports_;

    bool
    reallocImportsIfNeeded (const size_t newSize, const bool debug = false);

    Kokkos::DualView<size_t*, buffer_device_type> numImportPacketsPerLID_;

    Kokkos::DualView<packet_type*, buffer_device_type> exports_;

    Kokkos::DualView<size_t*, buffer_device_type> numExportPacketsPerLID_;

#ifdef HAVE_TPETRA_TRANSFER_TIMERS
  private:
    Teuchos::RCP<Teuchos::Time> doXferTimer_;
    Teuchos::RCP<Teuchos::Time> copyAndPermuteTimer_;
    Teuchos::RCP<Teuchos::Time> packAndPrepareTimer_;
    Teuchos::RCP<Teuchos::Time> doPostsAndWaitsTimer_;
    Teuchos::RCP<Teuchos::Time> unpackAndCombineTimer_;
#endif // HAVE_TPETRA_TRANSFER_TIMERS

  }; // class DistObject

} // namespace Classes

} // namespace Tpetra

#endif // TPETRA_DISTOBJECT_DECL_HPP