Tpetra parallel linear algebra  Version of the Day
Tpetra_Import_def.hpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // ************************************************************************
38 // @HEADER
39 
40 #ifndef TPETRA_IMPORT_DEF_HPP
41 #define TPETRA_IMPORT_DEF_HPP
42 
43 #include "Tpetra_Distributor.hpp"
44 #include "Tpetra_Map.hpp"
45 #include "Tpetra_ImportExportData.hpp"
46 #include "Tpetra_Util.hpp"
47 #include "Tpetra_Import_Util.hpp"
48 #include "Tpetra_Export.hpp"
49 #include "Tpetra_Details_Behavior.hpp"
50 #include "Tpetra_Details_DualViewUtil.hpp"
51 #include "Tpetra_Details_gathervPrint.hpp"
52 #include "Tpetra_Details_Profiling.hpp"
53 #include "Teuchos_as.hpp"
54 #ifdef HAVE_TPETRA_MMM_TIMINGS
55 #include "Teuchos_TimeMonitor.hpp"
56 #endif
57 #include <array>
58 #include <memory>
59 
60 namespace Teuchos {
61  template<class T>
62  std::string toString (const std::vector<T>& x)
63  {
64  std::ostringstream os;
65  os << "[";
66  const std::size_t N = x.size ();
67  for (std::size_t k = 0; k < N; ++k) {
68  os << x[k];
69  if (k + std::size_t (1) < N) {
70  os << ",";
71  }
72  }
73  os << "]";
74  return os.str ();
75  }
76 
77  template<class ElementType, class DeviceType>
78  std::string toString (const Kokkos::View<const ElementType*, DeviceType>& x)
79  {
80  std::ostringstream os;
81  os << "[";
82  const std::size_t N = std::size_t (x.extent (0));
83  for (std::size_t k = 0; k < N; ++k) {
84  os << x[k];
85  if (k + std::size_t (1) < N) {
86  os << ",";
87  }
88  }
89  os << "]";
90  return os.str ();
91  }
92 } // namespace Teuchos
93 
94 namespace Tpetra {
95 
96  // head: init(source, target, true, remotePIDs, Teuchos::null);
97 
98  template <class LocalOrdinal, class GlobalOrdinal, class Node>
99  void
100  Import<LocalOrdinal,GlobalOrdinal,Node>::
101  init (const Teuchos::RCP<const map_type>& source,
102  const Teuchos::RCP<const map_type>& /* target */,
103  bool useRemotePIDs,
104  Teuchos::Array<int> & remotePIDs,
105  const Teuchos::RCP<Teuchos::ParameterList>& plist)
106  {
107  using ::Tpetra::Details::ProfilingRegion;
108  using Teuchos::Array;
109  using Teuchos::null;
110  using Teuchos::Ptr;
111  using Teuchos::rcp;
112  using std::endl;
113  ProfilingRegion regionImportInit ("Tpetra::Import::init");
114 
115  std::unique_ptr<std::string> verbPrefix;
116  if (this->verbose ()) {
117  std::ostringstream os;
118  const int myRank = source->getComm ()->getRank ();
119  os << "Proc " << myRank << ": Tpetra::Import::init: ";
120  verbPrefix = std::unique_ptr<std::string> (new std::string (os.str ()));
121  os << endl;
122  this->verboseOutputStream () << os.str ();
123  }
124 
125  Array<GlobalOrdinal> remoteGIDs;
126 
127 #ifdef HAVE_TPETRA_MMM_TIMINGS
128  using Teuchos::TimeMonitor;
129  std::string label;
130  if(!plist.is_null())
131  label = plist->get("Timer Label",label);
132  std::string prefix = std::string("Tpetra ")+ label + std::string(":iport_ctor:preIData: ");
133 #else
134  (void)plist;
135 #endif
136  {
137 #ifdef HAVE_TPETRA_MMM_TIMINGS
138  auto MM(*TimeMonitor::getNewTimer(prefix));
139 #endif
140  if (this->verbose ()) {
141  std::ostringstream os;
142  os << *verbPrefix << "Call setupSamePermuteRemote" << endl;
143  this->verboseOutputStream () << os.str ();
144  }
145  setupSamePermuteRemote (remoteGIDs);
146  }
147  {
148 #ifdef HAVE_TPETRA_MMM_TIMINGS
149  prefix = std::string("Tpetra ")+ label + std::string(":iport_ctor:preSetupExport: ");
150  auto MM2(*TimeMonitor::getNewTimer(prefix));
151 #endif
152  if (source->isDistributed ()) {
153  if (this->verbose ()) {
154  std::ostringstream os;
155  os << *verbPrefix << "Call setupExport" << endl;
156  this->verboseOutputStream () << os.str ();
157  }
158  setupExport (remoteGIDs,useRemotePIDs,remotePIDs);
159  }
160  else if (this->verbose ()) {
161  std::ostringstream os;
162  os << *verbPrefix << "Source Map not distributed; skip setupExport"
163  << endl;
164  this->verboseOutputStream () << os.str ();
165  }
166  }
167 
168  TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_device () );
169  TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_host () );
170  TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_device () );
171  TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_host () );
172  TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_device () );
173  TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_host () );
174  TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_device () );
175  TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_host () );
176 
177  if (this->verbose ()) {
178  std::ostringstream os;
179  os << *verbPrefix << "Done!" << endl;
180  this->verboseOutputStream () << os.str ();
181  }
182  }
183 
184  template <class LocalOrdinal, class GlobalOrdinal, class Node>
185  Import<LocalOrdinal,GlobalOrdinal,Node>::
186  Import (const Teuchos::RCP<const map_type >& source,
187  const Teuchos::RCP<const map_type >& target) :
188  base_type (source, target, Teuchos::null, Teuchos::null, "Import")
189  {
190  Teuchos::Array<int> dummy;
191 #ifdef HAVE_TPETRA_MMM_TIMINGS
192  Teuchos::RCP<Teuchos::ParameterList> mypars = rcp(new Teuchos::ParameterList);
193  mypars->set("Timer Label","Naive_tAFC");
194  init(source, target, false, dummy, mypars);
195 #else
196  init (source, target, false, dummy, Teuchos::null);
197 #endif
198  }
199 
200  template <class LocalOrdinal, class GlobalOrdinal, class Node>
201  Import<LocalOrdinal,GlobalOrdinal,Node>::
202  Import (const Teuchos::RCP<const map_type>& source,
203  const Teuchos::RCP<const map_type>& target,
204  const Teuchos::RCP<Teuchos::FancyOStream>& out) :
205  base_type (source, target, out, Teuchos::null, "Import")
206  {
207  Teuchos::Array<int> dummy;
208  init (source, target, false, dummy, Teuchos::null);
209  }
210 
211  template <class LocalOrdinal, class GlobalOrdinal, class Node>
212  Import<LocalOrdinal,GlobalOrdinal,Node>::
213  Import (const Teuchos::RCP<const map_type>& source,
214  const Teuchos::RCP<const map_type>& target,
215  const Teuchos::RCP<Teuchos::ParameterList>& plist) :
216  base_type (source, target, Teuchos::null, plist, "Import")
217  {
218  Teuchos::Array<int> dummy;
219  init (source, target, false, dummy, plist);
220  }
221 
222  template <class LocalOrdinal, class GlobalOrdinal, class Node>
223  Import<LocalOrdinal,GlobalOrdinal,Node>::
224  Import (const Teuchos::RCP<const map_type>& source,
225  const Teuchos::RCP<const map_type>& target,
226  const Teuchos::RCP<Teuchos::FancyOStream>& out,
227  const Teuchos::RCP<Teuchos::ParameterList>& plist) :
228  base_type (source, target, out, plist, "Import")
229  {
230  Teuchos::Array<int> dummy;
231  init (source, target, false, dummy, plist);
232  }
233 
234  template <class LocalOrdinal, class GlobalOrdinal, class Node>
235  Import<LocalOrdinal,GlobalOrdinal,Node>::
236  Import (const Teuchos::RCP<const map_type>& source,
237  const Teuchos::RCP<const map_type>& target,
238  Teuchos::Array<int>& remotePIDs,
239  const Teuchos::RCP<Teuchos::ParameterList>& plist) :
240  base_type (source, target, Teuchos::null, plist, "Import")
241  {
242  init (source, target, true, remotePIDs, plist);
243  }
244 
245  template <class LocalOrdinal, class GlobalOrdinal, class Node>
246  Import<LocalOrdinal,GlobalOrdinal,Node>::
247  Import (const Import<LocalOrdinal,GlobalOrdinal,Node>& rhs) :
248  base_type (rhs)
249  {}
250 
251  template <class LocalOrdinal, class GlobalOrdinal, class Node>
252  Import<LocalOrdinal,GlobalOrdinal,Node>::
253  Import (const Export<LocalOrdinal,GlobalOrdinal,Node>& exporter) :
254  base_type (exporter, typename base_type::reverse_tag ())
255  {}
256 
257  // cblcbl
258  // This is the "createExpert" version of the constructor to be used with pid/gid pairs obtained from
259  // reverse communication
260  template <class LocalOrdinal, class GlobalOrdinal, class Node>
261  Import<LocalOrdinal,GlobalOrdinal,Node>::
262  Import (const Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >& source,
263  const Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >& target,
264  const Teuchos::ArrayView<int> & userRemotePIDs,
265  const Teuchos::ArrayView<const LocalOrdinal> & userExportLIDs,
266  const Teuchos::ArrayView<const int> & userExportPIDs,
267  const Teuchos::RCP<Teuchos::ParameterList>& plist,
268  const Teuchos::RCP<Teuchos::FancyOStream>& out) :
269  base_type (source, target, out, plist, "Import")
270  {
271  using ::Tpetra::Details::makeDualViewFromArrayView;
272  using Teuchos::arcp;
273  using Teuchos::Array;
274  using Teuchos::ArrayRCP;
275  using Teuchos::ArrayView;
276  using Teuchos::as;
277  using Teuchos::null;
278  using Teuchos::rcp;
279  using std::endl;
280  using LO = LocalOrdinal;
281  using GO = GlobalOrdinal;
282  using size_type = Teuchos::Array<int>::size_type;
283 
284  std::unique_ptr<std::string> prefix;
285  if (this->verbose ()) {
286  auto comm = source.is_null () ? Teuchos::null : source->getComm ();
287  const int myRank = comm.is_null () ? -1 : comm->getRank ();
288  std::ostringstream os;
289  os << "Proc " << myRank << ": Tpetra::Import createExpert ctor: ";
290  prefix = std::unique_ptr<std::string> (new std::string (os.str ()));
291  os << "Start" << endl;
292  this->verboseOutputStream () << os.str ();
293  }
294 
295  ArrayView<const GO> sourceGIDs = source->getNodeElementList ();
296  ArrayView<const GO> targetGIDs = target->getNodeElementList ();
297 
298  Array<GO> tRemoteGIDs;
299  if (this->verbose ()) {
300  std::ostringstream os;
301  os << *prefix << "Call setupSamePermuteRemote" << endl;
302  this->verboseOutputStream () << os.str ();
303  }
304  setupSamePermuteRemote (tRemoteGIDs);
305 
306  if (this->verbose ()) {
307  std::ostringstream os;
308  os << *prefix << "Sort & filter IDs" << endl;
309  this->verboseOutputStream () << os.str ();
310  }
311 
312  auto tRemoteLIDs = this->TransferData_->remoteLIDs_.view_host ();
313  this->TransferData_->remoteLIDs_.modify_host ();
314  Teuchos::Array<int> tRemotePIDs (userRemotePIDs);
315 
316  if (this->verbose () && this->getNumRemoteIDs () > 0 && ! source->isDistributed ()) {
317  std::ostringstream os;
318  os << *prefix << "Target Map has remote LIDs but source Map is not "
319  "distributed. Importing to a submap of the target Map." << endl;
320  this->verboseOutputStream () << os.str ();
321  }
322  // FIXME (mfh 03 Feb 2019) I don't see this as "abuse"; it's
323  // perfectly valid Petra Object Model behavior.
324  TPETRA_ABUSE_WARNING
325  (getNumRemoteIDs () > 0 && ! source->isDistributed (),
326  std::runtime_error,
327  "::constructExpert: Target Map has remote LIDs but source Map "
328  "is not distributed. Importing to a submap of the target Map.");
329  TEUCHOS_TEST_FOR_EXCEPTION
330  (tRemotePIDs.size () != tRemoteGIDs.size () ||
331  size_t (tRemoteGIDs.size ()) != size_t (tRemoteLIDs.extent (0)),
332  std::runtime_error, "Import::Import createExpert version: "
333  "Size mismatch on userRemotePIDs, remoteGIDs, and remoteLIDs "
334  "Array's to sort3.");
335 
336  sort3 (tRemotePIDs.begin (),
337  tRemotePIDs.end (),
338  tRemoteGIDs.begin (),
339  tRemoteLIDs.data ());
340 
341  //Get rid of IDs that don't exist in SourceMap
342  size_type cnt = 0;
343  size_type indexIntoRemotePIDs = tRemotePIDs.size ();
344  for (size_type i = 0; i < indexIntoRemotePIDs; ++i) {
345  if (tRemotePIDs[i] == -1) {
346  ++cnt;
347  }
348  }
349 
350  if (cnt == 0) { // done modifying remoteLIDs_
351  this->TransferData_->remoteLIDs_.sync_device ();
352  }
353  else {
354  if (indexIntoRemotePIDs - cnt > 0) {
355  Array<GO> newRemoteGIDs(indexIntoRemotePIDs-cnt);
356  Array<LO> newRemoteLIDs(indexIntoRemotePIDs-cnt);
357  Array<int> newRemotePIDs(indexIntoRemotePIDs-cnt);
358  cnt = 0;
359  for (size_type j = 0; j < indexIntoRemotePIDs; ++j)
360  if(tRemotePIDs[j] != -1) {
361  newRemoteGIDs[cnt] = tRemoteGIDs[j];
362  newRemotePIDs[cnt] = tRemotePIDs[j];
363  newRemoteLIDs[cnt] = target->getLocalElement(tRemoteGIDs[j]);
364  ++cnt;
365  }
366  indexIntoRemotePIDs = cnt;
367  tRemoteGIDs = newRemoteGIDs;
368  tRemotePIDs = newRemotePIDs;
369  makeDualViewFromArrayView (this->TransferData_->remoteLIDs_,
370  newRemoteLIDs ().getConst (),
371  "remoteLIDs");
372  }
373  else { //valid RemoteIDs empty
374  indexIntoRemotePIDs = 0;
375  tRemoteGIDs.clear();
376  tRemotePIDs.clear();
377  this->TransferData_->remoteLIDs_ = decltype (this->TransferData_->remoteLIDs_) ();
378  }
379  }
380 
381  this->TransferData_->exportPIDs_ = Teuchos::Array<int> (userExportPIDs);
382  makeDualViewFromArrayView (this->TransferData_->exportLIDs_,
383  userExportLIDs, "exportLIDs");
384 
385  bool locallyComplete = true;
386  for (size_type i = 0; i < userExportPIDs.size () && locallyComplete; ++i) {
387  if (userExportPIDs[i] == -1) {
388  locallyComplete = false;
389  }
390  }
391  this->TransferData_->isLocallyComplete_ = locallyComplete;
392 
393  if (this->verbose ()) {
394  std::ostringstream os;
395  os << *prefix << "locallyComplete: "
396  << (locallyComplete ? "true" : "false")
397  << "; call createFromSendsAndRecvs" << endl;
398  this->verboseOutputStream () << os.str ();
399  }
400  {
401 #ifdef HAVE_TPETRA_MMM_TIMINGS
402  std::string mmm_prefix =
403  std::string("Tpetra ") + std::string(":iport_ctor:cFSAR ");
404 
405  auto MM3(*Teuchos::TimeMonitor::getNewTimer(mmm_prefix));
406 #endif
407  Distributor& distributor = this->TransferData_->distributor_;
408  distributor.createFromSendsAndRecvs (this->TransferData_->exportPIDs_, tRemotePIDs);
409  }
410 
411  TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_device () );
412  TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_host () );
413  TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_device () );
414  TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_host () );
415  TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_device () );
416  TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_host () );
417  TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_device () );
418  TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_host () );
419  }
420 
421  template <class LocalOrdinal, class GlobalOrdinal, class Node>
422  Import<LocalOrdinal,GlobalOrdinal,Node>::
423  Import (const Teuchos::RCP<const map_type>& source,
424  const Teuchos::RCP<const map_type>& target,
425  const size_t numSameIDs,
426  Teuchos::Array<LocalOrdinal>& permuteToLIDs,
427  Teuchos::Array<LocalOrdinal>& permuteFromLIDs,
428  Teuchos::Array<LocalOrdinal>& remoteLIDs,
429  Teuchos::Array<LocalOrdinal>& exportLIDs,
430  Teuchos::Array<int>& exportPIDs,
431  Distributor& distributor,
432  const Teuchos::RCP<Teuchos::FancyOStream>& out,
433  const Teuchos::RCP<Teuchos::ParameterList>& plist) :
434  base_type (source, target, out, plist, "Import")
435  {
436  using ::Tpetra::Details::makeDualViewFromArrayView;
437  using std::endl;
438 
439  std::unique_ptr<std::string> prefix;
440  if (this->verbose ()) {
441  auto comm = source.is_null () ? Teuchos::null : source->getComm ();
442  const int myRank = comm.is_null () ? -1 : comm->getRank ();
443  std::ostringstream os;
444  os << "Proc " << myRank << ": Tpetra::Import export ctor: ";
445  prefix = std::unique_ptr<std::string> (new std::string (os.str ()));
446  os << "Start" << endl;
447  this->verboseOutputStream () << os.str ();
448  }
449 
450  bool locallyComplete = true;
451  for (Teuchos::Array<int>::size_type i = 0; i < exportPIDs.size (); ++i) {
452  if (exportPIDs[i] == -1) {
453  locallyComplete = false;
454  }
455  }
456  if (this->verbose ()) {
457  std::ostringstream os;
458  os << *prefix << "numSameIDs: " << numSameIDs << ", locallyComplete: "
459  << (locallyComplete ? "true" : "false") << endl;
460  this->verboseOutputStream () << os.str ();
461  }
462 
463  this->TransferData_->isLocallyComplete_ = locallyComplete;
464  this->TransferData_->numSameIDs_ = numSameIDs;
465 
466  makeDualViewFromArrayView (this->TransferData_->permuteToLIDs_,
467  permuteToLIDs ().getConst (),
468  "permuteToLIDs");
469  TEUCHOS_ASSERT( size_t (this->TransferData_->permuteToLIDs_.extent (0)) ==
470  size_t (permuteToLIDs.size ()) );
471  makeDualViewFromArrayView (this->TransferData_->permuteFromLIDs_,
472  permuteFromLIDs ().getConst (),
473  "permuteFromLIDs");
474  TEUCHOS_ASSERT( size_t (this->TransferData_->permuteFromLIDs_.extent (0)) ==
475  size_t (permuteFromLIDs.size ()) );
476  makeDualViewFromArrayView (this->TransferData_->remoteLIDs_,
477  remoteLIDs ().getConst (),
478  "remoteLIDs");
479  TEUCHOS_ASSERT( size_t (this->TransferData_->remoteLIDs_.extent (0)) ==
480  size_t (remoteLIDs.size ()) );
481  makeDualViewFromArrayView (this->TransferData_->exportLIDs_,
482  exportLIDs ().getConst (),
483  "exportLIDs");
484  TEUCHOS_ASSERT( size_t (this->TransferData_->exportLIDs_.extent (0)) ==
485  size_t (exportLIDs.size ()) );
486  this->TransferData_->exportPIDs_.swap (exportPIDs);
487  this->TransferData_->distributor_.swap (distributor);
488 
489  TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_device () );
490  TEUCHOS_ASSERT( ! this->TransferData_->permuteFromLIDs_.need_sync_host () );
491  TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_device () );
492  TEUCHOS_ASSERT( ! this->TransferData_->permuteToLIDs_.need_sync_host () );
493  TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_device () );
494  TEUCHOS_ASSERT( ! this->TransferData_->remoteLIDs_.need_sync_host () );
495  TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_device () );
496  TEUCHOS_ASSERT( ! this->TransferData_->exportLIDs_.need_sync_host () );
497  }
498 
499  namespace { // (anonymous)
500 
501  template <class LO, class GO, class NT>
502  struct ImportLocalSetupResult
503  {
504  Teuchos::RCP<const ::Tpetra::Map<LO, GO, NT> > targetMap;
505  LO numSameIDs;
506  // std::vector<LO> permuteToLIDs; // users aren't supposed to have permutes
507  // std::vector<LO> permuteFromLIDs; // users aren't suppoosed to have permutes
508  std::vector<GO> remoteGIDs;
509  std::vector<LO> remoteLIDs;
510  std::vector<int> remotePIDs;
511  LO numPermutes; // users aren't supposed to have permutes
512  };
513 
514  template<class T>
515  void printArray (std::ostream& out, const T x[], const std::size_t N)
516  {
517  out << "[";
518  for (std::size_t k = 0; k < N; ++k) {
519  out << x[k];
520  if (k + 1 < N) {
521  out << ", ";
522  }
523  }
524  out << "]";
525  }
526 
527  template<class LO, class GO, class NT>
528  ImportLocalSetupResult<LO, GO, NT>
529  setupSamePermuteRemoteFromUserGlobalIndexList (const ::Tpetra::Map<LO, GO, NT>& sourceMap,
530  const GO targetMapRemoteOrPermuteGlobalIndices[],
531  const int targetMapRemoteOrPermuteProcessRanks[],
532  const LO numTargetMapRemoteOrPermuteGlobalIndices,
533  const bool mayReorderTargetMapIndicesLocally,
534  Teuchos::FancyOStream* out, // only valid if verbose
535  const std::string* verboseHeader, // only valid if verbose
536  const bool verbose,
537  const bool debug)
538  {
539  using std::endl;
540  const int myRank = sourceMap.getComm ()->getRank ();
541  ImportLocalSetupResult<LO, GO, NT> result;
542 
543  if (verbose) {
544  std::ostringstream os;
545  os << *verboseHeader << "- Import::setupSPR w/ remote GIDs & PIDs: " << endl
546  << *verboseHeader << " Input GIDs: ";
547  printArray (os, targetMapRemoteOrPermuteGlobalIndices, numTargetMapRemoteOrPermuteGlobalIndices);
548  os << endl << " Input PIDs: ";
549  printArray (os, targetMapRemoteOrPermuteProcessRanks, numTargetMapRemoteOrPermuteGlobalIndices);
550  os << endl;
551  *out << os.str ();
552  }
553 
554  // In debug mode, check whether any of the input GIDs are
555  // actually in the source Map on the calling process. That's an
556  // error, because it means duplicate GIDs on the calling
557  // process. Also check if any of the input PIDs are invalid.
558  if (debug) {
559  std::vector<GO> badGIDs;
560  std::vector<int> badPIDs;
561  const Teuchos::Comm<int>& comm = * (sourceMap.getComm ());
562  const int numProcs = comm.getSize ();
563 
564  for (LO k = 0; k < numTargetMapRemoteOrPermuteGlobalIndices; ++k) {
565  const GO tgtGID = targetMapRemoteOrPermuteGlobalIndices[k];
566  if (sourceMap.isNodeGlobalElement (tgtGID)) {
567  badGIDs.push_back (tgtGID);
568  }
569  const int tgtPID = targetMapRemoteOrPermuteProcessRanks[k];
570  if (tgtPID < 0 || tgtPID >= numProcs) {
571  badPIDs.push_back (tgtPID);
572  }
573  }
574 
575  std::array<int, 2> lclStatus {{
576  badGIDs.size () == 0 ? 1 : 0,
577  badPIDs.size () == 0 ? 1 : 0
578  }};
579  std::array<int, 2> gblStatus {{0, 0}}; // output argument
580  Teuchos::reduceAll<int, int> (comm, Teuchos::REDUCE_MIN, 2,
581  lclStatus.data (), gblStatus.data ());
582  const bool good = gblStatus[0] == 1 && gblStatus[1] == 1;
583  // Don't actually print all the "bad" GIDs and/or PIDs unless
584  // in verbose mode, since there could be many of them.
585  if (verbose && gblStatus[0] != 1) {
586  std::ostringstream os;
587  os << *verboseHeader << "- Some input GIDs are already in the source Map: ";
588  printArray (os, badGIDs.data (), badGIDs.size ());
589  os << endl;
590  ::Tpetra::Details::gathervPrint (*out, os.str (), comm);
591  }
592  if (verbose && gblStatus[0] != 1) {
593  std::ostringstream os;
594  os << *verboseHeader << "- Some input PIDs are invalid: ";
595  printArray (os, badPIDs.data (), badPIDs.size ());
596  os << endl;
597  ::Tpetra::Details::gathervPrint (*out, os.str (), comm);
598  }
599 
600  if (! good) {
601  std::ostringstream os;
602  os << "Tpetra::Import constructor that takes remote GIDs and PIDs: ";
603  if (gblStatus[0] != 1) {
604  os << "Some input GIDs (global indices) are already in the source Map! ";
605  }
606  if (gblStatus[1] != 1) {
607  os << "Some input PIDs (process ranks) are invalid! ";
608  }
609  os << "Rerun with the environment variable TPETRA_VERBOSE=Tpetra::Import "
610  "to see what GIDs and/or PIDs are bad.";
611  TEUCHOS_TEST_FOR_EXCEPTION(true, std::invalid_argument, os.str ());
612  }
613  }
614 
615  // Create list of GIDs to go into target Map. We need to copy
616  // the GIDs into this list anyway, so once we have them, we can
617  // sort the "remotes" in place.
618  const LO numLclSrcIDs = static_cast<LO> (sourceMap.getNodeNumElements ());
619  const LO numLclTgtIDs = numLclSrcIDs + numTargetMapRemoteOrPermuteGlobalIndices;
620  if (verbose) {
621  std::ostringstream os;
622  os << *verboseHeader << "- Copy source Map GIDs into target Map GID list: "
623  "numLclSrcIDs=" << numLclSrcIDs
624  << ", numTargetMapRemoteOrPermuteGlobalIndices="
625  << numTargetMapRemoteOrPermuteGlobalIndices << endl;
626  *out << os.str ();
627  }
628  std::vector<GO> tgtGIDs (numLclTgtIDs); // will go into target Map ctor
629  if (sourceMap.isContiguous ()) {
630  GO curTgtGID = sourceMap.getMinGlobalIndex ();
631  for (LO k = 0; k < numLclSrcIDs; ++k, ++curTgtGID) {
632  tgtGIDs[k] = curTgtGID;
633  }
634  }
635  else { // avoid calling getNodeElementList on a contiguous Map
636  auto srcGIDs = sourceMap.getNodeElementList (); // Teuchos::ArrayView has a different
637  for (LO k = 0; k < numLclSrcIDs; ++k) { // iterator type, so can't std::copy
638  tgtGIDs[k] = srcGIDs[k];
639  }
640  }
641  std::copy (targetMapRemoteOrPermuteGlobalIndices,
642  targetMapRemoteOrPermuteGlobalIndices + numTargetMapRemoteOrPermuteGlobalIndices,
643  tgtGIDs.begin () + numLclSrcIDs);
644 
645  // Optionally, sort input by process rank, so that remotes
646  // coming from the same process are grouped together. Only sort
647  // remote GIDs. While doing so, detect permutes (input "remote"
648  // GIDs whose rank is the same as that of the calling process).
649  //
650  // Permutes are actually an error. We normally detect them in
651  // debug mode, but if we sort, we have a nearly free opportunity
652  // to do so. We may also safely ignore permutes as duplicates.
653  //
654  // NOTE: tgtPIDs only includes remotes, not source Map entries.
655  if (verbose) {
656  std::ostringstream os;
657  os << *verboseHeader << "- Sort by PID? "
658  << (mayReorderTargetMapIndicesLocally ? "true" : "false") << endl;
659  *out << os.str ();
660  }
661  std::vector<int> tgtPIDs (targetMapRemoteOrPermuteProcessRanks,
662  targetMapRemoteOrPermuteProcessRanks + numTargetMapRemoteOrPermuteGlobalIndices);
663  result.numPermutes = 0;
664  if (mayReorderTargetMapIndicesLocally) {
665  Tpetra::sort2 (tgtPIDs.begin (), tgtPIDs.end (), tgtGIDs.begin () + numLclSrcIDs);
666  auto range = std::equal_range (tgtPIDs.begin (), tgtPIDs.end (), myRank); // binary search
667  if (range.second > range.first) {
668  result.numPermutes = static_cast<LO> (range.second - range.first);
669  }
670  }
671  else { // don't sort; linear search to count permutes
672  result.numPermutes = static_cast<LO> (std::count (tgtPIDs.begin (), tgtPIDs.end (), myRank));
673  }
674  // The _actual_ number of remotes.
675  const LO numRemotes = numTargetMapRemoteOrPermuteGlobalIndices - result.numPermutes;
676  result.numSameIDs = static_cast<LO> (sourceMap.getNodeNumElements ());
677 
678  if (verbose) {
679  std::ostringstream os;
680  os << *verboseHeader << "- numSame=" << result.numSameIDs
681  << ", numPermutes=" << result.numPermutes
682  << ", numRemotes=" << numRemotes << endl;
683  *out << os.str ();
684  }
685 
686  if (result.numPermutes == 0) {
687  if (verbose) {
688  std::ostringstream os;
689  os << *verboseHeader << "- No permutes" << endl;
690  *out << os.str ();
691  }
692  result.remoteGIDs = std::vector<GO> (tgtGIDs.begin () + numLclSrcIDs, tgtGIDs.end ());
693  result.remotePIDs.swap (tgtPIDs);
694  result.remoteLIDs.resize (numRemotes);
695  for (LO k = 0; k < numRemotes; ++k) {
696  const LO tgtLid = result.numSameIDs + k;
697  result.remoteLIDs[k] = tgtLid;
698  }
699  if (verbose) {
700  std::ostringstream os;
701  os << *verboseHeader << "- Remote GIDs: "
702  << Teuchos::toString (result.remoteGIDs) << endl;
703  os << *verboseHeader << "- Remote PIDs: "
704  << Teuchos::toString (result.remotePIDs) << endl;
705  os << *verboseHeader << "- Remote LIDs: "
706  << Teuchos::toString (result.remoteLIDs) << endl;
707  *out << os.str ();
708  }
709  }
710  else { // separate permutes from remotes
711  // This case doesn't need to be optimal; it just needs to be
712  // correct. Users really shouldn't give permutes to this
713  // Import constructor.
714  result.remoteGIDs.reserve (numRemotes);
715  result.remoteLIDs.reserve (numRemotes);
716  result.remotePIDs.reserve (numRemotes);
717  for (LO k = 0; k < numTargetMapRemoteOrPermuteGlobalIndices; ++k) {
718  const LO tgtLid = result.numSameIDs + k;
719  const GO tgtGid = tgtGIDs[numLclSrcIDs + k];
720  const int tgtPid = tgtPIDs[k];
721 
722  if (tgtPid != myRank) { // it's a remote
723  result.remoteGIDs.push_back (tgtGid);
724  result.remoteLIDs.push_back (tgtLid);
725  result.remotePIDs.push_back (tgtPid);
726  }
727  }
728  if (verbose) {
729  std::ostringstream os;
730  os << *verboseHeader << "- Some permutes" << endl;
731  *out << os.str ();
732  }
733  }
734 
735  if (sourceMap.isDistributed ()) {
736  if (verbose) {
737  std::ostringstream os;
738  os << *verboseHeader << "- Sort remotes by PID, as Import always does"
739  << endl
740  << *verboseHeader << "-- remotePIDs before: "
741  << Teuchos::toString (result.remotePIDs) << endl
742  << *verboseHeader << "-- remoteGIDs before: "
743  << Teuchos::toString (result.remoteGIDs) << endl
744  << *verboseHeader << "-- remoteLIDs before: "
745  << Teuchos::toString (result.remoteLIDs) << endl;
746  *out << os.str ();
747  }
748  // Import always sorts these, regardless of what the user wanted.
749  sort3 (result.remotePIDs.begin (),
750  result.remotePIDs.end (),
751  result.remoteGIDs.begin (),
752  result.remoteLIDs.begin ());
753  if (verbose) {
754  std::ostringstream os;
755  os << *verboseHeader << "-- remotePIDs after: "
756  << Teuchos::toString (result.remotePIDs) << endl
757  << *verboseHeader << "-- remoteGIDs after: "
758  << Teuchos::toString (result.remoteGIDs) << endl
759  << *verboseHeader << "-- remoteLIDs after: "
760  << Teuchos::toString (result.remoteLIDs) << endl;
761  std::cerr << os.str ();
762  }
763  }
764 
765  if (verbose) {
766  std::ostringstream os;
767  os << *verboseHeader << "- Make target Map" << endl;
768  *out << os.str ();
769  }
770  using ::Teuchos::rcp;
771  typedef ::Tpetra::Map<LO, GO, NT> map_type;
772  typedef ::Tpetra::global_size_t GST;
773  const GST MAP_COMPUTES_GLOBAL_COUNT = ::Teuchos::OrdinalTraits<GST>::invalid ();
774  result.targetMap = rcp (new map_type (MAP_COMPUTES_GLOBAL_COUNT,
775  tgtGIDs.data (),
776  numLclTgtIDs,
777  sourceMap.getIndexBase (),
778  sourceMap.getComm ()));
779  if (verbose) {
780  std::ostringstream os;
781  os << *verboseHeader << "- Done with sameSPR..." << endl;
782  *out << os.str ();
783  }
784  return result;
785  }
786  } // namespace (anonymous)
787 
788  template <class LocalOrdinal, class GlobalOrdinal, class Node>
789  Import<LocalOrdinal,GlobalOrdinal,Node>::
790  Import (const Teuchos::RCP<const Map<LocalOrdinal, GlobalOrdinal, Node> >& sourceMap,
791  const GlobalOrdinal targetMapRemoteOrPermuteGlobalIndices[],
792  const int targetMapRemoteOrPermuteProcessRanks[],
793  const LocalOrdinal numTargetMapRemoteOrPermuteGlobalIndices,
794  const bool mayReorderTargetMapIndicesLocally,
795  const Teuchos::RCP<Teuchos::ParameterList>& plist,
796  const Teuchos::RCP<Teuchos::FancyOStream>& debugOutput) :
797  // Special case: target Map is null on base_type construction.
798  // It's worthwhile for invariants like out_ not being null.
799  // We'll set TransferData_ again below.
800  base_type (sourceMap, Teuchos::null, debugOutput, plist, "Import")
801  {
802  using ::Tpetra::Details::Behavior;
803  using ::Tpetra::Details::makeDualViewFromOwningHostView;
804  using ::Tpetra::Details::makeDualViewFromVector;
805  using ::Tpetra::Details::printDualView;
806  using ::Tpetra::Details::view_alloc_no_init;
807  using Teuchos::ArrayView;
808  using Teuchos::getFancyOStream;
809  using Teuchos::RCP;
810  using Teuchos::rcp;
811  using Teuchos::rcpFromRef;
812  using std::endl;
813  typedef LocalOrdinal LO;
814  typedef GlobalOrdinal GO;
815  typedef Node NT;
816 
817  const bool debug = Behavior::debug ("Import") ||
818  Behavior::debug ("Tpetra::Import");
819 
820  std::unique_ptr<std::string> verbPfx;
821  if (this->verbose ()) {
822  std::ostringstream os;
823  const int myRank = sourceMap->getComm ()->getRank ();
824  os << "Proc " << myRank << ": Tpetra::Import ctor from remotes: ";
825  verbPfx = std::unique_ptr<std::string> (new std::string (os.str ()));
826  os << "mayReorder=" << (mayReorderTargetMapIndicesLocally ? "true" : "false")
827  << endl;
828  this->verboseOutputStream () << os.str ();
829  }
830 
831  TEUCHOS_ASSERT( ! this->TransferData_.is_null () );
832  ImportLocalSetupResult<LO, GO, NT> localSetupResult =
833  setupSamePermuteRemoteFromUserGlobalIndexList<LO, GO, NT>
834  (*sourceMap,
835  targetMapRemoteOrPermuteGlobalIndices,
836  targetMapRemoteOrPermuteProcessRanks,
837  numTargetMapRemoteOrPermuteGlobalIndices,
838  mayReorderTargetMapIndicesLocally,
839  this->TransferData_->out_.getRawPtr (),
840  verbPfx.get (),
841  this->verbose (),
842  debug);
843 
844  // Since we invoked the base_type constructor above, we know that
845  // out_ is nonnull, so we don't have to waste time creating it
846  // again.
847  using data_type = ImportExportData<LO, GO, NT>;
848  TEUCHOS_ASSERT( ! this->TransferData_.is_null () );
849  this->TransferData_ = rcp (new data_type (sourceMap,
850  localSetupResult.targetMap,
851  this->TransferData_->out_,
852  plist));
853  this->TransferData_->numSameIDs_ = localSetupResult.numSameIDs;
854  // Skip permutes; they are user error, because they duplicate
855  // non-remote indices.
856  makeDualViewFromVector (this->TransferData_->remoteLIDs_,
857  localSetupResult.remoteLIDs,
858  "remoteLIDs");
859  // "Is locally complete" for an Import means that all target Map
860  // indices on the calling process exist on at least one process
861  // (not necessarily this one) in the source Map. For this
862  // constructor, this is true if and only if all input target PIDs
863  // are valid PIDs in the communicator.
864  //
865  // FIXME (mfh 20 Feb 2018) For now, assume this is always true.
866  this->TransferData_->isLocallyComplete_ = true;
867 
868  Teuchos::Array<GO> exportGIDs;
869  if (sourceMap->isDistributed ()) {
870  if (this->verbose ()) {
871  std::ostringstream os;
872  os << *verbPfx << "Make Distributor (createFromRecvs)" << endl;
873  this->verboseOutputStream () << os.str ();
874  }
875  ArrayView<const GO> remoteGIDs (localSetupResult.remoteGIDs.data (),
876  localSetupResult.remoteGIDs.size ());
877  ArrayView<const int> remotePIDs (localSetupResult.remotePIDs.data (),
878  localSetupResult.remotePIDs.size ());
879  // Call Distributor::createFromRecvs to turn the remote GIDs and
880  // their owning PIDs into a send-and-receive communication plan.
881  // remoteGIDs and remotePIDs are input; exportGIDs and
882  // exportPIDs are output arrays that createFromRecvs allocates.
883  Distributor& distributor = this->TransferData_->distributor_;
884  distributor.createFromRecvs (remoteGIDs,
885  remotePIDs,
886  exportGIDs,
887  this->TransferData_->exportPIDs_);
888  // Find the LIDs corresponding to the (outgoing) GIDs in
889  // exportGIDs. For sparse matrix-vector multiply, this tells
890  // the calling process how to index into the source vector to
891  // get the elements which it needs to send.
892  //
893  // NOTE (mfh 03 Mar 2014) This is now a candidate for a
894  // thread-parallel kernel, but only if using the new thread-safe
895  // Map implementation.
896  if (this->verbose ()) {
897  std::ostringstream os;
898  os << *verbPfx << "Compute exportLIDs" << endl;
899  this->verboseOutputStream () << os.str ();
900  }
901  using size_type = typename Teuchos::Array<GO>::size_type;
902  const size_type numExportIDs = exportGIDs.size ();
903 
904  typename decltype (this->TransferData_->exportLIDs_)::t_host
905  exportLIDs (view_alloc_no_init ("exportLIDs"), numExportIDs);
906  for (size_type k = 0; k < numExportIDs; ++k) {
907  exportLIDs[k] = sourceMap->getLocalElement (exportGIDs[k]);
908  }
909  makeDualViewFromOwningHostView (this->TransferData_->exportLIDs_, exportLIDs);
910  }
911 
912  if (this->verbose ()) {
913  std::ostringstream os;
914  os << *verbPfx;
915  printDualView (os, this->TransferData_->remoteLIDs_,
916  "ImportExportData::remoteLIDs_");
917  os << endl;
918  this->verboseOutputStream () << os.str ();
919  }
920  if (this->verbose ()) {
921  std::ostringstream os;
922  os << *verbPfx << "Done!" << endl;
923  this->verboseOutputStream () << os.str ();
924  }
925  }
926 
927  template <class LocalOrdinal, class GlobalOrdinal, class Node>
928  void
929  Import<LocalOrdinal,GlobalOrdinal,Node>::
930  describe (Teuchos::FancyOStream& out,
931  const Teuchos::EVerbosityLevel verbLevel) const
932  {
933  // Call the base class' method. It does all the work.
934  this->describeImpl (out, "Tpetra::Import", verbLevel);
935  }
936 
937  template <class LocalOrdinal, class GlobalOrdinal, class Node>
938  void Import<LocalOrdinal,GlobalOrdinal,Node>::
939  print (std::ostream& os) const
940  {
941  auto out = Teuchos::getFancyOStream (Teuchos::rcpFromRef (os));
942  // "Print" traditionally meant "everything."
943  this->describe (*out, Teuchos::VERB_EXTREME);
944  }
945 
946  template <class LocalOrdinal, class GlobalOrdinal, class Node>
947  void
948  Import<LocalOrdinal,GlobalOrdinal,Node>::
949  setupSamePermuteRemote (Teuchos::Array<GlobalOrdinal>& remoteGIDs)
950  {
951  using ::Tpetra::Details::makeDualViewFromOwningHostView;
952  using ::Tpetra::Details::ProfilingRegion;
953  using ::Tpetra::Details::view_alloc_no_init;
954  using Teuchos::arcp;
955  using Teuchos::Array;
956  using Teuchos::ArrayRCP;
957  using Teuchos::ArrayView;
958  using Teuchos::as;
959  using Teuchos::null;
960  typedef LocalOrdinal LO;
961  typedef GlobalOrdinal GO;
962  typedef typename ArrayView<const GO>::size_type size_type;
963  ProfilingRegion regionExport ("Tpetra::Import::setupSamePermuteRemote");
964 
965  const map_type& source = * (this->getSourceMap ());
966  const map_type& target = * (this->getTargetMap ());
967  ArrayView<const GO> sourceGIDs = source.getNodeElementList ();
968  ArrayView<const GO> targetGIDs = target.getNodeElementList ();
969 
970 #ifdef HAVE_TPETRA_DEBUG
971  ArrayView<const GO> rawSrcGids = sourceGIDs;
972  ArrayView<const GO> rawTgtGids = targetGIDs;
973 #else
974  const GO* const rawSrcGids = sourceGIDs.getRawPtr ();
975  const GO* const rawTgtGids = targetGIDs.getRawPtr ();
976 #endif // HAVE_TPETRA_DEBUG
977  const size_type numSrcGids = sourceGIDs.size ();
978  const size_type numTgtGids = targetGIDs.size ();
979  const size_type numGids = std::min (numSrcGids, numTgtGids);
980 
981  // Compute numSameIDs_: the number of initial GIDs that are the
982  // same (and occur in the same order) in both Maps. The point of
983  // numSameIDs_ is for the common case of an Import where all the
984  // overlapping GIDs are at the end of the target Map, but
985  // otherwise the source and target Maps are the same. This allows
986  // a fast contiguous copy for the initial "same IDs."
987  size_type numSameGids = 0;
988  for ( ; numSameGids < numGids && rawSrcGids[numSameGids] == rawTgtGids[numSameGids]; ++numSameGids)
989  {} // third clause of 'for' does everything
990  this->TransferData_->numSameIDs_ = numSameGids;
991 
992  // Compute permuteToLIDs_, permuteFromLIDs_, remoteGIDs, and
993  // remoteLIDs_. The first two arrays are IDs to be permuted, and
994  // the latter two arrays are IDs to be received ("imported"),
995  // called "remote" IDs.
996  //
997  // IDs to permute are in both the source and target Maps, which
998  // means we don't have to send or receive them, but we do have to
999  // rearrange (permute) them in general. IDs to receive are in the
1000  // target Map, but not the source Map.
1001 
1002  // Iterate over the target Map's LIDs, since we only need to do
1003  // GID -> LID lookups for the source Map.
1004  const LO LINVALID = Teuchos::OrdinalTraits<LO>::invalid ();
1005  const LO numTgtLids = as<LO> (numTgtGids);
1006  LO numPermutes = 0;
1007  for (LO tgtLid = numSameGids; tgtLid < numTgtLids; ++tgtLid) {
1008  const GO curTargetGid = rawTgtGids[tgtLid];
1009  // getLocalElement() returns LINVALID if the GID isn't in the
1010  // source Map. This saves us a lookup (which
1011  // isNodeGlobalElement() would do).
1012  const LO srcLid = source.getLocalElement (curTargetGid);
1013  if (srcLid != LINVALID) { // if source.isNodeGlobalElement (curTargetGid)
1014  ++numPermutes;
1015  }
1016  }
1017  const LO numRemotes = (numTgtLids - numSameGids) - numPermutes;
1018 
1019  using host_perm_type =
1020  typename decltype (this->TransferData_->permuteToLIDs_)::t_host;
1021  host_perm_type permuteToLIDs
1022  (view_alloc_no_init ("permuteToLIDs"), numPermutes);
1023  host_perm_type permuteFromLIDs
1024  (view_alloc_no_init ("permuteFromLIDs"), numPermutes);
1025  typename decltype (this->TransferData_->remoteLIDs_)::t_host remoteLIDs
1026  (view_alloc_no_init ("permuteFromLIDs"), numRemotes);
1027 
1028  {
1029  LO numPermutes2 = 0;
1030  LO numRemotes2 = 0;
1031  for (LO tgtLid = numSameGids; tgtLid < numTgtLids; ++tgtLid) {
1032  const GO curTargetGid = rawTgtGids[tgtLid];
1033  const LO srcLid = source.getLocalElement (curTargetGid);
1034  if (srcLid != LINVALID) {
1035  permuteToLIDs[numPermutes2] = tgtLid;
1036  permuteFromLIDs[numPermutes2] = srcLid;
1037  ++numPermutes2;
1038  }
1039  else {
1040  remoteGIDs.push_back (curTargetGid);
1041  remoteLIDs[numRemotes2] = tgtLid;
1042  ++numRemotes2;
1043  }
1044  }
1045  TEUCHOS_ASSERT( numPermutes == numPermutes2 );
1046  TEUCHOS_ASSERT( numRemotes == numRemotes2 );
1047  TEUCHOS_ASSERT( size_t (numPermutes) + remoteGIDs.size () == size_t (numTgtLids - numSameGids) );
1048  }
1049 
1050  makeDualViewFromOwningHostView (this->TransferData_->permuteToLIDs_, permuteToLIDs);
1051  makeDualViewFromOwningHostView (this->TransferData_->permuteFromLIDs_, permuteFromLIDs);
1052  makeDualViewFromOwningHostView (this->TransferData_->remoteLIDs_, remoteLIDs);
1053  if (remoteLIDs.extent (0) != 0 && ! source.isDistributed ()) {
1054  // This Import has remote LIDs, meaning that the target Map has
1055  // entries on this process that are not in the source Map on
1056  // this process. However, the source Map is not distributed
1057  // globally. This implies that this Import is not locally
1058  // complete on this process.
1059  this->TransferData_->isLocallyComplete_ = false;
1060  // mfh 12 Sep 2016: I disagree that this is "abuse"; it may be
1061  // correct behavior, depending on the circumstances.
1062  TPETRA_ABUSE_WARNING
1063  (true, std::runtime_error, "::setupSamePermuteRemote(): Target has "
1064  "remote LIDs but Source is not distributed globally. Importing to a "
1065  "submap of the target map.");
1066  }
1067  }
1068 
1069 
1070  template <class LocalOrdinal, class GlobalOrdinal, class Node>
1071  void Import<LocalOrdinal,GlobalOrdinal,Node>::
1072  setupExport (Teuchos::Array<GlobalOrdinal>& remoteGIDs,
1073  bool useRemotePIDs,
1074  Teuchos::Array<int>& userRemotePIDs,
1075  const Teuchos::RCP<Teuchos::ParameterList>& plist)
1076  {
1077  using ::Tpetra::Details::makeDualViewFromOwningHostView;
1078  using ::Tpetra::Details::view_alloc_no_init;
1079  using Teuchos::Array;
1080  using Teuchos::ArrayView;
1081  using std::endl;
1082  using GO = GlobalOrdinal;
1083  typedef typename Array<int>::difference_type size_type;
1084  const char tfecfFuncName[] = "setupExport: ";
1085  const char suffix[] = " Please report this bug to the Tpetra developers.";
1086 
1087  std::unique_ptr<std::string> prefix;
1088  if (this->verbose ()) {
1089  auto srcMap = this->getSourceMap ();
1090  auto comm = srcMap.is_null () ? Teuchos::null : srcMap->getComm ();
1091  const int myRank = comm.is_null () ? -1 : comm->getRank ();
1092  std::ostringstream os;
1093  os << "Proc " << myRank << ": Tpetra::Import::setupExport: ";
1094  prefix = std::unique_ptr<std::string> (new std::string (os.str ()));
1095  os << "Start" << std::endl;
1096  this->verboseOutputStream () << os.str ();
1097  }
1098 
1099  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1100  (this->getSourceMap ().is_null (), std::logic_error,
1101  "Source Map is null. " << suffix);
1102  const map_type& source = * (this->getSourceMap ());
1103 
1104  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1105  (! useRemotePIDs && (userRemotePIDs.size() > 0), std::invalid_argument,
1106  "remotePIDs are non-empty but their use has not been requested.");
1107  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1108  (userRemotePIDs.size () > 0 && remoteGIDs.size () != userRemotePIDs.size (),
1109  std::invalid_argument, "remotePIDs must either be of size zero or match "
1110  "the size of remoteGIDs.");
1111 
1112  // For each entry remoteGIDs[i], remoteProcIDs[i] will contain
1113  // the process ID of the process that owns that GID.
1114  ArrayView<GO> remoteGIDsView = remoteGIDs ();
1115  ArrayView<int> remoteProcIDsView;
1116 
1117  // lookup == IDNotPresent means that the source Map wasn't able to
1118  // figure out to which processes one or more of the GIDs in the
1119  // given list of remote GIDs belong.
1120  //
1121  // The previous abuse warning said "The target Map has GIDs not
1122  // found in the source Map." This statement could be confusing,
1123  // because it doesn't refer to ownership by the current process,
1124  // but rather to ownership by _any_ process participating in the
1125  // Map. (It could not possibly refer to ownership by the current
1126  // process, since remoteGIDs is exactly the list of GIDs owned by
1127  // the target Map but not owned by the source Map. It was
1128  // constructed that way by setupSamePermuteRemote().)
1129  //
1130  // What this statement means is that the source and target Maps
1131  // don't contain the same set of GIDs globally (over all
1132  // processes). That is, there is at least one GID owned by some
1133  // process in the target Map, which is not owned by _any_ process
1134  // in the source Map.
1135  Array<int> newRemotePIDs;
1136  LookupStatus lookup = AllIDsPresent;
1137 
1138  if (! useRemotePIDs) {
1139  newRemotePIDs.resize (remoteGIDsView.size ());
1140  if (this->verbose ()) {
1141  std::ostringstream os;
1142  os << *prefix << "Call sourceMap.getRemoteIndexList" << endl;
1143  this->verboseOutputStream () << os.str ();
1144  }
1145  lookup = source.getRemoteIndexList (remoteGIDsView, newRemotePIDs ());
1146  }
1147  Array<int>& remoteProcIDs = useRemotePIDs ? userRemotePIDs : newRemotePIDs;
1148 
1149  if (lookup == IDNotPresent) {
1150  // There is at least one GID owned by the calling process in the
1151  // target Map, which is not owned by any process in the source
1152  // Map.
1153  this->TransferData_->isLocallyComplete_ = false;
1154 
1155  // mfh 12 Sep 2016: I disagree that this is "abuse"; it may be
1156  // correct behavior, depending on the circumstances.
1157  TPETRA_ABUSE_WARNING
1158  (true, std::runtime_error, "::setupExport(): the source Map wasn't "
1159  "able to figure out which process owns one or more of the GIDs in the "
1160  "list of remote GIDs. This probably means that there is at least one "
1161  "GID owned by some process in the target Map which is not owned by any"
1162  " process in the source Map. (That is, the source and target Maps do "
1163  "not contain the same set of GIDs globally.)");
1164 
1165  // Ignore remote GIDs that aren't owned by any process in the
1166  // source Map. getRemoteIndexList() gives each of these a
1167  // process ID of -1.
1168 
1169  const size_type numInvalidRemote =
1170  std::count_if (remoteProcIDs.begin (), remoteProcIDs.end (),
1171  std::bind1st (std::equal_to<int> (), -1));
1172  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1173  (numInvalidRemote == 0, std::logic_error, "Calling getRemoteIndexList "
1174  "on the source Map returned IDNotPresent, but none of the returned "
1175  "\"remote\" process ranks are -1. Please report this bug to the "
1176  "Tpetra developers.");
1177 
1178 #ifdef HAVE_TPETRA_MMM_TIMINGS
1179  using Teuchos::TimeMonitor;
1180  std::string label;
1181  if(!plist.is_null())
1182  label = plist->get("Timer Label",label);
1183  std::string prefix = std::string("Tpetra ")+ label + std::string(":iport_ctor:setupExport:1 ");
1184  auto MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix)));
1185 #else
1186  (void)plist;
1187 #endif
1188 
1189  // If all of them are invalid, we can delete the whole array.
1190  const size_type totalNumRemote = this->getNumRemoteIDs ();
1191  if (numInvalidRemote == totalNumRemote) {
1192  // all remotes are invalid; we have no remotes; we can delete the remotes
1193  remoteProcIDs.clear ();
1194  remoteGIDs.clear (); // invalidates remoteGIDsView
1195  this->TransferData_->remoteLIDs_ =
1196  decltype (this->TransferData_->remoteLIDs_) ();
1197  }
1198  else {
1199  // Some remotes are valid; we need to keep the valid ones.
1200  // Pack and resize remoteProcIDs, remoteGIDs, and remoteLIDs_.
1201  size_type numValidRemote = 0;
1202 #ifdef HAVE_TPETRA_DEBUG
1203  ArrayView<GO> remoteGIDsPtr = remoteGIDsView;
1204 #else
1205  GO* const remoteGIDsPtr = remoteGIDsView.getRawPtr ();
1206 #endif // HAVE_TPETRA_DEBUG
1207 
1208  // Don't mark the DualView modified, since we'll reallocate it.
1209  auto remoteLIDs = this->TransferData_->remoteLIDs_.view_host ();
1210 
1211  for (size_type r = 0; r < totalNumRemote; ++r) {
1212  // Pack in all the valid remote PIDs and GIDs.
1213  if (remoteProcIDs[r] != -1) {
1214  remoteProcIDs[numValidRemote] = remoteProcIDs[r];
1215  remoteGIDsPtr[numValidRemote] = remoteGIDsPtr[r];
1216  remoteLIDs[numValidRemote] = remoteLIDs[r];
1217  ++numValidRemote;
1218  }
1219  }
1220  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
1221  (numValidRemote != totalNumRemote - numInvalidRemote,
1222  std::logic_error, "After removing invalid remote GIDs and packing "
1223  "the valid remote GIDs, numValidRemote = " << numValidRemote
1224  << " != totalNumRemote - numInvalidRemote = "
1225  << totalNumRemote - numInvalidRemote
1226  << ". Please report this bug to the Tpetra developers.");
1227 
1228  remoteProcIDs.resize (numValidRemote);
1229  remoteGIDs.resize (numValidRemote);
1230 
1231  Kokkos::resize (remoteLIDs, numValidRemote);
1232  this->TransferData_->remoteLIDs_ = decltype (this->TransferData_->remoteLIDs_) ();
1233  makeDualViewFromOwningHostView (this->TransferData_->remoteLIDs_, remoteLIDs);
1234  }
1235  // Revalidate the view after clear or resize.
1236  remoteGIDsView = remoteGIDs ();
1237  }
1238 
1239  // Sort remoteProcIDs in ascending order, and apply the resulting
1240  // permutation to remoteGIDs and remoteLIDs_. This ensures that
1241  // remoteProcIDs[i], remoteGIDs[i], and remoteLIDs_[i] all refer
1242  // to the same thing.
1243  {
1244  this->TransferData_->remoteLIDs_.modify_host ();
1245  auto remoteLIDs = this->TransferData_->remoteLIDs_.view_host ();
1246  sort3 (remoteProcIDs.begin (),
1247  remoteProcIDs.end (),
1248  remoteGIDsView.getRawPtr (),
1249  remoteLIDs.data ());
1250  this->TransferData_->remoteLIDs_.sync_device ();
1251  }
1252 
1253  // Call the Distributor's createFromRecvs() method to turn the
1254  // remote GIDs and their owning processes into a send-and-receive
1255  // communication plan. remoteGIDs and remoteProcIDs_ are input;
1256  // exportGIDs and exportProcIDs_ are output arrays which are
1257  // allocated by createFromRecvs().
1258  Array<GO> exportGIDs;
1259 
1260 #ifdef HAVE_TPETRA_MMM_TIMINGS
1261  using Teuchos::TimeMonitor;
1262  std::string label;
1263  if(!plist.is_null())
1264  label = plist->get("Timer Label",label);
1265  std::string prefix2 = std::string("Tpetra ")+ label + std::string(":iport_ctor:setupExport:3 ");
1266  auto MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix2)));
1267 #endif
1268 
1269  if (this->verbose ()) {
1270  std::ostringstream os;
1271  os << *prefix << "Call createFromRecvs" << endl;
1272  this->verboseOutputStream () << endl;
1273  }
1274  this->TransferData_->distributor_.createFromRecvs (remoteGIDsView ().getConst (),
1275  remoteProcIDs, exportGIDs,
1276  this->TransferData_->exportPIDs_);
1277 
1278  // Find the LIDs corresponding to the (outgoing) GIDs in
1279  // exportGIDs. For sparse matrix-vector multiply, this tells the
1280  // calling process how to index into the source vector to get the
1281  // elements which it needs to send.
1282 #ifdef HAVE_TPETRA_MMM_TIMINGS
1283  prefix2 = std::string("Tpetra ")+ label + std::string(":iport_ctor:setupExport:4 ");
1284  MM.release();
1285  MM = rcp(new TimeMonitor(*TimeMonitor::getNewTimer(prefix2)));
1286 #endif
1287 
1288  // NOTE (mfh 03 Mar 2014) This is now a candidate for a
1289  // thread-parallel kernel, but only if using the new thread-safe
1290  // Map implementation.
1291  const size_type numExportIDs = exportGIDs.size ();
1292  if (numExportIDs > 0) {
1293  typename decltype (this->TransferData_->exportLIDs_)::t_host
1294  exportLIDs (view_alloc_no_init ("exportLIDs"), numExportIDs);
1295  ArrayView<const GO> expGIDs = exportGIDs ();
1296  for (size_type k = 0; k < numExportIDs; ++k) {
1297  exportLIDs[k] = source.getLocalElement (expGIDs[k]);
1298  }
1299  makeDualViewFromOwningHostView (this->TransferData_->exportLIDs_, exportLIDs);
1300  }
1301 
1302  if (this->verbose ()) {
1303  std::ostringstream os;
1304  os << *prefix << "Done!" << endl;
1305  this->verboseOutputStream () << os.str ();
1306  }
1307  }
1308 
1309  template <class LocalOrdinal, class GlobalOrdinal, class Node>
1310  void
1311  Import<LocalOrdinal,GlobalOrdinal,Node>::
1312  findUnionTargetGIDs(Teuchos::Array<GlobalOrdinal>& unionTgtGIDs,
1313  Teuchos::Array<std::pair<int,GlobalOrdinal>>& remotePGIDs,
1314  typename Teuchos::Array<GlobalOrdinal>::size_type& numSameGIDs,
1315  typename Teuchos::Array<GlobalOrdinal>::size_type& numPermuteGIDs,
1316  typename Teuchos::Array<GlobalOrdinal>::size_type& numRemoteGIDs,
1317  const Teuchos::ArrayView<const GlobalOrdinal>& sameGIDs1,
1318  const Teuchos::ArrayView<const GlobalOrdinal>& sameGIDs2,
1319  Teuchos::Array<GlobalOrdinal>& permuteGIDs1,
1320  Teuchos::Array<GlobalOrdinal>& permuteGIDs2,
1321  Teuchos::Array<GlobalOrdinal>& remoteGIDs1,
1322  Teuchos::Array<GlobalOrdinal>& remoteGIDs2,
1323  Teuchos::Array<int>& remotePIDs1,
1324  Teuchos::Array<int>& remotePIDs2) const
1325  {
1326 
1327  typedef GlobalOrdinal GO;
1328  typedef typename Teuchos::Array<GO>::size_type size_type;
1329 
1330  const size_type numSameGIDs1 = sameGIDs1.size();
1331  const size_type numSameGIDs2 = sameGIDs2.size();
1332 
1333  // Sort the permute GIDs
1334  std::sort(permuteGIDs1.begin(), permuteGIDs1.end());
1335  std::sort(permuteGIDs2.begin(), permuteGIDs2.end());
1336 
1337  // Get the union of the two target maps
1338  // Reserve the maximum possible size to guard against reallocations from
1339  // push_back operations.
1340  unionTgtGIDs.reserve(numSameGIDs1 + numSameGIDs2 +
1341  permuteGIDs1.size() + permuteGIDs2.size() +
1342  remoteGIDs1.size() + remoteGIDs2.size());
1343 
1344  // Copy the same GIDs to unionTgtGIDs. Cases for numSameGIDs1 !=
1345  // numSameGIDs2 must be treated separately.
1346  typename Teuchos::Array<GO>::iterator permuteGIDs1_end;
1347  typename Teuchos::Array<GO>::iterator permuteGIDs2_end;
1348  if (numSameGIDs2 > numSameGIDs1) {
1349 
1350  numSameGIDs = numSameGIDs2;
1351  permuteGIDs2_end = permuteGIDs2.end();
1352 
1353  // Copy the same GIDs from tgtGIDs to the union
1354  std::copy(sameGIDs2.begin(), sameGIDs2.end(), std::back_inserter(unionTgtGIDs));
1355 
1356  // Remove GIDs from permuteGIDs1 that have already been copied in to unionTgtGIDs
1357  // set_difference allows the last (output) argument to alias the first.
1358  permuteGIDs1_end = std::set_difference(permuteGIDs1.begin(), permuteGIDs1.end(),
1359  unionTgtGIDs.begin()+numSameGIDs1, unionTgtGIDs.end(),
1360  permuteGIDs1.begin());
1361 
1362  } else {
1363 
1364  numSameGIDs = numSameGIDs1;
1365  permuteGIDs1_end = permuteGIDs1.end();
1366 
1367  // Copy the same GIDs from tgtGIDs to the union
1368  std::copy(sameGIDs1.begin(), sameGIDs1.end(), std::back_inserter(unionTgtGIDs));
1369 
1370  // Remove GIDs from permuteGIDs2 that have already been copied in to unionTgtGIDs
1371  // set_difference allows the last (output) argument to alias the first.
1372  permuteGIDs2_end = std::set_difference(permuteGIDs2.begin(), permuteGIDs2.end(),
1373  unionTgtGIDs.begin()+numSameGIDs2, unionTgtGIDs.end(),
1374  permuteGIDs2.begin());
1375 
1376  }
1377 
1378  // Get the union of the permute GIDs and push it back on unionTgtGIDs
1379  std::set_union(permuteGIDs1.begin(), permuteGIDs1_end,
1380  permuteGIDs2.begin(), permuteGIDs2_end,
1381  std::back_inserter(unionTgtGIDs));
1382 
1383  // Sort the PID,GID pairs and find the unique set
1384  Teuchos::Array<std::pair<int,GO>> remotePGIDs1(remoteGIDs1.size());
1385  for (size_type k=0; k<remoteGIDs1.size(); k++)
1386  remotePGIDs1[k] = std::make_pair(remotePIDs1[k], remoteGIDs1[k]);
1387  std::sort(remotePGIDs1.begin(), remotePGIDs1.end());
1388 
1389  Teuchos::Array<std::pair<int,GO>> remotePGIDs2(remoteGIDs2.size());
1390  for (size_type k=0; k<remoteGIDs2.size(); k++)
1391  remotePGIDs2[k] = std::make_pair(remotePIDs2[k], remoteGIDs2[k]);
1392  std::sort(remotePGIDs2.begin(), remotePGIDs2.end());
1393 
1394  remotePGIDs.reserve(remotePGIDs1.size()+remotePGIDs2.size());
1395  std::merge(remotePGIDs1.begin(), remotePGIDs1.end(),
1396  remotePGIDs2.begin(), remotePGIDs2.end(),
1397  std::back_inserter(remotePGIDs));
1398  auto it = std::unique(remotePGIDs.begin(), remotePGIDs.end());
1399  remotePGIDs.resize(std::distance(remotePGIDs.begin(), it));
1400 
1401  // Finally, insert remote GIDs
1402  const size_type oldSize = unionTgtGIDs.size();
1403  unionTgtGIDs.resize(oldSize+remotePGIDs.size());
1404  for (size_type start=oldSize, k=0; k<remotePGIDs.size(); k++)
1405  unionTgtGIDs[start+k] = remotePGIDs[k].second;
1406 
1407  // Compute output only quantities
1408  numRemoteGIDs = remotePGIDs.size();
1409  numPermuteGIDs = unionTgtGIDs.size() - numSameGIDs - numRemoteGIDs;
1410 
1411  return;
1412  }
1413 
1414  template <class LocalOrdinal, class GlobalOrdinal, class Node>
1415  Teuchos::RCP<const Import<LocalOrdinal, GlobalOrdinal, Node> >
1416  Import<LocalOrdinal,GlobalOrdinal,Node>::
1417  setUnion (const Import<LocalOrdinal, GlobalOrdinal, Node>& rhs) const
1418  {
1419  using ::Tpetra::Details::Behavior;
1420  using Teuchos::Array;
1421  using Teuchos::ArrayView;
1422  using Teuchos::as;
1423  using Teuchos::Comm;
1424  using Teuchos::RCP;
1425  using Teuchos::rcp;
1426  using Teuchos::outArg;
1427  using Teuchos::REDUCE_MIN;
1428  using Teuchos::reduceAll;
1429  using GST = Tpetra::global_size_t;
1430  using LO = LocalOrdinal;
1431  using GO = GlobalOrdinal;
1432  using import_type = Import<LO, GO, Node>;
1433  using size_type = typename Array<GO>::size_type;
1434 
1435 #ifdef HAVE_TPETRA_MMM_TIMINGS
1436  using Teuchos::TimeMonitor;
1437  std::string label = std::string("Tpetra::Import::setUnion");
1438  TimeMonitor MM(*TimeMonitor::getNewTimer(label));
1439 #endif
1440 
1441  RCP<const map_type> srcMap = this->getSourceMap ();
1442  RCP<const map_type> tgtMap1 = this->getTargetMap ();
1443  RCP<const map_type> tgtMap2 = rhs.getTargetMap ();
1444  RCP<const Comm<int> > comm = srcMap->getComm ();
1445 
1446  const bool debug = Behavior::debug ("Import::setUnion") ||
1447  Behavior::debug ("Tpetra::Import::setUnion");
1448 
1449  if (debug) {
1450  TEUCHOS_TEST_FOR_EXCEPTION
1451  (! srcMap->isSameAs (* (rhs.getSourceMap ())), std::invalid_argument,
1452  "Tpetra::Import::setUnion: The source Map of the input Import must be the "
1453  "same as (in the sense of Map::isSameAs) the source Map of this Import.");
1454  const Comm<int>& comm1 = * (tgtMap1->getComm ());
1455  const Comm<int>& comm2 = * (tgtMap2->getComm ());
1456  TEUCHOS_TEST_FOR_EXCEPTION
1457  (! ::Tpetra::Details::congruent (comm1, comm2),
1458  std::invalid_argument, "Tpetra::Import::setUnion: "
1459  "The target Maps must have congruent communicators.");
1460  }
1461 
1462  // It's probably worth the one all-reduce to check whether the two
1463  // Maps are the same. If so, we can just return a copy of *this.
1464  // isSameAs() bypasses the all-reduce if the pointers are equal.
1465  if (tgtMap1->isSameAs (*tgtMap2)) {
1466  return rcp (new import_type (*this));
1467  }
1468 
1469  // Alas, the two target Maps are not the same. That means we have
1470  // to compute their union, and the union Import object.
1471 
1472  // Get the same GIDs (same GIDs are a subview of the first numSame target
1473  // GIDs)
1474  const size_type numSameGIDs1 = this->getNumSameIDs();
1475  ArrayView<const GO> sameGIDs1 = (tgtMap1->getNodeElementList())(0,numSameGIDs1);
1476 
1477  const size_type numSameGIDs2 = rhs.getNumSameIDs();
1478  ArrayView<const GO> sameGIDs2 = (tgtMap2->getNodeElementList())(0,numSameGIDs2);
1479 
1480  // Get permute GIDs
1481  ArrayView<const LO> permuteToLIDs1 = this->getPermuteToLIDs();
1482  Array<GO> permuteGIDs1(permuteToLIDs1.size());
1483  for (size_type k=0; k<permuteGIDs1.size(); k++)
1484  permuteGIDs1[k] = tgtMap1->getGlobalElement(permuteToLIDs1[k]);
1485 
1486  ArrayView<const LO> permuteToLIDs2 = rhs.getPermuteToLIDs();
1487  Array<GO> permuteGIDs2(permuteToLIDs2.size());
1488  for (size_type k=0; k<permuteGIDs2.size(); k++)
1489  permuteGIDs2[k] = tgtMap2->getGlobalElement(permuteToLIDs2[k]);
1490 
1491  // Get remote GIDs
1492  ArrayView<const LO> remoteLIDs1 = this->getRemoteLIDs();
1493  Array<GO> remoteGIDs1(remoteLIDs1.size());
1494  for (size_type k=0; k<remoteLIDs1.size(); k++)
1495  remoteGIDs1[k] = this->getTargetMap()->getGlobalElement(remoteLIDs1[k]);
1496 
1497  ArrayView<const LO> remoteLIDs2 = rhs.getRemoteLIDs();
1498  Array<GO> remoteGIDs2(remoteLIDs2.size());
1499  for (size_type k=0; k<remoteLIDs2.size(); k++)
1500  remoteGIDs2[k] = rhs.getTargetMap()->getGlobalElement(remoteLIDs2[k]);
1501 
1502  // Get remote PIDs
1503  Array<int> remotePIDs1;
1504  Tpetra::Import_Util::getRemotePIDs(*this, remotePIDs1);
1505 
1506  Array<int> remotePIDs2;
1507  Tpetra::Import_Util::getRemotePIDs(rhs, remotePIDs2);
1508 
1509  // Get the union of the target GIDs
1510  Array<GO> unionTgtGIDs;
1511  Array<std::pair<int,GO>> remotePGIDs;
1512  size_type numSameIDsUnion, numPermuteIDsUnion, numRemoteIDsUnion;
1513 
1514  findUnionTargetGIDs(unionTgtGIDs, remotePGIDs,
1515  numSameIDsUnion, numPermuteIDsUnion, numRemoteIDsUnion,
1516  sameGIDs1, sameGIDs2, permuteGIDs1, permuteGIDs2,
1517  remoteGIDs1, remoteGIDs2, remotePIDs1, remotePIDs2);
1518 
1519  // Extract GIDs and compute LIDS, PIDs for the remotes in the union
1520  Array<LO> remoteLIDsUnion(numRemoteIDsUnion);
1521  Array<GO> remoteGIDsUnion(numRemoteIDsUnion);
1522  Array<int> remotePIDsUnion(numRemoteIDsUnion);
1523  const size_type unionRemoteIDsStart = numSameIDsUnion + numPermuteIDsUnion;
1524  for (size_type k = 0; k < numRemoteIDsUnion; ++k) {
1525  remoteLIDsUnion[k] = unionRemoteIDsStart + k;
1526  remotePIDsUnion[k] = remotePGIDs[k].first;
1527  remoteGIDsUnion[k] = remotePGIDs[k].second;
1528  }
1529 
1530  // Compute the permute-to LIDs (in the union target Map).
1531  // Convert the permute GIDs to permute-from LIDs in the source Map.
1532  Array<LO> permuteToLIDsUnion(numPermuteIDsUnion);
1533  Array<LO> permuteFromLIDsUnion(numPermuteIDsUnion);
1534  for (size_type k = 0; k < numPermuteIDsUnion; ++k) {
1535  size_type idx = numSameIDsUnion + k;
1536  permuteToLIDsUnion[k] = static_cast<LO>(idx);
1537  permuteFromLIDsUnion[k] = srcMap->getLocalElement(unionTgtGIDs[idx]);
1538  }
1539 
1540 #ifdef HAVE_TPETRA_MMM_TIMINGS
1541  MM.disableTimer(label);
1542  label = "Tpetra::Import::setUnion : Construct Target Map";
1543  TimeMonitor MM2(*TimeMonitor::getNewTimer(label));
1544 #endif
1545 
1546  // Create the union target Map.
1547  const GST INVALID = Teuchos::OrdinalTraits<GST>::invalid ();
1548  const GO indexBaseUnion = std::min(tgtMap1->getIndexBase(), tgtMap2->getIndexBase());
1549  RCP<const map_type> unionTgtMap =
1550  rcp(new map_type(INVALID, unionTgtGIDs(), indexBaseUnion, comm));
1551 
1552 #ifdef HAVE_TPETRA_MMM_TIMINGS
1553  MM2.disableTimer(label);
1554  label = "Tpetra::Import::setUnion : Export GIDs";
1555  TimeMonitor MM3(*TimeMonitor::getNewTimer(label));
1556 #endif
1557 
1558  // Thus far, we have computed the following in the union Import:
1559  // - numSameIDs
1560  // - numPermuteIDs and permuteFromLIDs
1561  // - numRemoteIDs, remoteGIDs, remoteLIDs, and remotePIDs
1562  //
1563  // Now it's time to compute the export IDs and initialize the
1564  // Distributor.
1565 
1566  Array<GO> exportGIDsUnion;
1567  Array<LO> exportLIDsUnion;
1568  Array<int> exportPIDsUnion;
1569  Distributor distributor (comm, this->TransferData_->out_);
1570 
1571 #ifdef TPETRA_IMPORT_SETUNION_USE_CREATE_FROM_SENDS
1572  // Compute the export IDs without communication, by merging the
1573  // lists of (export LID, export PID) pairs from the two input
1574  // Import objects. The export LIDs in both input Import objects
1575  // are LIDs in the source Map. Then, use the export PIDs to
1576  // initialize the Distributor via createFromSends.
1577 
1578  // const size_type numExportIDs1 = this->getNumExportIDs ();
1579  ArrayView<const LO> exportLIDs1 = this->getExportLIDs ();
1580  ArrayView<const LO> exportPIDs1 = this->getExportPIDs ();
1581 
1582  // const size_type numExportIDs2 = rhs.getNumExportIDs ();
1583  ArrayView<const LO> exportLIDs2 = rhs.getExportLIDs ();
1584  ArrayView<const LO> exportPIDs2 = rhs.getExportPIDs ();
1585 
1586  // We have to keep the export LIDs in PID-sorted order, then merge
1587  // them. So, first key-value merge (LID,PID) pairs, treating PIDs
1588  // as values, merging values by replacement. Then, sort the
1589  // (LID,PID) pairs again by PID.
1590 
1591  // Sort (LID,PID) pairs by LID for the later merge, and make
1592  // each sequence unique by LID.
1593  Array<LO> exportLIDs1Copy (exportLIDs1.begin (), exportLIDs1.end ());
1594  Array<int> exportPIDs1Copy (exportLIDs1.begin (), exportLIDs1.end ());
1595  sort2 (exportLIDs1Copy.begin (), exportLIDs1Copy.end (),
1596  exportPIDs1Copy.begin ());
1597  typename ArrayView<LO>::iterator exportLIDs1_end = exportLIDs1Copy.end ();
1598  typename ArrayView<LO>::iterator exportPIDs1_end = exportPIDs1Copy.end ();
1599  merge2 (exportLIDs1_end, exportPIDs1_end,
1600  exportLIDs1Copy.begin (), exportLIDs1_end,
1601  exportPIDs1Copy.begin (), exportPIDs1_end,
1602  project1st<LO, LO> ());
1603 
1604  Array<LO> exportLIDs2Copy (exportLIDs2.begin (), exportLIDs2.end ());
1605  Array<int> exportPIDs2Copy (exportLIDs2.begin (), exportLIDs2.end ());
1606  sort2 (exportLIDs2Copy.begin (), exportLIDs2Copy.end (),
1607  exportPIDs2Copy.begin ());
1608  typename ArrayView<LO>::iterator exportLIDs2_end = exportLIDs2Copy.end ();
1609  typename ArrayView<LO>::iterator exportPIDs2_end = exportPIDs2Copy.end ();
1610  merge2 (exportLIDs2_end, exportPIDs2_end,
1611  exportLIDs2Copy.begin (), exportLIDs2_end,
1612  exportPIDs2Copy.begin (), exportPIDs2_end,
1613  project1st<LO, LO> ());
1614 
1615  // Merge export (LID,PID) pairs. In this merge operation, the
1616  // LIDs are the "keys" and the PIDs their "values." We combine
1617  // the "values" (PIDs) in the pairs by replacement, rather than
1618  // by adding them together.
1619  keyValueMerge (exportLIDs1Copy.begin (), exportLIDs1Copy.end (),
1620  exportPIDs1Copy.begin (), exportPIDs1Copy.end (),
1621  exportLIDs2Copy.begin (), exportLIDs2Copy.end (),
1622  exportPIDs2Copy.begin (), exportPIDs2Copy.end (),
1623  std::back_inserter (exportLIDsUnion),
1624  std::back_inserter (exportPIDsUnion),
1625  project1st<int, int> ());
1626 
1627  // Resort the merged (LID,PID) pairs by PID.
1628  sort2 (exportPIDsUnion.begin (), exportPIDsUnion.end (),
1629  exportLIDsUnion.begin ());
1630 
1631  // Initialize the Distributor. Using createFromSends instead of
1632  // createFromRecvs avoids the initialization and use of a
1633  // temporary Distributor object.
1634  (void) distributor.createFromSends (exportPIDsUnion ().getConst ());
1635 #else // NOT TPETRA_IMPORT_SETUNION_USE_CREATE_FROM_SENDS
1636 
1637  // Call the Distributor's createFromRecvs() method to turn the
1638  // remote GIDs and their owning processes into a send-and-receive
1639  // communication plan. remoteGIDsUnion and remotePIDsUnion are
1640  // input; exportGIDsUnion and exportPIDsUnion are output arrays
1641  // which are allocated by createFromRecvs().
1642  distributor.createFromRecvs (remoteGIDsUnion().getConst(),
1643  remotePIDsUnion().getConst(),
1644  exportGIDsUnion, exportPIDsUnion);
1645 
1646  // Find the (source Map) LIDs corresponding to the export GIDs.
1647  const size_type numExportIDsUnion = exportGIDsUnion.size ();
1648  exportLIDsUnion.resize (numExportIDsUnion);
1649  for (size_type k = 0; k < numExportIDsUnion; ++k) {
1650  exportLIDsUnion[k] = srcMap->getLocalElement (exportGIDsUnion[k]);
1651  }
1652 #endif // TPETRA_IMPORT_SETUNION_USE_CREATE_FROM_SENDS
1653 
1654  // Create and return the union Import. This uses the "expert" constructor
1655 #ifdef HAVE_TPETRA_MMM_TIMINGS
1656  MM3.disableTimer(label);
1657  label = "Tpetra::Import::setUnion : Construct Import";
1658  TimeMonitor MM4(*TimeMonitor::getNewTimer(label));
1659 #endif
1660  RCP<const import_type> unionImport =
1661  rcp (new import_type (srcMap, unionTgtMap,
1662  as<size_t> (numSameIDsUnion),
1663  permuteToLIDsUnion, permuteFromLIDsUnion,
1664  remoteLIDsUnion, exportLIDsUnion,
1665  exportPIDsUnion, distributor,
1666  this->TransferData_->out_));
1667  return unionImport;
1668  }
1669 
1670  template <class LocalOrdinal, class GlobalOrdinal, class Node>
1671  Teuchos::RCP<const Import<LocalOrdinal, GlobalOrdinal, Node> >
1672  Import<LocalOrdinal,GlobalOrdinal,Node>::
1673  setUnion () const
1674  {
1675  using Teuchos::Array;
1676  using Teuchos::ArrayView;
1677  using Teuchos::as;
1678  using Teuchos::Comm;
1679  using Teuchos::RCP;
1680  using Teuchos::rcp;
1681  using Teuchos::outArg;
1682  using Teuchos::REDUCE_MIN;
1683  using Teuchos::reduceAll;
1684  typedef LocalOrdinal LO;
1685  typedef GlobalOrdinal GO;
1686  Teuchos::RCP<const Import<LocalOrdinal, GlobalOrdinal, Node> > unionImport;
1687  RCP<const map_type> srcMap = this->getSourceMap ();
1688  RCP<const map_type> tgtMap = this->getTargetMap ();
1689  RCP<const Comm<int> > comm = srcMap->getComm ();
1690 
1691  ArrayView<const GO> srcGIDs = srcMap->getNodeElementList ();
1692  ArrayView<const GO> tgtGIDs = tgtMap->getNodeElementList ();
1693 
1694  // All elements in srcMap will be in the "new" target map, so...
1695  size_t numSameIDsNew = srcMap->getNodeNumElements ();
1696  size_t numRemoteIDsNew = this->getNumRemoteIDs ();
1697  Array<LO> permuteToLIDsNew, permuteFromLIDsNew; // empty on purpose
1698 
1699  // Grab some old data
1700  ArrayView<const LO> remoteLIDsOld = this->getRemoteLIDs ();
1701  ArrayView<const LO> exportLIDsOld = this->getExportLIDs ();
1702 
1703  // Build up the new map (same part)
1704  Array<GO> GIDs(numSameIDsNew + numRemoteIDsNew);
1705  for(size_t i=0; i<numSameIDsNew; i++)
1706  GIDs[i] = srcGIDs[i];
1707 
1708  // Build up the new map (remote part) and remotes list
1709  Array<LO> remoteLIDsNew(numRemoteIDsNew);
1710  for(size_t i=0; i<numRemoteIDsNew; i++) {
1711  GIDs[numSameIDsNew + i] = tgtGIDs[remoteLIDsOld[i]];
1712  remoteLIDsNew[i] = numSameIDsNew+i;
1713  }
1714 
1715  // Build the new target map
1716  GO GO_INVALID = Teuchos::OrdinalTraits<GO>::invalid();
1717  RCP<const map_type> targetMapNew =
1718  rcp (new map_type (GO_INVALID, GIDs, tgtMap->getIndexBase (),
1719  tgtMap->getComm ()));
1720 
1721  // Exports are trivial (since the sourcemap doesn't change)
1722  Array<int> exportPIDsnew (this->getExportPIDs ());
1723  Array<LO> exportLIDsnew (this->getExportLIDs ());
1724 
1725  // Copy the Distributor (due to how the Import constructor works)
1726  Distributor D (this->getDistributor ());
1727 
1728  // Build the importer using the "expert" constructor
1729  unionImport = rcp(new Import<LocalOrdinal, GlobalOrdinal, Node>(srcMap,
1730  targetMapNew,
1731  numSameIDsNew,
1732  permuteToLIDsNew,
1733  permuteFromLIDsNew,
1734  remoteLIDsNew,
1735  exportLIDsnew,
1736  exportPIDsnew,D));
1737 
1738  return unionImport;
1739  }
1740 
1741  template <class LocalOrdinal, class GlobalOrdinal, class Node>
1742  Teuchos::RCP<const Import<LocalOrdinal, GlobalOrdinal, Node> >
1743  Import<LocalOrdinal,GlobalOrdinal,Node>::
1744  createRemoteOnlyImport (const Teuchos::RCP<const map_type>& remoteTarget) const
1745  {
1746  using ::Tpetra::Details::Behavior;
1747  using ::Tpetra::Details::gathervPrint;
1748  using Teuchos::outArg;
1749  using Teuchos::rcp;
1750  using Teuchos::REDUCE_MIN;
1751  using Teuchos::reduceAll;
1752  using std::endl;
1753  using LO = LocalOrdinal;
1754  using GO = GlobalOrdinal;
1755  using import_type = Import<LocalOrdinal,GlobalOrdinal,Node>;
1756 
1757  const char funcPrefix[] = "Tpetra::createRemoteOnlyImport: ";
1758  int lclSuccess = 1;
1759  int gblSuccess = 1;
1760  const bool debug = Behavior::debug ();
1761 
1762  const size_t NumRemotes = this->getNumRemoteIDs ();
1763  std::unique_ptr<std::string> procPrefix;
1764  Teuchos::RCP<const Teuchos::Comm<int>> comm;
1765  if (debug) {
1766  comm = remoteTarget.is_null () ? Teuchos::null :
1767  remoteTarget->getComm ();
1768  std::ostringstream os;
1769  os << "Proc ";
1770  if (comm.is_null ()) {
1771  os << "?";
1772  }
1773  else {
1774  os << comm->getRank ();
1775  }
1776  os << ": ";
1777  procPrefix = std::unique_ptr<std::string> (new std::string (os.str ()));
1778  }
1779 
1780  if (debug) {
1781  std::ostringstream lclErr;
1782  if (remoteTarget.is_null ()) {
1783  lclSuccess = -1;
1784  }
1785  else if (NumRemotes != remoteTarget->getNodeNumElements ()) {
1786  lclSuccess = 0;
1787  lclErr << *procPrefix << "getNumRemoteIDs() = " << NumRemotes
1788  << " != remoteTarget->getNodeNumElements() = "
1789  << remoteTarget->getNodeNumElements () << "." << endl;
1790  }
1791 
1792  if (comm.is_null ()) {
1793  lclSuccess = gblSuccess;
1794  }
1795  else {
1796  reduceAll (*comm, REDUCE_MIN, lclSuccess, outArg (gblSuccess));
1797  }
1798  TEUCHOS_TEST_FOR_EXCEPTION
1799  (gblSuccess == -1, std::invalid_argument, funcPrefix
1800  << "Input target Map is null on at least one process.");
1801 
1802  if (gblSuccess != 1) {
1803  if (comm.is_null ()) {
1804  TEUCHOS_TEST_FOR_EXCEPTION
1805  (true, std::runtime_error, lclErr.str ());
1806  }
1807  else {
1808  std::ostringstream gblErr;
1809  gblErr << funcPrefix << endl;
1810  gathervPrint (gblErr, lclErr.str (), *comm);
1811  TEUCHOS_TEST_FOR_EXCEPTION
1812  (true, std::runtime_error, gblErr.str ());
1813  }
1814  }
1815  }
1816 
1817  // Compute the new Remote LIDs
1818  Teuchos::ArrayView<const LO> oldRemoteLIDs = this->getRemoteLIDs ();
1819  Teuchos::Array<LO> newRemoteLIDs (NumRemotes);
1820  const map_type& tgtMap = * (this->getTargetMap ());
1821  size_t badCount = 0;
1822 
1823  std::unique_ptr<std::vector<size_t>> badIndices;
1824  if (debug) {
1825  badIndices = std::unique_ptr<std::vector<size_t>> (new std::vector<size_t>);
1826  }
1827  for (size_t i = 0; i < NumRemotes; ++i) {
1828  const LO oldLclInd = oldRemoteLIDs[i];
1829  if (oldLclInd == Teuchos::OrdinalTraits<LO>::invalid ()) {
1830  ++badCount;
1831  if (debug) { badIndices->push_back (i); }
1832  continue;
1833  }
1834  const GO gblInd = tgtMap.getGlobalElement (oldLclInd);
1835  if (gblInd == Teuchos::OrdinalTraits<GO>::invalid ()) {
1836  ++badCount;
1837  if (debug) { badIndices->push_back (i); }
1838  continue;
1839  }
1840  const LO newLclInd = remoteTarget->getLocalElement (gblInd);
1841  if (newLclInd == Teuchos::OrdinalTraits<LO>::invalid ()) {
1842  ++badCount;
1843  if (debug) { badIndices->push_back (i); }
1844  continue;
1845  }
1846  newRemoteLIDs[i] = newLclInd;
1847  // Now we make sure these guys are in sorted order (AztecOO-ML ordering)
1848  if (i > 0 && newRemoteLIDs[i] < newRemoteLIDs[i-1]) {
1849  ++badCount;
1850  if (debug) { badIndices->push_back (i); }
1851  }
1852  }
1853 
1854  if (badCount != 0) {
1855  lclSuccess = 0;
1856  }
1857 
1858  if (debug) {
1859  if (comm.is_null ()) {
1860  lclSuccess = gblSuccess;
1861  }
1862  else {
1863  reduceAll (*comm, REDUCE_MIN, lclSuccess, outArg (gblSuccess));
1864  }
1865  std::ostringstream lclErr;
1866  if (lclSuccess != 1) {
1867  lclErr << *procPrefix << "Count of bad indices: " << badCount
1868  << ", bad indices: [";
1869  // TODO (mfh 04 Sep 2019) Limit the maximum number of bad
1870  // indices to print.
1871  for (size_t k = 0; k < badCount; ++k) {
1872  const size_t badIndex = (*badIndices)[k];
1873  lclErr << "(" << badIndex << ","
1874  << oldRemoteLIDs[badIndex] << ")";
1875  if (k + size_t (1) < badCount) {
1876  lclErr << ", ";
1877  }
1878  }
1879  lclErr << "]" << endl;
1880  }
1881 
1882  if (gblSuccess != 1) {
1883  std::ostringstream gblErr;
1884  gblErr << funcPrefix << "this->getRemoteLIDs() has \"bad\" "
1885  "indices on one or more processes. \"Bad\" means that the "
1886  "indices are invalid, they don't exist in the target Map, "
1887  "they don't exist in remoteTarget, or they are not in "
1888  "sorted order. In what follows, I will show the \"bad\" "
1889  "indices as (k, LID) pairs, where k is the zero-based "
1890  "index of the LID in this->getRemoteLIDs()." << endl;
1891  if (comm.is_null ()) {
1892  gblErr << lclErr.str ();
1893  }
1894  else {
1895  gathervPrint (gblErr, lclErr.str (), *comm);
1896  }
1897  TEUCHOS_TEST_FOR_EXCEPTION
1898  (true, std::runtime_error, gblErr.str ());
1899  }
1900  }
1901  else { // not debug
1902  TEUCHOS_TEST_FOR_EXCEPTION
1903  (lclSuccess == 0, std::runtime_error, funcPrefix
1904  << "this->getRemoteLIDs() has " << badCount
1905  << "ind" << (badCount == 1 ? "ex" : "ices")
1906  << " \"bad\" indices on this process." << endl);
1907  }
1908 
1909  // Copy ExportPIDs and such
1910  // NOTE: Be careful: The "Expert" Import constructor we use does a "swap"
1911  // for most of the LID/PID lists and the Distributor, meaning it
1912  // ruins the existing object if we pass things in directly. Hence
1913  // we copy them first.
1914  Teuchos::Array<int> newExportPIDs (this->getExportPIDs ());
1915  Teuchos::Array<LO> newExportLIDs (this->getExportLIDs ());
1916  Teuchos::Array<LO> dummy;
1917  Distributor newDistor (this->getDistributor ());
1918 
1919  return rcp (new import_type (this->getSourceMap (), remoteTarget,
1920  static_cast<size_t> (0), dummy, dummy,
1921  newRemoteLIDs, newExportLIDs,
1922  newExportPIDs, newDistor));
1923  }
1924 
1925 } // namespace Tpetra
1926 
1927 #define TPETRA_IMPORT_CLASS_INSTANT(LO, GO, NODE) \
1928  template class Import< LO , GO , NODE >;
1929 
1930 // Explicit instantiation macro.
1931 // Only invoke this when in the Tpetra namespace.
1932 // Most users do not need to use this.
1933 //
1934 // LO: The local ordinal type.
1935 // GO: The global ordinal type.
1936 // NODE: The Kokkos Node type.
1937 #define TPETRA_IMPORT_INSTANT(LO, GO, NODE) \
1938  TPETRA_IMPORT_CLASS_INSTANT(LO, GO, NODE)
1939 
1940 #endif // TPETRA_IMPORT_DEF_HPP
Tpetra_Import_Util.hpp
Internal functions and macros designed for use with Tpetra::Import and Tpetra::Export objects.
TPETRA_ABUSE_WARNING
#define TPETRA_ABUSE_WARNING(throw_exception_test, Exception, msg)
Handle an abuse warning, according to HAVE_TPETRA_THROW_ABUSE_WARNINGS and HAVE_TPETRA_PRINT_ABUSE_WA...
Definition: Tpetra_Util.hpp:166
Tpetra::IDNotPresent
@ IDNotPresent
Definition: Tpetra_ConfigDefs.hpp:126
Tpetra::Details::congruent
bool congruent(const Teuchos::Comm< int > &comm1, const Teuchos::Comm< int > &comm2)
Whether the two communicators are congruent.
Definition: Tpetra_Util.cpp:65
Tpetra::Import::Import
Import(const Teuchos::RCP< const map_type > &source, const Teuchos::RCP< const map_type > &target)
Construct an Import from the source and target Maps.
Definition: Tpetra_Import_def.hpp:186
Tpetra::Import::setUnion
Teuchos::RCP< const Import< LocalOrdinal, GlobalOrdinal, Node > > setUnion() const
Return the union of this Import this->getSourceMap()
Definition: Tpetra_Import_def.hpp:1673
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >
Tpetra::Distributor::createFromSends
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
Definition: Tpetra_Distributor.cpp:1041
Tpetra::Export
Communication plan for data redistribution from a (possibly) multiply-owned to a uniquely-owned distr...
Definition: Tpetra_Export_decl.hpp:124
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getPermuteToLIDs
Teuchos::ArrayView< const LocalOrdinal > getPermuteToLIDs() const
List of local IDs in the target Map that are permuted.
Definition: Tpetra_Details_Transfer_def.hpp:174
Tpetra::Import::createRemoteOnlyImport
Teuchos::RCP< const Import< LocalOrdinal, GlobalOrdinal, Node > > createRemoteOnlyImport(const Teuchos::RCP< const map_type > &remoteTarget) const
Returns an importer that contains only the remote entries of this.
Definition: Tpetra_Import_def.hpp:1744
Tpetra::ImportExportData
Implementation detail of Import and Export.
Definition: Tpetra_ImportExportData_decl.hpp:83
Tpetra::Map::getGlobalElement
GlobalOrdinal getGlobalElement(LocalOrdinal localIndex) const
The global index corresponding to the given local index.
Definition: Tpetra_Map_def.hpp:1246
Tpetra::sort2
void sort2(const IT1 &first1, const IT1 &last1, const IT2 &first2)
Sort the first array, and apply the resulting permutation to the second array.
Definition: Tpetra_Util.hpp:532
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getTargetMap
Teuchos::RCP< const map_type > getTargetMap() const
The target Map used to construct this Export or Import.
Definition: Tpetra_Details_Transfer_def.hpp:255
Tpetra_Details_Behavior.hpp
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getExportPIDs
Teuchos::ArrayView< const int > getExportPIDs() const
List of processes to which entries will be sent.
Definition: Tpetra_Details_Transfer_def.hpp:241
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getNumSameIDs
size_t getNumSameIDs() const
Number of initial identical IDs.
Definition: Tpetra_Details_Transfer_def.hpp:121
Tpetra::Map
A parallel distribution of indices over processes.
Definition: Tpetra_Map_decl.hpp:246
Tpetra::Import
Communication plan for data redistribution from a uniquely-owned to a (possibly) multiply-owned distr...
Definition: Tpetra_Import_decl.hpp:116
Tpetra::Distributor::createFromSendsAndRecvs
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
Definition: Tpetra_Distributor.cpp:1377
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getExportLIDs
Teuchos::ArrayView< const LocalOrdinal > getExportLIDs() const
List of entries in the source Map that will be sent to other processes.
Definition: Tpetra_Details_Transfer_def.hpp:234
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getNumRemoteIDs
size_t getNumRemoteIDs() const
Number of entries not on the calling process.
Definition: Tpetra_Details_Transfer_def.hpp:181
Tpetra::merge2
KOKKOS_INLINE_FUNCTION void merge2(IT1 &indResultOut, IT2 &valResultOut, IT1 indBeg, IT1 indEnd, IT2 valBeg, IT2)
Merge values in place, additively, with the same index.
Definition: Tpetra_Util.hpp:633
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::TransferData_
Teuchos::RCP< ImportExportData< LocalOrdinal, GlobalOrdinal, Node > > TransferData_
All the data needed for executing the Export communication plan.
Definition: Tpetra_Details_Transfer_decl.hpp:231
Tpetra::Import::describe
virtual void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
Definition: Tpetra_Import_def.hpp:930
Tpetra::AllIDsPresent
@ AllIDsPresent
Definition: Tpetra_ConfigDefs.hpp:125
Tpetra::Import::print
virtual void print(std::ostream &os) const
Print the Import's data to the given output stream.
Definition: Tpetra_Import_def.hpp:939
Tpetra_Details_Profiling.hpp
Declaration of Tpetra::Details::Profiling, a scope guard for Kokkos Profiling.
Tpetra::Distributor
Sets up and executes a communication plan for a Tpetra DistObject.
Definition: Tpetra_Distributor.hpp:192
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getRemoteLIDs
Teuchos::ArrayView< const LocalOrdinal > getRemoteLIDs() const
List of entries in the target Map to receive from other processes.
Definition: Tpetra_Details_Transfer_def.hpp:204
Tpetra::LookupStatus
LookupStatus
Return status of Map remote index lookup (getRemoteIndexList()).
Definition: Tpetra_ConfigDefs.hpp:124
Tpetra_Util.hpp
Stand-alone utility functions and macros.
Tpetra::Import::findUnionTargetGIDs
void findUnionTargetGIDs(Teuchos::Array< GlobalOrdinal > &unionTgtGIDs, Teuchos::Array< std::pair< int, GlobalOrdinal >> &remotePGIDs, typename Teuchos::Array< GlobalOrdinal >::size_type &numSameGIDs, typename Teuchos::Array< GlobalOrdinal >::size_type &numPermuteGIDs, typename Teuchos::Array< GlobalOrdinal >::size_type &numRemoteGIDs, const Teuchos::ArrayView< const GlobalOrdinal > &sameGIDs1, const Teuchos::ArrayView< const GlobalOrdinal > &sameGIDs2, Teuchos::Array< GlobalOrdinal > &permuteGIDs1, Teuchos::Array< GlobalOrdinal > &permuteGIDs2, Teuchos::Array< GlobalOrdinal > &remoteGIDs1, Teuchos::Array< GlobalOrdinal > &remoteGIDs2, Teuchos::Array< int > &remotePIDs1, Teuchos::Array< int > &remotePIDs2) const
Find the union of the target IDs from two Import objects.
Definition: Tpetra_Import_def.hpp:1312
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::verboseOutputStream
Teuchos::FancyOStream & verboseOutputStream() const
Valid (nonnull) output stream for verbose output.
Definition: Tpetra_Details_Transfer_def.hpp:285
Tpetra::sort3
void sort3(const IT1 &first1, const IT1 &last1, const IT2 &first2, const IT3 &first3)
Sort the first array, and apply the same permutation to the second and third arrays.
Definition: Tpetra_Util.hpp:566
Tpetra::project1st
Binary function that returns its first argument.
Definition: Tpetra_ConfigDefs.hpp:172
Tpetra::Details::makeDualViewFromOwningHostView
void makeDualViewFromOwningHostView(Kokkos::DualView< ElementType *, DeviceType > &dv, const typename Kokkos::DualView< ElementType *, DeviceType >::t_host &hostView)
Initialize dv such that its host View is hostView.
Definition: Tpetra_Details_DualViewUtil.hpp:71
Tpetra::global_size_t
size_t global_size_t
Global size_t object.
Definition: Tpetra_ConfigDefs.hpp:109
Tpetra
Namespace Tpetra contains the class and methods constituting the Tpetra library.
Tpetra::Distributor::createFromRecvs
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::getSourceMap
Teuchos::RCP< const map_type > getSourceMap() const
The source Map used to construct this Export or Import.
Definition: Tpetra_Details_Transfer_def.hpp:248
Tpetra::Details::view_alloc_no_init
auto view_alloc_no_init(const std::string &label) -> decltype(Kokkos::view_alloc(label, Kokkos::WithoutInitializing))
Use in place of the string label as the first argument of Kokkos::View's constructor,...
Definition: Tpetra_Details_DualViewUtil.cpp:48
Tpetra::keyValueMerge
void keyValueMerge(KeyInputIterType keyBeg1, KeyInputIterType keyEnd1, ValueInputIterType valBeg1, ValueInputIterType valEnd1, KeyInputIterType keyBeg2, KeyInputIterType keyEnd2, ValueInputIterType valBeg2, ValueInputIterType valEnd2, KeyOutputIterType keyOut, ValueOutputIterType valOut, BinaryFunction f)
Merge two sorted (by keys) sequences of unique (key,value) pairs by combining pairs with equal keys.
Definition: Tpetra_Util.hpp:790
Tpetra::Details::Transfer< LocalOrdinal, GlobalOrdinal, Node >::verbose
bool verbose() const
Whether to print verbose debugging output.
Definition: Tpetra_Details_Transfer_def.hpp:295
Generated by doxygen 1.8.20