Line data Source code
1 : /*
2 : * Copyright (c) 2011-2015: G-CSC, Goethe University Frankfurt
3 : * Author: Sebastian Reiter
4 : *
5 : * This file is part of UG4.
6 : *
7 : * UG4 is free software: you can redistribute it and/or modify it under the
8 : * terms of the GNU Lesser General Public License version 3 (as published by the
9 : * Free Software Foundation) with the following additional attribution
10 : * requirements (according to LGPL/GPL v3 §7):
11 : *
12 : * (1) The following notice must be displayed in the Appropriate Legal Notices
13 : * of covered and combined works: "Based on UG4 (www.ug4.org/license)".
14 : *
15 : * (2) The following notice must be displayed at a prominent place in the
16 : * terminal output of covered works: "Based on UG4 (www.ug4.org/license)".
17 : *
18 : * (3) The following bibliography is recommended for citation and must be
19 : * preserved in all covered files:
20 : * "Reiter, S., Vogel, A., Heppner, I., Rupp, M., and Wittum, G. A massively
21 : * parallel geometric multigrid solver on hierarchically distributed grids.
22 : * Computing and visualization in science 16, 4 (2013), 151-164"
23 : * "Vogel, A., Reiter, S., Rupp, M., Nägel, A., and Wittum, G. UG4 -- a novel
24 : * flexible software system for simulating pde based models on high performance
25 : * computers. Computing and visualization in science 16, 4 (2013), 165-179"
26 : *
27 : * This program is distributed in the hope that it will be useful,
28 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
29 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
30 : * GNU Lesser General Public License for more details.
31 : */
32 :
33 : #ifndef __H__UG__domain_distribution_impl__
34 : #define __H__UG__domain_distribution_impl__
35 :
36 : #include <functional> // for std::function
37 : #include "domain_distribution.h"
38 : #include "lib_grid/algorithms/attachment_util.h"
39 : #include "lib_grid/parallelization/deprecated/load_balancing.h"
40 : #include "common/serialization.h"
41 :
42 : #ifdef UG_PARALLEL
43 : #include "pcl/pcl.h"
44 : #include "lib_grid/parallelization/distribution.h"
45 : #endif
46 :
47 :
48 : namespace ug
49 : {
50 :
51 : /// partitions a domain by sorting all elements into a regular grid
52 : template <typename TDomain>
53 0 : static bool PartitionDomain_RegularGrid(TDomain& domain, PartitionMap& partitionMap,
54 : int numCellsX, int numCellsY, int numCellsZ,
55 : bool surfaceOnly)
56 : {
57 : PROFILE_FUNC_GROUP("parallelization");
58 : // prepare the partition map and a vertex position attachment accessor
59 : SmartPtr<MultiGrid> pMG = domain.grid();
60 0 : partitionMap.assign_grid(*pMG);
61 :
62 : #ifdef UG_PARALLEL
63 :
64 : SubsetHandler& partitionHandler = *partitionMap.get_partition_handler();
65 :
66 : // a distributed grid manager is required
67 : if(!domain.distributed_grid_manager()){
68 : UG_LOG("A distributed grid manager is required in the given domain.\n");
69 : return false;
70 : }
71 :
72 : typedef typename TDomain::position_attachment_type TAPos;
73 : Grid::AttachmentAccessor<Vertex, TAPos> aaPos(*pMG,
74 : domain.position_attachment());
75 :
76 : // this callback allows us to only distribute surface elements, which are no ghosts
77 : IsRegularSurfaceElem cbConsiderElem(*domain.distributed_grid_manager());
78 :
79 : // we need a process to which elements which are not considered will be send.
80 : // Those elements should stay on the current process.
81 : int localProc = 0;
82 : localProc = pcl::ProcRank();
83 :
84 : int bucketSubset = partitionMap.find_target_proc(localProc);
85 : if(bucketSubset == -1)
86 : bucketSubset = (int)partitionMap.num_target_procs();
87 :
88 : // partition the grid
89 : if(pMG->num<Volume>() > 0){
90 : if(!surfaceOnly)
91 : PartitionElements_RegularGrid<Volume>(
92 : partitionHandler,
93 : pMG->begin<Volume>(), pMG->end<Volume>(),
94 : numCellsX, numCellsY, numCellsZ, aaPos,
95 : ConsiderAll(), bucketSubset);
96 : else
97 : PartitionElements_RegularGrid<Volume>(
98 : partitionHandler,
99 : pMG->begin<Volume>(), pMG->end<Volume>(),
100 : numCellsX, numCellsY, numCellsZ, aaPos,
101 : cbConsiderElem, bucketSubset);
102 : }
103 : else if(pMG->num<Face>() > 0){
104 : if(!surfaceOnly)
105 : PartitionElements_RegularGrid<Face>(
106 : partitionHandler,
107 : pMG->begin<Face>(), pMG->end<Face>(),
108 : numCellsX, numCellsY, numCellsZ, aaPos,
109 : ConsiderAll(), bucketSubset);
110 : else
111 : PartitionElements_RegularGrid<Face>(
112 : partitionHandler,
113 : pMG->begin<Face>(), pMG->end<Face>(),
114 : numCellsX, numCellsY, numCellsZ, aaPos,
115 : cbConsiderElem, bucketSubset);
116 : }
117 : else if(pMG->num<Edge>() > 0){
118 : if(!surfaceOnly)
119 : PartitionElements_RegularGrid<Edge>(
120 : partitionHandler,
121 : pMG->begin<Edge>(), pMG->end<Edge>(),
122 : numCellsX, numCellsY, numCellsZ, aaPos,
123 : ConsiderAll(), bucketSubset);
124 : else
125 : PartitionElements_RegularGrid<Edge>(
126 : partitionHandler,
127 : pMG->begin<Edge>(), pMG->end<Edge>(),
128 : numCellsX, numCellsY, numCellsZ, aaPos,
129 : cbConsiderElem, bucketSubset);
130 : }
131 : else if(pMG->num<Vertex>() > 0){
132 : if(!surfaceOnly)
133 : PartitionElements_RegularGrid<Vertex>(
134 : partitionHandler,
135 : pMG->begin<Vertex>(), pMG->end<Vertex>(),
136 : numCellsX, numCellsY, numCellsZ, aaPos,
137 : ConsiderAll(), bucketSubset);
138 : else
139 : PartitionElements_RegularGrid<Vertex>(
140 : partitionHandler,
141 : pMG->begin<Vertex>(), pMG->end<Vertex>(),
142 : numCellsX, numCellsY, numCellsZ, aaPos,
143 : cbConsiderElem, bucketSubset);
144 : }
145 : else{
146 : LOG("partitioning could not be performed - "
147 : << "grid doesn't contain any elements!\n");
148 : return false;
149 : }
150 :
151 : // if elements have been assigned to bucketProc, then we have to make sure,
152 : // that it is also present in the process-map
153 : if(!partitionHandler.empty(bucketSubset)){
154 : if(bucketSubset >= (int)partitionMap.num_target_procs())
155 : partitionMap.add_target_proc(localProc);
156 : }
157 :
158 : return true;
159 : #endif
160 :
161 : UG_LOG("WARNING: PartitionDomain_RegularGrid is currently only implemented for");
162 : UG_LOG(" parallel environments.\n");
163 0 : return false;
164 : }
165 :
166 : template <typename TDomain>
167 : static bool
168 0 : PartitionDomain_MetisKWay(TDomain& domain, PartitionMap& partitionMap,
169 : int numPartitions, size_t baseLevel,
170 : int hWeight, int vWeight)
171 : {
172 : PROFILE_FUNC_GROUP("parallelization");
173 : // prepare the partition map
174 : SmartPtr<MultiGrid> pMG = domain.grid();
175 0 : partitionMap.assign_grid(*pMG);
176 :
177 : #ifdef UG_PARALLEL
178 :
179 : SubsetHandler& partitionHandler = *partitionMap.get_partition_handler();
180 :
181 : // we need a process to which elements which are not considered will be send.
182 : // Those elements should stay on the current process.
183 : int localProc = 0;
184 : localProc = pcl::ProcRank();
185 :
186 : int bucketSubset = partitionMap.find_target_proc(localProc);
187 : if(bucketSubset == -1)
188 : bucketSubset = (int)partitionMap.num_target_procs();
189 :
190 : // call the actual partitioning routine
191 : if(pMG->num<Volume>() > 0){
192 : PartitionMultiGrid_MetisKway<Volume>(partitionHandler, *pMG, numPartitions,
193 : baseLevel, hWeight, vWeight);
194 : // assign all elements below baseLevel to bucketSubset
195 : for(size_t lvl = 0; lvl < baseLevel; ++lvl)
196 : partitionHandler.assign_subset(pMG->begin<Volume>(lvl), pMG->end<Volume>(lvl),
197 : bucketSubset);
198 : }
199 : else if(pMG->num<Face>() > 0){
200 : PartitionMultiGrid_MetisKway<Face>(partitionHandler, *pMG, numPartitions,
201 : baseLevel, hWeight, vWeight);
202 : // assign all elements below baseLevel to bucketSubset
203 : for(size_t lvl = 0; lvl < baseLevel; ++lvl)
204 : partitionHandler.assign_subset(pMG->begin<Face>(lvl), pMG->end<Face>(lvl),
205 : bucketSubset);
206 : }
207 : else if(pMG->num<Edge>() > 0){
208 : PartitionMultiGrid_MetisKway<Edge>(partitionHandler, *pMG, numPartitions,
209 : baseLevel, hWeight, vWeight);
210 : // assign all elements below baseLevel to bucketSubset
211 : for(size_t lvl = 0; lvl < baseLevel; ++lvl)
212 : partitionHandler.assign_subset(pMG->begin<Edge>(lvl), pMG->end<Edge>(lvl),
213 : bucketSubset);
214 : }
215 :
216 : if(!partitionHandler.empty(bucketSubset)){
217 : if(bucketSubset >= (int)partitionMap.num_target_procs())
218 : partitionMap.add_target_proc(localProc);
219 : }
220 :
221 : return true;
222 : #else
223 : UG_LOG("WARNING in PartitionDomain_MetisKWay: Only available in parallel builds.\n");
224 0 : return false;
225 : #endif
226 : }
227 :
228 : template <typename TDomain>
229 : static bool
230 0 : PartitionDomain_MetisKWay(TDomain& domain, PartitionMap& partitionMap,
231 : int numPartitions, size_t baseLevel,
232 : SmartPtr<PartitionWeighting> weightFct)
233 : {
234 : PROFILE_FUNC_GROUP("parallelization");
235 : // prepare the partition map
236 : SmartPtr<MultiGrid> pMG = domain.grid();
237 0 : partitionMap.assign_grid(*pMG);
238 :
239 : #ifdef UG_PARALLEL
240 :
241 : SubsetHandler& partitionHandler = *partitionMap.get_partition_handler();
242 :
243 : PartitionWeighting& wFct = *weightFct;
244 : wFct.set_subset_handler(domain.subset_handler().operator->());
245 : // we need a process to which elements which are not considered will be send.
246 : // Those elements should stay on the current process.
247 : int localProc = 0;
248 : localProc = pcl::ProcRank();
249 :
250 : int bucketSubset = partitionMap.find_target_proc(localProc);
251 : if(bucketSubset == -1)
252 : bucketSubset = (int)partitionMap.num_target_procs();
253 :
254 : // call the actual partitioning routine
255 : if(pMG->num<Volume>() > 0){
256 : // do not use boost::function<...> f = wFct, since this leads to slicing
257 : // of wFct and losing properties of derived objects
258 : std::function<int (Volume*, Volume*)> f = std::ref(wFct);
259 : PartitionMultiGrid_MetisKway<Volume>(partitionHandler, *pMG, numPartitions, baseLevel, f);
260 : // assign all elements below baseLevel to bucketSubset
261 : for(size_t lvl = 0; lvl < baseLevel; ++lvl)
262 : partitionHandler.assign_subset(pMG->begin<Volume>(lvl), pMG->end<Volume>(lvl),
263 : bucketSubset);
264 : }
265 : else if(pMG->num<Face>() > 0){
266 : std::function<int (Face*, Face*)> f = std::ref(wFct);
267 : PartitionMultiGrid_MetisKway<Face>(partitionHandler, *pMG, numPartitions, baseLevel, f);
268 : // assign all elements below baseLevel to bucketSubset
269 : for(size_t lvl = 0; lvl < baseLevel; ++lvl)
270 : partitionHandler.assign_subset(pMG->begin<Face>(lvl), pMG->end<Face>(lvl),
271 : bucketSubset);
272 : }
273 : else if(pMG->num<Edge>() > 0){
274 : std::function<int (Edge*, Edge*)> f = std::ref(wFct);
275 : PartitionMultiGrid_MetisKway<Edge>(partitionHandler, *pMG, numPartitions, baseLevel, f);
276 : // assign all elements below baseLevel to bucketSubset
277 : for(size_t lvl = 0; lvl < baseLevel; ++lvl)
278 : partitionHandler.assign_subset(pMG->begin<Edge>(lvl), pMG->end<Edge>(lvl),
279 : bucketSubset);
280 : }
281 :
282 : if(!partitionHandler.empty(bucketSubset)){
283 : if(bucketSubset >= (int)partitionMap.num_target_procs())
284 : partitionMap.add_target_proc(localProc);
285 : }
286 :
287 : return true;
288 : #else
289 : UG_LOG("WARNING in PartitionDomain_MetisKWay: Only available in parallel builds.\n");
290 0 : return false;
291 : #endif
292 : }
293 :
294 :
295 : template <typename TDomain>
296 : static bool
297 0 : PartitionDomain_LevelBased(TDomain& domain, PartitionMap& partitionMap,
298 : int numPartitions, size_t level)
299 : {
300 : PROFILE_FUNC_GROUP("parallelization");
301 : // prepare the partition map
302 : SmartPtr<MultiGrid> pMG = domain.grid();
303 0 : partitionMap.assign_grid(*pMG);
304 0 : SubsetHandler& partitionHandler = *partitionMap.get_partition_handler();
305 :
306 : // call the actual partitioning routine
307 0 : switch(domain.domain_info().element_type()){
308 : case VOLUME:
309 0 : PartitionMultiGridLevel_MetisKway<Volume>(partitionHandler, *pMG, numPartitions, level);
310 : break;
311 :
312 : case FACE:
313 0 : PartitionMultiGridLevel_MetisKway<Face>(partitionHandler, *pMG, numPartitions, level);
314 : break;
315 :
316 : case EDGE:
317 0 : PartitionMultiGridLevel_MetisKway<Edge>(partitionHandler, *pMG, numPartitions, level);
318 : break;
319 :
320 0 : default:
321 0 : UG_THROW("Partitioning only works for element types EDGE, FACE, and VOLUME!");
322 : break;
323 : }
324 :
325 0 : return true;
326 : }
327 :
328 :
329 : template <typename TDomain>
330 : static bool
331 0 : PartitionDistributedDomain_LevelBased(TDomain& domain, PartitionMap& partitionMap,
332 : int numPartitions, size_t level)
333 : {
334 : PROFILE_FUNC_GROUP("parallelization");
335 : // prepare the partition map
336 : SmartPtr<MultiGrid> pMG = domain.grid();
337 0 : partitionMap.assign_grid(*pMG);
338 0 : SubsetHandler& partitionHandler = *partitionMap.get_partition_handler();
339 :
340 : // call the actual partitioning routine
341 0 : switch(domain.domain_info().element_type()){
342 : case VOLUME:
343 0 : PartitionMultiGridLevel_ParmetisKway<Volume>(partitionHandler, *pMG, numPartitions, level);
344 : break;
345 :
346 : case FACE:
347 0 : PartitionMultiGridLevel_ParmetisKway<Face>(partitionHandler, *pMG, numPartitions, level);
348 : break;
349 :
350 : case EDGE:
351 0 : PartitionMultiGridLevel_ParmetisKway<Edge>(partitionHandler, *pMG, numPartitions, level);
352 : break;
353 :
354 0 : default:
355 0 : UG_THROW("Partitioning only works for element types EDGE, FACE, and VOLUME!");
356 : break;
357 : }
358 :
359 0 : return true;
360 : }
361 :
362 :
363 : template <typename TDomain>
364 0 : static bool DistributeDomain(TDomain& domainOut,
365 : PartitionMap& partitionMap,
366 : bool createVerticalInterfaces)
367 : {
368 : PROFILE_FUNC_GROUP("parallelization");
369 : //todo Use a process-communicator to restrict communication
370 :
371 : // make sure that the input is fine
372 : typedef typename TDomain::grid_type GridType;
373 : SmartPtr<GridType> pGrid = domainOut.grid();
374 0 : SubsetHandler& partitionHandler = *partitionMap.get_partition_handler();
375 :
376 0 : if(partitionHandler.grid() != pGrid.get()){
377 0 : partitionMap.assign_grid(*pGrid);
378 : }
379 :
380 : #ifdef UG_PARALLEL
381 :
382 : typedef typename TDomain::position_attachment_type position_attachment_type;
383 :
384 : // used to check whether all processes are correctly prepared for redistribution
385 : //bool performDistribution = true;
386 :
387 : // make sure that the number of subsets and target processes match
388 : // THIS MAKES NO SENSE FOR PARALLEL REDISTRIBUTION - IT IS CLEAR THAT SOME
389 : // PROCS WON'T DELIVER TO ALL PROCS IN THE MAP.
390 : /* const int numSubs = partitionHandler.num_subsets();
391 : const int numTargetProcs = (int)partitionMap.num_target_procs();
392 : if(numSubs > numTargetProcs){
393 : UG_LOG("ERROR in RedistributeDomain: More partitions than target processes.\n");
394 : performDistribution = false;
395 : }
396 : else if(numSubs < numTargetProcs){
397 : UG_LOG("ERROR in RedistributeDomain: More target processes than partitions.\n");
398 : performDistribution = false;
399 : }
400 : */
401 :
402 : //todo: check whether all target-processes in partitionMap are in the valid range.
403 :
404 : PCL_PROFILE(RedistributeDomain);
405 :
406 : //todo Use a process-communicator to restrict communication
407 : /*
408 : if(!pcl::AllProcsTrue(performDistribution))
409 : return false;
410 : */
411 :
412 : // data serialization
413 : SPVertexDataSerializer posSerializer =
414 : GeomObjAttachmentSerializer<Vertex, position_attachment_type>::
415 : create(*pGrid, domainOut.position_attachment());
416 :
417 : SPGridDataSerializer shSerializer = SubsetHandlerSerializer::
418 : create(*domainOut.subset_handler());
419 :
420 : GridDataSerializationHandler serializer;
421 : serializer.add(posSerializer);
422 : serializer.add(shSerializer);
423 :
424 : std::vector<std::string> additionalSHNames = domainOut.additional_subset_handler_names();
425 : for(size_t i = 0; i < additionalSHNames.size(); ++i){
426 : SmartPtr<ISubsetHandler> sh = domainOut.additional_subset_handler(additionalSHNames[i]);
427 : if(sh.valid()){
428 : SPGridDataSerializer shSerializer = SubsetHandlerSerializer::create(*sh);
429 : serializer.add(shSerializer);
430 : }
431 : }
432 :
433 : // now call redistribution
434 : DistributeGrid(*pGrid, partitionHandler, serializer, createVerticalInterfaces,
435 : &partitionMap.get_target_proc_vec());
436 :
437 : PCL_PROFILE_END();
438 : #endif
439 :
440 : // in the serial case there's nothing to do.
441 0 : return true;
442 : }
443 :
444 : }// end of namespace
445 :
446 : #endif
|
