--- Begin Message ---
------------------------------------------------------------
revno: 5270
committer: Garth N. Wells <gnw20@xxxxxxxxx>
branch nick: dolfin-all
timestamp: Mon 2010-11-15 17:50:12 +0000
message:
Add initial interface for Zoltan.
Zoltan is part of Trilinos, and provides partitioning, renumbering and colouring functionality. It can act as an interface for ParMETIS, SCOTCH, etc.
added:
dolfin/graph/Zoltan.cpp
dolfin/graph/Zoltan.h
modified:
dolfin/fem/UFC.cpp
dolfin/fem/UFC.h
dolfin/fem/dolfin_fem.h
dolfin/graph/dolfin_graph.h
dolfin/la/SparsityPattern.h
--
lp:dolfin
https://code.launchpad.net/~dolfin-core/dolfin/main
Your team DOLFIN Core Team is subscribed to branch lp:dolfin.
To unsubscribe from this branch go to https://code.launchpad.net/~dolfin-core/dolfin/main/+edit-subscription
=== modified file 'dolfin/fem/UFC.cpp'
--- dolfin/fem/UFC.cpp 2010-11-14 23:11:09 +0000
+++ dolfin/fem/UFC.cpp 2010-11-15 17:50:12 +0000
@@ -2,10 +2,10 @@
// Licensed under the GNU LGPL Version 2.1.
//
// Modified by Ola Skavhaug, 2009
-// Modified by Garth N. Wells, 2009
+// Modified by Garth N. Wells, 2010
//
// First added: 2007-01-17
-// Last changed: 2010-11-09
+// Last changed: 2010-11-15
#include <dolfin/common/types.h>
#include <dolfin/function/FunctionSpace.h>
@@ -58,6 +58,12 @@
//-----------------------------------------------------------------------------
void UFC::init(const Form& form)
{
+ // Initialize mesh
+ this->mesh.init(form.mesh());
+
+ // Get function spaces for arguments
+ std::vector<boost::shared_ptr<const FunctionSpace> > V = form.function_spaces();
+
// Create finite elements
for (uint i = 0; i < this->form.rank(); i++)
{
@@ -84,27 +90,26 @@
for (uint i = 0; i < this->form.num_interior_facet_integrals(); i++)
interior_facet_integrals.push_back( boost::shared_ptr<ufc::interior_facet_integral>(this->form.create_interior_facet_integral(i)) );
- // Initialize mesh
- this->mesh.init(form.mesh());
-
- // Get function spaces for arguments
- std::vector<boost::shared_ptr<const FunctionSpace> > V = form.function_spaces();
+ // Get maximum local dimensions
+ std::vector<uint> max_local_dimension;
+ std::vector<uint> max_macro_local_dimension;
+ for (uint i = 0; i < this->form.rank(); i++)
+ {
+ max_local_dimension.push_back(V[i]->dofmap().max_local_dimension());
+ max_macro_local_dimension.push_back(2*V[i]->dofmap().max_local_dimension());
+ }
// Initialize local tensor
uint num_entries = 1;
for (uint i = 0; i < this->form.rank(); i++)
- num_entries *= V[i]->dofmap().max_local_dimension();
+ num_entries *= max_local_dimension[i];
A.reset(new double[num_entries]);
- for (uint i = 0; i < num_entries; i++)
- A[i] = 0.0;
// Initialize local tensor for macro element
num_entries = 1;
for (uint i = 0; i < this->form.rank(); i++)
- num_entries *= 2*V[i]->dofmap().max_local_dimension();
+ num_entries *= max_macro_local_dimension[i];
macro_A.reset(new double[num_entries]);
- for (uint i = 0; i < num_entries; i++)
- macro_A[i] = 0.0;
// Allocate memory for storing local dimensions
local_dimensions.reset(new uint[this->form.rank()]);
@@ -118,34 +123,17 @@
// Initialize dofs
dofs = new uint*[this->form.rank()];
for (uint i = 0; i < this->form.rank(); i++)
- {
- dofs[i] = new uint[V[i]->dofmap().max_local_dimension()];
- for (uint j = 0; j < V[i]->dofmap().max_local_dimension(); j++)
- dofs[i][j] = 0;
- }
- //dofs.ressize(this->form.rank());
- //for (uint i = 0; i < this->form.rank(); i++)
- // dof[i].resize(local_dimensions[i]);
+ dofs[i] = new uint[max_local_dimension[i]];
// Initialize dofs on macro element
macro_dofs = new uint*[this->form.rank()];
for (uint i = 0; i < this->form.rank(); i++)
- {
- const uint max_dimension = 2*V[i]->dofmap().max_local_dimension();
- macro_dofs[i] = new uint[max_dimension];
- for (uint j = 0; j < max_dimension; j++)
- macro_dofs[i][j] = 0;
- }
+ macro_dofs[i] = new uint[max_macro_local_dimension[i]];
// Initialize coefficients
w = new double*[this->form.num_coefficients()];
for (uint i = 0; i < this->form.num_coefficients(); i++)
- {
- const uint n = coefficient_elements[i].space_dimension();
- w[i] = new double[n];
- for (uint j = 0; j < n; j++)
- w[i][j] = 0.0;
- }
+ w[i] = new double[coefficient_elements[i].space_dimension()];
// Initialize coefficients on macro element
macro_w = new double*[this->form.num_coefficients()];
@@ -153,8 +141,6 @@
{
const uint n = 2*coefficient_elements[i].space_dimension();
macro_w[i] = new double[n];
- for (uint j = 0; j < n; j++)
- macro_w[i][j] = 0.0;
}
}
//-----------------------------------------------------------------------------
@@ -165,7 +151,10 @@
// Update local dimensions
for (uint i = 0; i < form.rank(); i++)
- local_dimensions[i] = dolfin_form.function_space(i)->dofmap().local_dimension(this->cell);
+ {
+ local_dimensions[i]
+ = dolfin_form.function_space(i)->dofmap().local_dimension(this->cell);
+ }
// Restrict coefficients to cell
for (uint i = 0; i < coefficients.size(); ++i)
@@ -182,7 +171,10 @@
// Update local dimensions
for (uint i = 0; i < form.rank(); i++)
- local_dimensions[i] = dolfin_form.function_space(i)->dofmap().local_dimension(this->cell);
+ {
+ local_dimensions[i]
+ = dolfin_form.function_space(i)->dofmap().local_dimension(this->cell);
+ }
// Restrict coefficients to facet
for (uint i = 0; i < coefficients.size(); ++i)
@@ -211,8 +203,10 @@
for (uint i = 0; i < coefficients.size(); ++i)
{
const uint offset = coefficient_elements[i].space_dimension();
- coefficients[i]->restrict(macro_w[i], coefficient_elements[i], cell0, this->cell0, local_facet0);
- coefficients[i]->restrict(macro_w[i] + offset, coefficient_elements[i], cell1, this->cell1, local_facet1);
+ coefficients[i]->restrict(macro_w[i], coefficient_elements[i],
+ cell0, this->cell0, local_facet0);
+ coefficients[i]->restrict(macro_w[i] + offset, coefficient_elements[i],
+ cell1, this->cell1, local_facet1);
}
}
//-----------------------------------------------------------------------------
=== modified file 'dolfin/fem/UFC.h'
--- dolfin/fem/UFC.h 2010-11-14 23:11:09 +0000
+++ dolfin/fem/UFC.h 2010-11-15 17:50:12 +0000
@@ -104,7 +104,6 @@
// Mapped dofs for primary arguments
uint** dofs;
- //std::vector<std::vector<uint> > dofs;
// Mapped dofs of macro element for primary arguments
uint** macro_dofs;
=== modified file 'dolfin/fem/dolfin_fem.h'
--- dolfin/fem/dolfin_fem.h 2010-11-08 09:16:09 +0000
+++ dolfin/fem/dolfin_fem.h 2010-11-15 17:50:12 +0000
@@ -14,6 +14,7 @@
#include <dolfin/fem/assemble.h>
#include <dolfin/fem/Form.h>
#include <dolfin/fem/Assembler.h>
+#include <dolfin/fem/SparsityPatternBuilder.h>
#include <dolfin/fem/SystemAssembler.h>
#include <dolfin/fem/MulticoreAssembler.h>
#include <dolfin/fem/VariationalProblem.h>
=== added file 'dolfin/graph/Zoltan.cpp'
--- dolfin/graph/Zoltan.cpp 1970-01-01 00:00:00 +0000
+++ dolfin/graph/Zoltan.cpp 2010-11-15 17:50:12 +0000
@@ -0,0 +1,156 @@
+// Copyright (C) 2010 Garth N. Wells.
+// Licensed under the GNU LGPL Version 2.1.
+//
+// First added: 2010-11-15
+// Last changed:
+
+#ifdef HAS_TRILINOS
+
+#include "dolfin/log/log.h"
+#include "Zoltan.h"
+
+using namespace dolfin;
+
+//-----------------------------------------------------------------------------
+ZoltanInterface::ZoltanInterface(const SparsityPattern& sparsity_pattern)
+ : sparsity_pattern(sparsity_pattern)
+{
+ if (sparsity_pattern.rank() != 2)
+ error("Can only create Zoltan object for SparsityPattern of rank 2.");
+
+ if (sparsity_pattern.size(0) != sparsity_pattern.size(1))
+ error("Can only create Zoltan object square SparsityPattern (for now).");
+}
+//-----------------------------------------------------------------------------
+int ZoltanInterface::num_global_objects() const
+{
+ return sparsity_pattern.size(0);
+}
+//-----------------------------------------------------------------------------
+int ZoltanInterface::num_local_objects() const
+{
+ return sparsity_pattern.size(0);
+}
+//-----------------------------------------------------------------------------
+void ZoltanInterface::num_edges_per_vertex(uint* num_edges) const
+{
+ sparsity_pattern.num_nonzeros_diagonal(num_edges);
+}
+//-----------------------------------------------------------------------------
+const std::vector<Set<dolfin::uint> >& ZoltanInterface::edges() const
+{
+ return sparsity_pattern.diagonal_pattern();
+}
+//-----------------------------------------------------------------------------
+std::vector<dolfin::uint> ZoltanInterface::local_renumbering_map()
+{
+ // Initialise Zoltan
+ float version;
+ int argc = 0;
+ char** argv = NULL;
+ Zoltan_Initialize(argc, argv, &version);
+
+ // Create Zoltan object
+ Zoltan zoltan(MPI::COMM_WORLD);
+
+ // Set parameters
+ zoltan.Set_Param( "ORDER_METHOD", "METIS");
+ zoltan.Set_Param( "NUM_GID_ENTRIES", "1"); /* global ID is 1 integer */
+ zoltan.Set_Param( "NUM_LID_ENTRIES", "1"); /* local ID is 1 integer */
+ zoltan.Set_Param( "OBJ_WEIGHT_DIM", "0"); /* omit object weights */
+
+ // Set call-back functions
+ zoltan.Set_Num_Obj_Fn(ZoltanInterface::get_number_of_objects, this);
+ zoltan.Set_Obj_List_Fn(ZoltanInterface::get_object_list, this);
+ zoltan.Set_Num_Edges_Multi_Fn(ZoltanInterface::get_number_edges, this);
+ zoltan.Set_Edge_List_Multi_Fn(ZoltanInterface::get_all_edges, this);
+
+ // Create array for global ids that should be renumbered
+ ZOLTAN_ID_PTR global_ids = new ZOLTAN_ID_TYPE[num_global_objects()];
+ for (int i = 0; i < num_global_objects(); ++i)
+ global_ids[i] = i;
+
+ // Create array for renumbered vertices
+ ZOLTAN_ID_PTR new_id = new ZOLTAN_ID_TYPE[num_global_objects()];
+
+ // Perform re-ordering
+ int rc = zoltan.Order(1, num_global_objects(), global_ids, new_id);
+
+ // Check for errors
+ if (rc != ZOLTAN_OK)
+ error("Partitioning failed");
+
+ // Copy renumber into a vector
+ std::vector<uint> map(num_global_objects());
+ for (uint i = 0; i < map.size(); ++i)
+ map[i] = new_id[i];
+
+ // Clean up
+ delete global_ids;
+ delete new_id;
+
+ return map;
+}
+//-----------------------------------------------------------------------------
+//-----------------------------------------------------------------------------
+int ZoltanInterface::get_number_of_objects(void *data, int *ierr)
+{
+ ZoltanInterface *objs = (ZoltanInterface *)data;
+ *ierr = ZOLTAN_OK;
+ return objs->num_local_objects();
+}
+//-----------------------------------------------------------------------------
+void ZoltanInterface::get_object_list(void *data, int sizeGID, int sizeLID,
+ ZOLTAN_ID_PTR globalID, ZOLTAN_ID_PTR localID,
+ int wgt_dim, float *obj_wgts, int *ierr)
+{
+ ZoltanInterface *objs = (ZoltanInterface *)data;
+ *ierr = ZOLTAN_OK;
+ for (int i = 0; i< objs->num_local_objects(); i++)
+ {
+ globalID[i] = i;
+ localID[i] = i;
+ }
+}
+//-----------------------------------------------------------------------------
+void ZoltanInterface::get_number_edges(void *data, int num_gid_entries,
+ int num_lid_entries,
+ int num_obj, ZOLTAN_ID_PTR global_ids,
+ ZOLTAN_ID_PTR local_ids, int *num_edges,
+ int *ierr)
+{
+ ZoltanInterface *objs = (ZoltanInterface *)data;
+
+ //std::cout << "Testing global id entires: " << num_gid_entries << " " << objs->num_global_objects() << std::endl;
+ //std::cout << "Testing local id entires: " << num_lid_entries << " " << objs->num_local_objects() << std::endl;
+ //assert(num_gid_entries == objs->num_global_objects());
+ //assert(num_lid_entries == objs->num_local_objects());
+
+ objs->num_edges_per_vertex(reinterpret_cast<uint*>(num_edges));
+}
+//-----------------------------------------------------------------------------
+void ZoltanInterface::get_all_edges(void *data, int num_gid_entries,
+ int num_lid_entries, int num_obj,
+ ZOLTAN_ID_PTR global_ids,
+ ZOLTAN_ID_PTR local_ids,
+ int *num_edges,
+ ZOLTAN_ID_PTR nbor_global_id,
+ int *nbor_procs, int wgt_dim,
+ float *ewgts, int *ierr)
+{
+ std::cout << "Testing:" << num_gid_entries << " " << num_lid_entries << std::endl;
+
+ ZoltanInterface *objs = (ZoltanInterface *)data;
+ const std::vector<Set<uint> >& edges = objs->edges();
+
+ uint sum = 0;
+ for (uint i = 0; i < edges.size(); ++i)
+ {
+ assert(edges[i].size() == (uint) num_edges[i]);
+ for (uint j = 0; j < edges[i].size(); ++j)
+ nbor_global_id[sum*num_gid_entries + j] = edges[i][j];
+ sum += edges[i].size();
+ }
+}
+//-----------------------------------------------------------------------------
+#endif
=== added file 'dolfin/graph/Zoltan.h'
--- dolfin/graph/Zoltan.h 1970-01-01 00:00:00 +0000
+++ dolfin/graph/Zoltan.h 2010-11-15 17:50:12 +0000
@@ -0,0 +1,84 @@
+// Copyright (C) 2010 Garth N. Wells.
+// Licensed under the GNU LGPL Version 2.1.
+//
+// First added: 2010-11-15
+// Last changed:
+
+#ifndef __DOLFIN_ZOLTAN_H
+#define __DOLFIN_ZOLTAN_H
+
+#ifdef HAS_TRILINOS
+
+#include <vector>
+#include <zoltan_cpp.h>
+#include "dolfin/common/Set.h"
+#include "dolfin/common/types.h"
+#include <dolfin/la/SparsityPattern.h>
+
+namespace dolfin
+{
+
+ /// This class provides an interface for Zoltan
+
+ class ZoltanInterface
+ {
+
+ public:
+
+ /// Build distribted dual graph for mesh
+ ZoltanInterface(const SparsityPattern& sparsity_pattern);
+
+ private:
+
+ /// Number of global graph vertices
+ int num_global_objects() const;
+
+ /// Number of local graph vertices
+ int num_local_objects() const;
+
+ /// Number of edges per vertex
+ void num_edges_per_vertex(uint* num_edges) const;
+
+ /// Vertex edges
+ const std::vector<Set<uint> >& edges() const;
+
+ public:
+
+ /// Runumbering map on on process (map[old] -> new)
+ std::vector<uint> local_renumbering_map();
+
+ private:
+
+ // Zoltan call-back functions
+
+ static int get_number_of_objects(void *data, int *ierr);
+
+ static void get_object_list(void *data, int sizeGID, int sizeLID,
+ ZOLTAN_ID_PTR globalID,
+ ZOLTAN_ID_PTR localID, int wgt_dim,
+ float *obj_wgts, int *ierr);
+
+ static void get_number_edges(void *data, int num_gid_entries,
+ int num_lid_entries,
+ int num_obj, ZOLTAN_ID_PTR global_ids,
+ ZOLTAN_ID_PTR local_ids, int *num_edges,
+ int *ierr);
+
+ static void get_all_edges(void *data, int num_gid_entries,
+ int num_lid_entries, int num_obj,
+ ZOLTAN_ID_PTR global_ids,
+ ZOLTAN_ID_PTR local_ids,
+ int *num_edges,
+ ZOLTAN_ID_PTR nbor_global_id,
+ int *nbor_procs, int wgt_dim,
+ float *ewgts, int *ierr);
+
+
+ const SparsityPattern& sparsity_pattern;
+
+ };
+
+}
+
+#endif
+#endif
=== modified file 'dolfin/graph/dolfin_graph.h'
--- dolfin/graph/dolfin_graph.h 2010-02-10 19:28:38 +0000
+++ dolfin/graph/dolfin_graph.h 2010-11-15 17:50:12 +0000
@@ -3,5 +3,6 @@
// DOLFIN graph
+#include <dolfin/graph/Zoltan.h>
#endif
=== modified file 'dolfin/la/SparsityPattern.h'
--- dolfin/la/SparsityPattern.h 2010-02-24 21:47:47 +0000
+++ dolfin/la/SparsityPattern.h 2010-11-15 17:50:12 +0000
@@ -11,6 +11,7 @@
#include <vector>
#include "dolfin/common/Set.h"
+#include "dolfin/common/types.h"
#include "GenericSparsityPattern.h"
namespace dolfin
--- End Message ---
