dolfin team mailing list archive

[Anders Logg <logg@xxxxxxxxx>]

On Fri, Mar 12, 2010 at 06:58:22PM -0000, noreply@xxxxxxxxxxxxx wrote:
> ------------------------------------------------------------
> revno: 4635
> committer: Garth N. Wells <gnw20@xxxxxxxxx>
> branch nick: dolfin-all
> timestamp: Fri 2010-03-12 18:53:05 +0000
> message:
>   Work on reading Vectors in parallel. Some issues to resolve still.
>
>   Some issues:
>   - How should files be named when in parallel?
>   - Should we have a 'master' xml file which points to the files
>   - from different processes?

I think this should be done in the same way as for Meshes. We
discussed the following design:

1. Reading a single file "foo.xml" results in each process reading the
entire file but skipping data located on another process as determined
by local_range. This is what is implemented now for meshes (followed
by communication and mesh partitioning). The difference for vectors
would be that no extra communication is necessary.

2. Reading a set of files "foo*.xml" results in each process reading
its portion stored in "foo%d.xml" % p. The File interface then needs
to check for the occurence of '*' and figure out the correct file name
based on its process number.

Note that since all auxiliary parallel data is stored in MeshData, it
should be possible to read in a mesh (stored in one file), partition
it and then store it to "foo*.xml". Then restart the program and
instead read the already partitioned data from the local file.

--
Anders


> modified:
>   dolfin/io/XMLArray.cpp
>   dolfin/io/XMLArray.h
>   dolfin/io/XMLFile.cpp
>   dolfin/io/XMLFile.h
>   dolfin/io/XMLVector.cpp
>   dolfin/io/XMLVector.h
>   dolfin/main/MPI.cpp
>   dolfin/main/MPI.h
>
>

> === modified file 'dolfin/io/XMLArray.cpp'
> --- dolfin/io/XMLArray.cpp	2010-03-09 23:40:57 +0000
> +++ dolfin/io/XMLArray.cpp	2010-03-12 18:53:05 +0000
> @@ -14,53 +14,56 @@
>
>  //-----------------------------------------------------------------------------
>  XMLArray::XMLArray(std::vector<int>& ix, XMLFile& parser)
> -  : XMLHandler(parser), ix(&ix), ux(0), dx(0), state(OUTSIDE_ARRAY),
> -    atype(INT), size(0)
> +  : XMLHandler(parser), ix(&ix), ux(0), dx(0),
> +    state(OUTSIDE_ARRAY), atype(INT)
>  {
>    // Do nothing
>  }
>  //-----------------------------------------------------------------------------
>  XMLArray::XMLArray(std::vector<uint>& ux, XMLFile& parser)
>    : XMLHandler(parser), ix(0), ux(&ux), dx(0), state(OUTSIDE_ARRAY),
> -    atype(UINT), size(0)
> +    atype(UINT)
>  {
>    // Do nothing
>  }
>  //-----------------------------------------------------------------------------
>  XMLArray::XMLArray(std::vector<double>& dx, XMLFile& parser)
>    : XMLHandler(parser), ix(0), ux(0), dx(&dx), state(OUTSIDE_ARRAY),
> -    atype(DOUBLE), size(0)
> +    atype(DOUBLE)
>  {
>    // Do nothing
>  }
>  //-----------------------------------------------------------------------------
>  XMLArray::XMLArray(std::vector<int>& ix, XMLFile& parser, uint size)
> -  : XMLHandler(parser), ix(&ix), ux(0), dx(0), state(INSIDE_ARRAY), atype(INT),
> -    size(size)
> +  : XMLHandler(parser), ix(&ix), ux(0), dx(0), state(INSIDE_ARRAY), atype(INT)
>  {
> +  indices.reserve(size);
>    this->ix->clear();
> -  this->ix->resize(size);
> +  this->ix->reserve(size);
>    std::fill(this->ix->begin(), this->ix->end(), 0);
>  }
>  //-----------------------------------------------------------------------------
>  XMLArray::XMLArray(std::vector<uint>& ux, XMLFile& parser, uint size)
> -  : XMLHandler(parser), ix(0), ux(&ux), dx(0), state(INSIDE_ARRAY), atype(UINT), size(size)
> +  : XMLHandler(parser), ix(0), ux(&ux), dx(0), state(INSIDE_ARRAY), atype(UINT)
>  {
> +  indices.reserve(size);
>    this->ux->clear();
> -  this->ux->resize(size);
> +  this->ux->reserve(size);
>    std::fill(this->ux->begin(), this->ux->end(), 0);
>  }
>  //-----------------------------------------------------------------------------
> -XMLArray::XMLArray(std::vector<double>& dx, XMLFile& parser, uint size)
> -  : XMLHandler(parser), ix(0), ux(0), dx(&dx), state(INSIDE_ARRAY),
> -    atype(DOUBLE), size(size)
> +XMLArray::XMLArray(std::vector<double>& dx,
> +                   XMLFile& parser, uint size)
> +  : XMLHandler(parser), indices(indices), ix(0), ux(0), dx(&dx), state(INSIDE_ARRAY),
> +    atype(DOUBLE)
>  {
> +  indices.reserve(size);
>    this->dx->clear();
> -  this->dx->resize(size);
> +  this->dx->reserve(size);
>    std::fill(this->dx->begin(), this->dx->end(), 0.0);
>  }
>  //-----------------------------------------------------------------------------
> -void XMLArray::start_element(const xmlChar *name, const xmlChar **attrs)
> +void XMLArray::start_element(const xmlChar* name, const xmlChar** attrs)
>  {
>    switch ( state )
>    {
> @@ -147,12 +150,14 @@
>    outfile << indent() << "</array>" << std::endl;
>  }
>  //-----------------------------------------------------------------------------
> -void XMLArray::read_array_tag(const xmlChar *name, const xmlChar **attrs)
> +void XMLArray::read_array_tag(const xmlChar* name, const xmlChar** attrs)
>  {
>    state = INSIDE_ARRAY;
>
>    // Parse size of array
> -  size = parse_uint(name, attrs, "size");
> +  const uint size = parse_uint(name, attrs, "size");
> +  indices.clear();
> +  indices.reserve(size);
>
>    std::string array_type = parse_string(name, attrs, "type");
>
> @@ -164,8 +169,7 @@
>        if (!array_type.compare("int") == 0)
>          error("Array file of type '%s', expected 'int'.", array_type.c_str());
>        ix->clear();
> -      ix->resize(size);
> -      std::fill(ix->begin(), ix->end(), 0);
> +      ix->reserve(size);
>        break;
>
>      case UINT:
> @@ -173,8 +177,7 @@
>        if (! array_type.compare("uint") == 0 )
>          error("Array file of type '%s', expected 'uint'.", array_type.c_str());
>        ux->clear();
> -      ux->resize(size);
> -      std::fill(ux->begin(), ux->end(), 0);
> +      ux->reserve(size);
>        break;
>
>      case DOUBLE:
> @@ -182,8 +185,7 @@
>        if (! array_type.compare("double") == 0 )
>          error("Array file of type '%s', expected 'double'.", array_type.c_str());
>        dx->clear();
> -      dx->resize(size);
> -      std::fill(dx->begin(), dx->end(), 0.0);
> +      dx->reserve(size);
>        break;
>
>      default:
> @@ -191,15 +193,20 @@
>    }
>  }
>  //-----------------------------------------------------------------------------
> -void XMLArray::read_entry(const xmlChar *name, const xmlChar **attrs)
> +void XMLArray::read_entry(const xmlChar* name, const xmlChar** attrs)
>  {
>    // Parse index
> -  uint index = parse_uint(name, attrs, "index");
> +  const uint index = parse_uint(name, attrs, "index");
> +  indices.push_back(index);
>
> +  /*
>    // Check values
> -  if (index >= size)
> +  if (index >= global_size)
> +  {
>      error("Illegal XML data for Array: row index %d out of range (0 - %d)",
> -          index, size - 1);
> +          index, global_size - 1);
> +  }
> +  */
>
>    int ivalue = 0;
>    uint uvalue = 0;
> @@ -211,19 +218,19 @@
>      case INT:
>        assert(ix);
>        ivalue = parse_int(name, attrs, "value");
> -      (*ix)[index] = ivalue;
> +      ix->push_back(ivalue);
>        break;
>
>       case UINT:
>        assert(ux);
>        uvalue = parse_uint(name, attrs, "value");
> -      (*ux)[index] = uvalue;
> +      ux->push_back(uvalue);
>        break;
>
>       case DOUBLE:
>        assert(dx);
>        dvalue = parse_float(name, attrs, "value");
> -      (*dx)[index] = dvalue;
> +      dx->push_back(dvalue);
>        break;
>
>       default:
>
> === modified file 'dolfin/io/XMLArray.h'
> --- dolfin/io/XMLArray.h	2010-03-09 23:40:57 +0000
> +++ dolfin/io/XMLArray.h	2010-03-12 18:53:05 +0000
> @@ -28,8 +28,8 @@
>      XMLArray(std::vector<double>& dx, XMLFile& parser);
>      XMLArray(std::vector<double>& dx, XMLFile& parser, uint size);
>
> -    void start_element (const xmlChar *name, const xmlChar **attrs);
> -    void end_element   (const xmlChar *name);
> +    void start_element(const xmlChar *name, const xmlChar **attrs);
> +    void end_element(const xmlChar *name);
>
>
>      /// Write to file
> @@ -44,12 +44,14 @@
>
>      void read_array_tag(const xmlChar *name, const xmlChar **attrs);
>
> +    std::vector<uint> indices;
> +
>    private:
>
>      enum parser_state { OUTSIDE_ARRAY, INSIDE_ARRAY, ARRAY_DONE };
>      enum array_type { INT, UINT, DOUBLE, UNSET };
>
> -    void read_entry  (const xmlChar *name, const xmlChar **attrs);
> +    void read_entry(const xmlChar *name, const xmlChar **attrs);
>
>      std::vector<int>*  ix;
>      std::vector<uint>* ux;
> @@ -57,8 +59,6 @@
>      parser_state state;
>      array_type atype;
>
> -    uint size;
> -
>    };
>
>  }
>
> === modified file 'dolfin/io/XMLFile.cpp'
> --- dolfin/io/XMLFile.cpp	2010-02-10 19:28:38 +0000
> +++ dolfin/io/XMLFile.cpp	2010-03-12 18:53:05 +0000
> @@ -92,15 +92,13 @@
>                                       (xmlStructuredErrorFunc)rng_valid_error,
>                                       stderr);
>    ret = xmlRelaxNGValidateDoc(validator, document);
> -  if ( ret == 0 ) {
> +  if ( ret == 0 )
>      info(0, "%s validates", filename.c_str());
> -  }
> -  else if ( ret < 0 ) {
> +  else if ( ret < 0 )
>      error("%s failed to load", filename.c_str());
> -  }
> -  else {
> +  else
>      error("%s fails to validate", filename.c_str());
> -  }
> +
>    xmlRelaxNGFreeValidCtxt(validator);
>  }
>  //-----------------------------------------------------------------------------
>
> === modified file 'dolfin/io/XMLFile.h'
> --- dolfin/io/XMLFile.h	2010-03-07 15:19:31 +0000
> +++ dolfin/io/XMLFile.h	2010-03-12 18:53:05 +0000
> @@ -71,17 +71,17 @@
>
>      void operator>> (Mesh& input)                  { read_xml(input); }
>      void operator>> (LocalMeshData& input)         { read_xml(input); }
> -    void operator>> (GenericMatrix&  input)        { read_xml(input); }
> -    void operator>> (GenericVector&  input)        { read_xml(input); }
> -    void operator>> (Parameters&  input)        { read_xml(input); }
> +    void operator>> (GenericMatrix& input)         { read_xml(input); }
> +    void operator>> (GenericVector& input)         { read_xml(input); }
> +    void operator>> (Parameters& input)            { read_xml(input); }
>      void operator>> (FunctionPlotData&  input)     { read_xml(input); }
>      void operator>> (MeshFunction<int>&  input)    { read_xml(input); }
>      void operator>> (MeshFunction<uint>&  input)   { read_xml(input); }
>      void operator>> (MeshFunction<double>&  input) { read_xml(input); }
>
> -    void operator>> (std::vector<int> & x)                            { read_xml_array(x); }
> -    void operator>> (std::vector<uint> & x)                           { read_xml_array(x); }
> -    void operator>> (std::vector<double> & x)                         { read_xml_array(x); }
> +    void operator>> (std::vector<int>& x)                             { read_xml_array(x); }
> +    void operator>> (std::vector<uint>& x)                            { read_xml_array(x); }
> +    void operator>> (std::vector<double>& x)                          { read_xml_array(x); }
>      void operator>> (std::map<uint, int>& map)                        { read_xml_map(map); }
>      void operator>> (std::map<uint, uint>& map)                       { read_xml_map(map); }
>      void operator>> (std::map<uint, double>& map)                     { read_xml_map(map); }
> @@ -94,15 +94,15 @@
>      void operator<< (const Mesh& output)                  { write_xml(output); }
>      void operator<< (const GenericMatrix& output)         { write_xml(output); }
>      void operator<< (const GenericVector& output)         { write_xml(output); }
> -    void operator<< (const Parameters& output)         { write_xml(output); }
> +    void operator<< (const Parameters& output)            { write_xml(output); }
>      void operator<< (const FunctionPlotData& output)      { write_xml(output); }
>      void operator<< (const MeshFunction<int>&  output)    { write_xml(output); }
>      void operator<< (const MeshFunction<uint>&  output)   { write_xml(output); }
>      void operator<< (const MeshFunction<double>&  output) { write_xml(output); }
>
> -    void operator<< (const std::vector<int> & x)                            { write_xml_array(x); }
> -    void operator<< (const std::vector<uint> & x)                           { write_xml_array(x); }
> -    void operator<< (const std::vector<double> & x)                         { write_xml_array(x); }
> +    void operator<< (const std::vector<int>& x)                             { write_xml_array(x); }
> +    void operator<< (const std::vector<uint>& x)                            { write_xml_array(x); }
> +    void operator<< (const std::vector<double>& x)                          { write_xml_array(x); }
>      void operator<< (const std::map<uint, int>& map)                        { write_xml_map(map); }
>      void operator<< (const std::map<uint, uint>& map)                       { write_xml_map(map); }
>      void operator<< (const std::map<uint, double>& map)                     { write_xml_map(map); }
> @@ -111,8 +111,8 @@
>      void operator<< (const std::map<uint, std::vector<double> >& array_map) { write_xml_map(array_map); }
>
>      // Friends
> -    friend void sax_start_element (void *ctx, const xmlChar *name, const xmlChar **attrs);
> -    friend void sax_end_element   (void *ctx, const xmlChar *name);
> +    friend void sax_start_element(void *ctx, const xmlChar *name, const xmlChar **attrs);
> +    friend void sax_end_element(void *ctx, const xmlChar *name);
>
>      void validate(std::string filename);
>
>
> === modified file 'dolfin/io/XMLVector.cpp'
> --- dolfin/io/XMLVector.cpp	2010-03-09 23:40:57 +0000
> +++ dolfin/io/XMLVector.cpp	2010-03-12 18:53:05 +0000
> @@ -6,10 +6,11 @@
>  // First added:  2009-03-06
>  // Last changed: 2009-06-15
>
> -
> +#include <algorithm>
>  #include <dolfin/common/Array.h>
>  #include <dolfin/log/dolfin_log.h>
>  #include <dolfin/la/GenericVector.h>
> +#include <dolfin/main/MPI.h>
>  #include "XMLIndent.h"
>  #include "XMLVector.h"
>
> @@ -71,10 +72,10 @@
>
>    const std::pair<uint, uint> range = vector.local_range();
>    const uint n0 = range.first;
> -  const uint size = range.second - range.first;
> +  const uint local_size  = range.second - range.first;
>
> -  // Get data
> -  Array<double> vector_values(size);
> +  // Get local data
> +  Array<double> vector_values(local_size);
>    vector.get_local(vector_values);
>
>    // Write array
> @@ -93,19 +94,24 @@
>  //-----------------------------------------------------------------------------
>  void XMLVector::read_array_tag(const xmlChar *name, const xmlChar **attrs)
>  {
> -  std::auto_ptr<XMLArray> _xml_array(new XMLArray(values, parser));
> -  xml_array = _xml_array;
> +  xml_array.reset(new XMLArray(values, parser));
>    xml_array->read_array_tag(name, attrs);
>    xml_array->handle();
>  }
>  //-----------------------------------------------------------------------------
>  void XMLVector::end_vector()
>  {
> -  // Copy values to vector
> -  x.resize(values.size());
> -  Array<double> v(values.size());
> -  for (uint i = 0; i< values.size(); ++i)
> -    v[i] = values[i];
> -  x.set_local(v);
> +  // Get global size
> +  const uint local_size = values.size();
> +  const uint global_size = dolfin::MPI::sum(local_size);
> +
> +  // Resize vector
> +  x.resize(global_size);
> +
> +  // Set values
> +  const std::vector<uint>& indices = xml_array->indices;
> +  assert(indices.size() == local_size);
> +  assert(*std::max_element(indices.begin(), indices.end()) < global_size);
> +  x.set(&values[0], local_size, &indices[0]);
>  }
>  //-----------------------------------------------------------------------------
>
> === modified file 'dolfin/io/XMLVector.h'
> --- dolfin/io/XMLVector.h	2009-06-15 12:34:15 +0000
> +++ dolfin/io/XMLVector.h	2010-03-12 18:53:05 +0000
> @@ -10,12 +10,13 @@
>  #define __XML_VECTOR_H
>
>  #include <memory>
> +#include <boost/scoped_ptr.hpp>
>  #include "XMLArray.h"
>  #include "XMLHandler.h"
>
>  namespace dolfin
>  {
> -
> +
>    class GenericVector;
>
>
> @@ -45,7 +46,7 @@
>      parser_state state;
>
>      std::vector<double> values;
> -    std::auto_ptr<XMLArray> xml_array;
> +    boost::scoped_ptr<XMLArray> xml_array;
>
>    };
>
>
> === modified file 'dolfin/main/MPI.cpp'
> --- dolfin/main/MPI.cpp	2010-02-28 00:09:17 +0000
> +++ dolfin/main/MPI.cpp	2010-03-12 18:53:05 +0000
> @@ -385,6 +385,14 @@
>    return recv_value;
>  }
>  //-----------------------------------------------------------------------------
> +dolfin::uint dolfin::MPI::sum(uint value)
> +{
> +  uint recv_value = 0.0;
> +  MPICommunicator comm;
> +  MPI_Allreduce(&value, &recv_value, 1, MPI_UNSIGNED, MPI_SUM, *comm);
> +  return recv_value;
> +}
> +//-----------------------------------------------------------------------------
>  dolfin::uint dolfin::MPI::global_offset(uint range, bool exclusive)
>  {
>    uint offset = 0;
> @@ -568,19 +576,21 @@
>  //-----------------------------------------------------------------------------
>  dolfin::uint dolfin::MPI::global_maximum(uint size)
>  {
> -  error("MPI::global_maximum() requires MPI.");
> -  return 0;
> +  return size;
>  }
>  //-----------------------------------------------------------------------------
>  double dolfin::MPI::sum(double value)
>  {
> -  error("MPI::sum() requires MPI.");
> -  return 0.0;
> +  return value;
> +}
> +//-----------------------------------------------------------------------------
> +dolfin::uint dolfin::MPI::sum(uint value)
> +{
> +  return value;
>  }
>  //-----------------------------------------------------------------------------
>  dolfin::uint dolfin::MPI::global_offset(uint range, bool exclusive)
>  {
> -  error("MPI::global_offset() requires MPI.");
>    return 0;
>  }
>  //-----------------------------------------------------------------------------
>
> === modified file 'dolfin/main/MPI.h'
> --- dolfin/main/MPI.h	2010-02-28 00:09:17 +0000
> +++ dolfin/main/MPI.h	2010-03-12 18:53:05 +0000
> @@ -97,6 +97,7 @@
>
>      /// Sum values and return sum
>      static double sum(double value);
> +    static uint sum(uint value);
>
>      /// Find global offset (index) (wrapper for MPI_(Ex)Scan with MPI_SUM as reduction op)
>      static uint global_offset(uint range, bool exclusive);
>

Attachment: signature.asc
Description: Digital signature