Actual source code: vscreate.c
petsc-3.13.6 2020-09-29
Report Typos and Errors

      1: /*
      2:     The VECSCATTER (vec scatter) interface routines, callable by users.
      3: */

      5:  #include <petsc/private/vecscatterimpl.h>

      7: /* Logging support */
      8: PetscClassId VEC_SCATTER_CLASSID;

     10: PetscFunctionList VecScatterList              = NULL;
     11: PetscBool         VecScatterRegisterAllCalled = PETSC_FALSE;

     13: /*@C
     14:   VecScatterSetType - Builds a vector scatter, for a particular vector scatter implementation.

     16:   Collective on VecScatter

     18:   Input Parameters:
     19: + vscat - The vector scatter object
     20: - type - The name of the vector scatter type

     22:   Options Database Key:
     23: . -vecscatter_type <type> - Sets the vector scatter type; use -help for a list
     24:                      of available types

     26:   Notes:
     27:   See "petsc/include/petscvec.h" for available vector scatter types (for instance, VECSCATTERMPI1, or VECSCATTERMPI3NODE).

     29:   Use VecScatterDuplicate() to form additional vectors scatter of the same type as an existing vector scatter.

     31:   Level: intermediate

     33: .seealso: VecScatterGetType(), VecScatterCreate()
     34: @*/
     35: PetscErrorCode VecScatterSetType(VecScatter vscat, VecScatterType type)
     36: {
     37:   PetscBool      match;
     39:   PetscErrorCode (*r)(VecScatter);

     43:   PetscObjectTypeCompare((PetscObject)vscat, type, &match);
     44:   if (match) return(0);

     46:   PetscFunctionListFind(VecScatterList,type,&r);
     47:   if (!r) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_UNKNOWN_TYPE, "Unknown vector scatter type: %s",type);

     49:   if (vscat->ops->destroy) {
     50:     (*vscat->ops->destroy)(vscat);
     51:     vscat->ops->destroy = NULL;
     52:   }

     54:   (*r)(vscat);
     55:   return(0);
     56: }

     58: /*@C
     59:   VecScatterGetType - Gets the vector scatter type name (as a string) from the VecScatter.

     61:   Not Collective

     63:   Input Parameter:
     64: . vscat  - The vector scatter

     66:   Output Parameter:
     67: . type - The vector scatter type name

     69:   Level: intermediate

     71: .seealso: VecScatterSetType(), VecScatterCreate()
     72: @*/
     73: PetscErrorCode VecScatterGetType(VecScatter vscat, VecScatterType *type)
     74: {

     80:   VecScatterRegisterAll();
     81:   *type = ((PetscObject)vscat)->type_name;
     82:   return(0);
     83: }

     85: /*@
     86:   VecScatterSetFromOptions - Configures the vector scatter from the options database.

     88:   Collective on VecScatter

     90:   Input Parameter:
     91: . vscat - The vector scatter

     93:   Notes:
     94:     To see all options, run your program with the -help option, or consult the users manual.
     95:           Must be called before VecScatterSetUp() but before the vector scatter is used.

     97:   Level: beginner

    100: .seealso: VecScatterCreate(), VecScatterDestroy(), VecScatterSetUp()
    101: @*/
    102: PetscErrorCode VecScatterSetFromOptions(VecScatter vscat)
    103: {
    105:   PetscBool      opt;
    106:   VecScatterType defaultType;
    107:   char           typeName[256];
    108:   PetscMPIInt    size;

    113:   PetscObjectOptionsBegin((PetscObject)vscat);
    114:   MPI_Comm_size(PetscObjectComm((PetscObject)vscat), &size);

    116:   /* Handle vector type options */
    117:   if (((PetscObject)vscat)->type_name) {
    118:     defaultType = ((PetscObject)vscat)->type_name;
    119:   } else {
    120:     if (size > 1) defaultType = VECSCATTERMPI1;
    121:     else defaultType = VECSCATTERSEQ;
    122:   }

    124:   VecScatterRegisterAll();
    125:   PetscOptionsFList("-vecscatter_type","Vector Scatter type","VecScatterSetType",VecScatterList,defaultType,typeName,256,&opt);
    126:   if (size > 1 && opt) {
    127:     VecScatterSetType(vscat,typeName);
    128:   } else {
    129:     VecScatterSetType(vscat,defaultType);
    130:   }

    132:   vscat->beginandendtogether = PETSC_FALSE;
    133:   PetscOptionsBool("-vecscatter_merge","Use combined (merged) vector scatter begin and end","VecScatterCreate",vscat->beginandendtogether,&vscat->beginandendtogether,NULL);
    134:   if (vscat->beginandendtogether) {
    135:     PetscInfo(vscat,"Using combined (merged) vector scatter begin and end\n");
    136:   }

    138:   vscat->packongpu = PETSC_TRUE;
    139:   PetscOptionsBool("-vecscatter_packongpu","For GPU vectors, pack needed entries on GPU, then copy packed data to CPU, then do MPI","VecScatterCreate",vscat->packongpu,&vscat->packongpu,NULL);
    140:   if (vscat->packongpu) {
    141:     PetscInfo(vscat,"For GPU vectors, pack needed entries on GPU, then copy packed data to CPU, then do MPI\n");
    142:   }

    144:   PetscOptionsEnd();
    145:   return(0);
    146: }

    148: /*@C
    149:   VecScatterRegister -  Adds a new vector scatter component implementation

    151:   Not Collective

    153:   Input Parameters:
    154: + name        - The name of a new user-defined creation routine
    155: - create_func - The creation routine itself

    157:   Notes:
    158:   VecScatterRegister() may be called multiple times to add several user-defined vectors

    160:   Sample usage:
    161: .vb
    162:     VecScatterRegister("my_vecscatter",MyVecScatterCreate);
    163: .ve

    165:   Then, your vector scatter type can be chosen with the procedural interface via
    166: .vb
    167:     VecScatterCreate(MPI_Comm, VecScatter *);
    168:     VecScatterSetType(VecScatter,"my_vectorscatter_name");
    169: .ve
    170:    or at runtime via the option
    171: .vb
    172:     -vecscatter_type my_vectorscatter_name
    173: .ve

    175:   Level: advanced

    177: .seealso: VecScatterRegisterAll(), VecScatterRegisterDestroy()
    178: @*/
    179: PetscErrorCode VecScatterRegister(const char sname[], PetscErrorCode (*function)(VecScatter))
    180: {

    184:   VecInitializePackage();
    185:   PetscFunctionListAdd(&VecScatterList,sname,function);
    186:   return(0);
    187: }

    189: /* ---------------------------------------------------------------- */
    190: /*@
    191:    VecScatterCreate - Creates a vector scatter context.

    193:    Collective on Vec

    195:    Input Parameters:
    196: +  xin - a vector that defines the shape (parallel data layout of the vector)
    197:          of vectors from which we scatter
    198: .  yin - a vector that defines the shape (parallel data layout of the vector)
    199:          of vectors to which we scatter
    200: .  ix - the indices of xin to scatter (if NULL scatters all values)
    201: -  iy - the indices of yin to hold results (if NULL fills entire vector yin)

    203:    Output Parameter:
    204: .  newctx - location to store the new scatter context

    206:    Options Database Keys:
    207: +  -vecscatter_view         - Prints detail of communications
    208: .  -vecscatter_view ::ascii_info    - Print less details about communication
    209: .  -vecscatter_merge        - VecScatterBegin() handles all of the communication, VecScatterEnd() is a nop
    210:                               eliminates the chance for overlap of computation and communication
    211: .  -vecscatter_packtogether - Pack all messages before sending, receive all messages before unpacking
    212:                               will make the results of scatters deterministic when otherwise they are not (it may be slower also).
    213: .  -vecscatter_type sf      - Use the PetscSF implementation of vecscatter (Default). One can use PetscSF options to control the communication.
    214: -  -vecscatter_packongpu    - For GPU vectors, pack needed entries on GPU, then copy packed data to CPU, then do MPI.
    215:                               Otherwise, we might copy a segment encompassing needed entries. Default is TRUE.

    217:     Level: intermediate

    219:   Notes:
    220:    If both xin and yin are parallel, their communicator must be on the same
    221:    set of processes, but their process order can be different.
    222:    In calls to VecScatter() you can use different vectors than the xin and
    223:    yin you used above; BUT they must have the same parallel data layout, for example,
    224:    they could be obtained from VecDuplicate().
    225:    A VecScatter context CANNOT be used in two or more simultaneous scatters;
    226:    that is you cannot call a second VecScatterBegin() with the same scatter
    227:    context until the VecScatterEnd() has been called on the first VecScatterBegin().
    228:    In this case a separate VecScatter is needed for each concurrent scatter.

    230:    Currently the MPI_Send() use PERSISTENT versions.
    231:    (this unfortunately requires that the same in and out arrays be used for each use, this
    232:     is why  we always need to pack the input into the work array before sending
    233:     and unpack upon receiving instead of using MPI datatypes to avoid the packing/unpacking).

    235:    Both ix and iy cannot be NULL at the same time.

    237:    Use VecScatterCreateToAll() to create a vecscatter that copies an MPI vector to sequential vectors on all MPI ranks.
    238:    Use VecScatterCreateToZero() to create a vecscatter that copies an MPI vector to a sequential vector on MPI rank 0.
    239:    These special vecscatters have better performance than general ones.

    241: .seealso: VecScatterDestroy(), VecScatterCreateToAll(), VecScatterCreateToZero(), PetscSFCreate()
    242: @*/
    243: PetscErrorCode VecScatterCreate(Vec xin,IS ix,Vec yin,IS iy,VecScatter *newctx)
    244: {
    245:   VecScatter        ctx;
    246:   PetscErrorCode    ierr;
    247:   PetscMPIInt       xsize,ysize,result;
    248:   MPI_Comm          comm,xcomm,ycomm;

    252:   *newctx = NULL;

    254:   if (!ix && !iy) SETERRQ(PetscObjectComm((PetscObject)xin),PETSC_ERR_SUP,"Cannot pass default in for both input and output indices");

    256:   /* Get comm from xin and yin */
    257:   PetscObjectGetComm((PetscObject)xin,&xcomm);
    258:   MPI_Comm_size(xcomm,&xsize);
    259:   PetscObjectGetComm((PetscObject)yin,&ycomm);
    260:   MPI_Comm_size(ycomm,&ysize);
    261:   if (xsize > 1 && ysize > 1) {
    262:     MPI_Comm_compare(xcomm,ycomm,&result);
    263:     if (result == MPI_UNEQUAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NOTSAMECOMM,"VecScatterCreate: parallel vectors xin and yin must have identical/congruent/similar communicators");
    264:   }

    266:   /* If xsize==ysize, we use ycomm as comm of ctx, which does affect correctness of -vecscatter_type mpi1. But it does not
    267:      affect -vecscatter_type sf, since sf internally uses the roots' comm (equal to xcomm when x is parallel) as the sf
    268:      object's comm. One day when we degrade VecScatter from a petsc object to a simple wrapper around PetscSF, we can get
    269:      rid of this confusion.
    270:    */
    271:   comm = xsize > ysize ? xcomm : ycomm;

    273:   VecScatterInitializePackage();
    274:   PetscHeaderCreate(ctx,VEC_SCATTER_CLASSID,"VecScatter","Vector Scatter","Vec",comm,VecScatterDestroy,VecScatterView);

    276:   ctx->from_v  = xin;
    277:   ctx->to_v    = yin;
    278:   ctx->from_is = ix;
    279:   ctx->to_is   = iy;
    280:   VecGetLocalSize(xin,&ctx->from_n);
    281:   VecGetLocalSize(yin,&ctx->to_n);

    283:   /* Set default scatter type */
    284:   VecScatterSetType(ctx,VECSCATTERSF);

    286:   VecScatterSetFromOptions(ctx);
    287:   VecScatterSetUp(ctx);

    289:   *newctx = ctx;
    290:   return(0);
    291: }