Actual source code: mal.c
petsc-3.13.6 2020-09-29
Report Typos and Errors

      1: /*
      2:     Code that allows a user to dictate what malloc() PETSc uses.
      3: */
      4:  #include <petscsys.h>
      5: #include <stdarg.h>
      6: #if defined(PETSC_HAVE_MALLOC_H)
      7: #include <malloc.h>
      8: #endif
      9: #if defined(PETSC_HAVE_MEMKIND)
     10: #include <errno.h>
     11: #include <memkind.h>
     12: typedef enum {PETSC_MK_DEFAULT=0,PETSC_MK_HBW_PREFERRED=1} PetscMemkindType;
     13: PetscMemkindType currentmktype = PETSC_MK_HBW_PREFERRED;
     14: PetscMemkindType previousmktype = PETSC_MK_HBW_PREFERRED;
     15: #endif
     16: /*
     17:         We want to make sure that all mallocs of double or complex numbers are complex aligned.
     18:     1) on systems with memalign() we call that routine to get an aligned memory location
     19:     2) on systems without memalign() we
     20:        - allocate one sizeof(PetscScalar) extra space
     21:        - we shift the pointer up slightly if needed to get PetscScalar aligned
     22:        - if shifted we store at ptr[-1] the amount of shift (plus a classid)
     23: */
     24: #define SHIFT_CLASSID 456123

     26: PETSC_EXTERN PetscErrorCode PetscMallocAlign(size_t mem,PetscBool clear,int line,const char func[],const char file[],void **result)
     27: {

     30:   if (!mem) {*result = NULL; return 0;}
     31: #if defined(PETSC_HAVE_MEMKIND)
     32:   {
     33:     if (!currentmktype) memkind_posix_memalign(MEMKIND_DEFAULT,result,PETSC_MEMALIGN,mem);
     34:     else memkind_posix_memalign(MEMKIND_HBW_PREFERRED,result,PETSC_MEMALIGN,mem);
     35:     if (ierr == EINVAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
     36:     if (ierr == ENOMEM) PetscInfo1(0,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
     37:     if (clear) {PetscMemzero(*result,mem);}
     38:   }
     39: #else
     40: #  if defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)
     41:   if (clear) {
     42:     *result = calloc(1+mem/sizeof(int),sizeof(int));
     43:   } else {
     44:     *result = malloc(mem);
     45:   }
     46:   if (PetscLogMemory) {PetscMemzero(*result,mem);}

     48: #  elif defined(PETSC_HAVE_MEMALIGN)
     49:   *result = memalign(PETSC_MEMALIGN,mem);
     50:   if (clear || PetscLogMemory) {
     51:     PetscMemzero(*result,mem);
     52:   }
     53: #  else
     54:   {
     55:     int *ptr;
     56:     /*
     57:       malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
     58:     */
     59:     if (clear) {
     60:       ptr = (int*)calloc(1+(mem + 2*PETSC_MEMALIGN)/sizeof(int),sizeof(int));
     61:     } else {
     62:       ptr = (int*)malloc(mem + 2*PETSC_MEMALIGN);
     63:     }
     64:     if (ptr) {
     65:       int shift    = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
     66:       shift        = (2*PETSC_MEMALIGN - shift)/sizeof(int);
     67:       ptr[shift-1] = shift + SHIFT_CLASSID;
     68:       ptr         += shift;
     69:       *result      = (void*)ptr;
     70:       if (PetscLogMemory) {PetscMemzero(*result,mem);}
     71:     } else {
     72:       *result      = NULL;
     73:     }
     74:   }
     75: #  endif
     76: #endif

     78:   if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
     79:   return 0;
     80: }

     82: PETSC_EXTERN PetscErrorCode PetscFreeAlign(void *ptr,int line,const char func[],const char file[])
     83: {
     84:   if (!ptr) return 0;
     85: #if defined(PETSC_HAVE_MEMKIND)
     86:   memkind_free(0,ptr); /* specify the kind to 0 so that memkind will look up for the right type */
     87: #else
     88: #  if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN))
     89:   {
     90:     /*
     91:       Previous int tells us how many ints the pointer has been shifted from
     92:       the original address provided by the system malloc().
     93:     */
     94:     int shift = *(((int*)ptr)-1) - SHIFT_CLASSID;
     95:     if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
     96:     if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
     97:     ptr = (void*)(((int*)ptr) - shift);
     98:   }
     99: #  endif

    101: #  if defined(PETSC_HAVE_FREE_RETURN_INT)
    102:   int err = free(ptr);
    103:   if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err);
    104: #  else
    105:   free(ptr);
    106: #  endif
    107: #endif
    108:   return 0;
    109: }

    111: PETSC_EXTERN PetscErrorCode PetscReallocAlign(size_t mem, int line, const char func[], const char file[], void **result)
    112: {

    115:   if (!mem) {
    116:     PetscFreeAlign(*result, line, func, file);
    117:     if (ierr) return ierr;
    118:     *result = NULL;
    119:     return 0;
    120:   }
    121: #if defined(PETSC_HAVE_MEMKIND)
    122:   if (!currentmktype) *result = memkind_realloc(MEMKIND_DEFAULT,*result,mem);
    123:   else *result = memkind_realloc(MEMKIND_HBW_PREFERRED,*result,mem);
    124: #else
    125: #  if (!(defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !defined(PETSC_HAVE_MEMALIGN))
    126:   {
    127:     /*
    128:       Previous int tells us how many ints the pointer has been shifted from
    129:       the original address provided by the system malloc().
    130:     */
    131:     int shift = *(((int*)*result)-1) - SHIFT_CLASSID;
    132:     if (shift > PETSC_MEMALIGN-1) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
    133:     if (shift < 0) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"Likely memory corruption in heap");
    134:     *result = (void*)(((int*)*result) - shift);
    135:   }
    136: #  endif

    138: #  if (defined(PETSC_HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) || defined(PETSC_HAVE_MEMALIGN)
    139:   *result = realloc(*result, mem);
    140: #  else
    141:   {
    142:     /*
    143:       malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
    144:     */
    145:     int *ptr = (int *) realloc(*result, mem + 2*PETSC_MEMALIGN);
    146:     if (ptr) {
    147:       int shift    = (int)(((PETSC_UINTPTR_T) ptr) % PETSC_MEMALIGN);
    148:       shift        = (2*PETSC_MEMALIGN - shift)/sizeof(int);
    149:       ptr[shift-1] = shift + SHIFT_CLASSID;
    150:       ptr         += shift;
    151:       *result      = (void*)ptr;
    152:     } else {
    153:       *result      = NULL;
    154:     }
    155:   }
    156: #  endif
    157: #endif
    158:   if (!*result) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
    159: #if defined(PETSC_HAVE_MEMALIGN)
    160:   /* There are no standard guarantees that realloc() maintains the alignment of memalign(), so I think we have to
    161:    * realloc and, if the alignment is wrong, malloc/copy/free. */
    162:   if (((size_t) (*result)) % PETSC_MEMALIGN) {
    163:     void *newResult;
    164: #  if defined(PETSC_HAVE_MEMKIND)
    165:     {
    166:       int ierr;
    167:       if (!currentmktype) memkind_posix_memalign(MEMKIND_DEFAULT,&newResult,PETSC_MEMALIGN,mem);
    168:       else memkind_posix_memalign(MEMKIND_HBW_PREFERRED,&newResult,PETSC_MEMALIGN,mem);
    169:       if (ierr == EINVAL) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_MEM,"Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
    170:       if (ierr == ENOMEM) PetscInfo1(0,"Memkind: fail to request HBW memory %.0f, falling back to normal memory\n",(PetscLogDouble)mem);
    171:     }
    172: #  else
    173:     newResult = memalign(PETSC_MEMALIGN,mem);
    174: #  endif
    175:     if (!newResult) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_MEM,PETSC_ERROR_INITIAL,"Memory requested %.0f",(PetscLogDouble)mem);
    176:     PetscMemcpy(newResult,*result,mem);
    177:     if (ierr) return ierr;
    178: #  if defined(PETSC_HAVE_FREE_RETURN_INT)
    179:     {
    180:       int err = free(*result);
    181:       if (err) return PetscError(PETSC_COMM_SELF,line,func,file,PETSC_ERR_PLIB,PETSC_ERROR_INITIAL,"System free returned error %d\n",err);
    182:     }
    183: #  else
    184: #    if defined(PETSC_HAVE_MEMKIND)
    185:     memkind_free(0,*result);
    186: #    else
    187:     free(*result);
    188: #    endif
    189: #  endif
    190:     *result = newResult;
    191:   }
    192: #endif
    193:   return 0;
    194: }

    196: PetscErrorCode (*PetscTrMalloc)(size_t,PetscBool,int,const char[],const char[],void**) = PetscMallocAlign;
    197: PetscErrorCode (*PetscTrFree)(void*,int,const char[],const char[])                     = PetscFreeAlign;
    198: PetscErrorCode (*PetscTrRealloc)(size_t,int,const char[],const char[],void**)          = PetscReallocAlign;

    200: PETSC_INTERN PetscBool petscsetmallocvisited;
    201: PetscBool petscsetmallocvisited = PETSC_FALSE;

    203: /*@C
    204:    PetscMallocSet - Sets the routines used to do mallocs and frees.
    205:    This routine MUST be called before PetscInitialize() and may be
    206:    called only once.

    208:    Not Collective

    210:    Input Parameters:
    211: + imalloc - the routine that provides the malloc (also provides calloc(), which is used depends on the second argument)
    212: . ifree - the routine that provides the free
    213: - iralloc - the routine that provides the realloc

    215:    Level: developer

    217: @*/
    218: PetscErrorCode PetscMallocSet(PetscErrorCode (*imalloc)(size_t,PetscBool,int,const char[],const char[],void**),
    219:                               PetscErrorCode (*ifree)(void*,int,const char[],const char[]),
    220:                               PetscErrorCode (*iralloc)(size_t, int, const char[], const char[], void **))
    221: {
    223:   if (petscsetmallocvisited && (imalloc != PetscTrMalloc || ifree != PetscTrFree)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"cannot call multiple times");
    224:   PetscTrMalloc         = imalloc;
    225:   PetscTrFree           = ifree;
    226:   PetscTrRealloc        = iralloc;
    227:   petscsetmallocvisited = PETSC_TRUE;
    228:   return(0);
    229: }

    231: /*@C
    232:    PetscMallocClear - Resets the routines used to do mallocs and frees to the defaults.

    234:    Not Collective

    236:    Level: developer

    238:    Notes:
    239:     In general one should never run a PETSc program with different malloc() and
    240:     free() settings for different parts; this is because one NEVER wants to
    241:     free() an address that was malloced by a different memory management system

    243:     Called in PetscFinalize() so that if PetscInitialize() is called again it starts with a fresh slate of allocation information

    245: @*/
    246: PetscErrorCode PetscMallocClear(void)
    247: {
    249:   PetscTrMalloc         = PetscMallocAlign;
    250:   PetscTrFree           = PetscFreeAlign;
    251:   PetscTrRealloc        = PetscReallocAlign;
    252:   petscsetmallocvisited = PETSC_FALSE;
    253:   return(0);
    254: }

    256: PetscErrorCode PetscMemoryTrace(const char label[])
    257: {
    258:   PetscErrorCode        ierr;
    259:   PetscLogDouble        mem,mal;
    260:   static PetscLogDouble oldmem = 0,oldmal = 0;

    263:   PetscMemoryGetCurrentUsage(&mem);
    264:   PetscMallocGetCurrentUsage(&mal);

    266:   PetscPrintf(PETSC_COMM_WORLD,"%s High water  %8.3f MB increase %8.3f MB Current %8.3f MB increase %8.3f MB\n",label,mem*1e-6,(mem - oldmem)*1e-6,mal*1e-6,(mal - oldmal)*1e-6);
    267:   oldmem = mem;
    268:   oldmal = mal;
    269:   return(0);
    270: }

    272: static PetscErrorCode (*PetscTrMallocOld)(size_t,PetscBool,int,const char[],const char[],void**) = PetscMallocAlign;
    273: static PetscErrorCode (*PetscTrReallocOld)(size_t,int,const char[],const char[],void**)          = PetscReallocAlign;
    274: static PetscErrorCode (*PetscTrFreeOld)(void*,int,const char[],const char[])                     = PetscFreeAlign;

    276: /*@C
    277:    PetscMallocSetDRAM - Set PetscMalloc to use DRAM.
    278:      If memkind is available, change the memkind type. Otherwise, switch the
    279:      current malloc and free routines to the PetscMallocAlign and
    280:      PetscFreeAlign (PETSc default).

    282:    Not Collective

    284:    Level: developer

    286:    Notes:
    287:      This provides a way to do the allocation on DRAM temporarily. One
    288:      can switch back to the previous choice by calling PetscMallocReset().

    290: .seealso: PetscMallocReset()
    291: @*/
    292: PetscErrorCode PetscMallocSetDRAM(void)
    293: {
    295:   if (PetscTrMalloc == PetscMallocAlign) {
    296: #if defined(PETSC_HAVE_MEMKIND)
    297:     previousmktype = currentmktype;
    298:     currentmktype  = PETSC_MK_DEFAULT;
    299: #endif
    300:   } else {
    301:     /* Save the previous choice */
    302:     PetscTrMallocOld  = PetscTrMalloc;
    303:     PetscTrReallocOld = PetscTrRealloc;
    304:     PetscTrFreeOld    = PetscTrFree;
    305:     PetscTrMalloc     = PetscMallocAlign;
    306:     PetscTrFree       = PetscFreeAlign;
    307:     PetscTrRealloc    = PetscReallocAlign;
    308:   }
    309:   return(0);
    310: }

    312: /*@C
    313:    PetscMallocResetDRAM - Reset the changes made by PetscMallocSetDRAM

    315:    Not Collective

    317:    Level: developer

    319: .seealso: PetscMallocSetDRAM()
    320: @*/
    321: PetscErrorCode PetscMallocResetDRAM(void)
    322: {
    324:   if (PetscTrMalloc == PetscMallocAlign) {
    325: #if defined(PETSC_HAVE_MEMKIND)
    326:     currentmktype = previousmktype;
    327: #endif
    328:   } else {
    329:     /* Reset to the previous choice */
    330:     PetscTrMalloc  = PetscTrMallocOld;
    331:     PetscTrRealloc = PetscTrReallocOld;
    332:     PetscTrFree    = PetscTrFreeOld;
    333:   }
    334:   return(0);
    335: }

    337: static PetscBool petscmalloccoalesce =
    338: #if defined(PETSC_USE_MALLOC_COALESCED)
    339:   PETSC_TRUE;
    340: #else
    341:   PETSC_FALSE;
    342: #endif

    344: /*@C
    345:    PetscMallocSetCoalesce - Use coalesced malloc when allocating groups of objects

    347:    Not Collective

    349:    Input Parameters:
    350: .  coalesce - PETSC_TRUE to use coalesced malloc for multi-object allocation.

    352:    Options Database Keys:
    353: .  -malloc_coalesce - turn coalesced malloc on or off

    355:    Note:
    356:    PETSc uses coalesced malloc by default for optimized builds and not for debugging builds.  This default can be changed via the command-line option -malloc_coalesce or by calling this function.
    357:    This function can only be called immediately after PetscInitialize()

    359:    Level: developer

    361: .seealso: PetscMallocA()
    362: @*/
    363: PetscErrorCode PetscMallocSetCoalesce(PetscBool coalesce)
    364: {
    366:   petscmalloccoalesce = coalesce;
    367:   return(0);
    368: }

    370: /*@C
    371:    PetscMallocA - Allocate and optionally clear one or more objects, possibly using coalesced malloc

    373:    Not Collective

    375:    Input Parameters:
    376: +  n - number of objects to allocate (at least 1)
    377: .  clear - use calloc() to allocate space initialized to zero
    378: .  lineno - line number to attribute allocation (typically __LINE__)
    379: .  function - function to attribute allocation (typically PETSC_FUNCTION_NAME)
    380: .  filename - file name to attribute allocation (typically __FILE__)
    381: -  bytes0 - first of n object sizes

    383:    Output Parameters:
    384: .  ptr0 - first of n pointers to allocate

    386:    Notes:
    387:    This function is not normally called directly by users, but rather via the macros PetscMalloc1(), PetscMalloc2(), or PetscCalloc1(), etc.

    389:    Level: developer

    391: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMalloc1(), PetscMalloc2(), PetscMalloc3(), PetscMalloc4(), PetscMalloc5(), PetscMalloc6(), PetscMalloc7(), PetscCalloc1(), PetscCalloc2(), PetscCalloc3(), PetscCalloc4(), PetscCalloc5(), PetscCalloc6(), PetscCalloc7(), PetscFreeA()
    392: @*/
    393: PetscErrorCode PetscMallocA(int n,PetscBool clear,int lineno,const char *function,const char *filename,size_t bytes0,void *ptr0,...)
    394: {
    396:   va_list        Argp;
    397:   size_t         bytes[8],sumbytes;
    398:   void           **ptr[8];
    399:   int            i;

    402:   if (n > 8) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Attempt to allocate %d objects but only 8 supported",n);
    403:   bytes[0] = bytes0;
    404:   ptr[0] = (void**)ptr0;
    405:   sumbytes = (bytes0 + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
    406:   va_start(Argp,ptr0);
    407:   for (i=1; i<n; i++) {
    408:     bytes[i] = va_arg(Argp,size_t);
    409:     ptr[i] = va_arg(Argp,void**);
    410:     sumbytes += (bytes[i] + PETSC_MEMALIGN-1) & ~(PETSC_MEMALIGN-1);
    411:   }
    412:   va_end(Argp);
    413:   if (petscmalloccoalesce) {
    414:     char *p;
    415:     (*PetscTrMalloc)(sumbytes,clear,lineno,function,filename,(void**)&p);
    416:     for (i=0; i<n; i++) {
    417:       *ptr[i] = bytes[i] ? p : NULL;
    418:       p = (char*)PetscAddrAlign(p + bytes[i]);
    419:     }
    420:   } else {
    421:     for (i=0; i<n; i++) {
    422:       (*PetscTrMalloc)(bytes[i],clear,lineno,function,filename,(void**)ptr[i]);
    423:     }
    424:   }
    425:   return(0);
    426: }

    428: /*@C
    429:    PetscFreeA - Free one or more objects, possibly allocated using coalesced malloc

    431:    Not Collective

    433:    Input Parameters:
    434: +  n - number of objects to free (at least 1)
    435: .  lineno - line number to attribute deallocation (typically __LINE__)
    436: .  function - function to attribute deallocation (typically PETSC_FUNCTION_NAME)
    437: .  filename - file name to attribute deallocation (typically __FILE__)
    438: -  ptr0 ... - first of n pointers to free

    440:    Note:
    441:    This function is not normally called directly by users, but rather via the macros PetscFree(), PetscFree2(), etc.

    443:    The pointers are zeroed to prevent users from accidently reusing space that has been freed.

    445:    Level: developer

    447: .seealso: PetscMallocAlign(), PetscMallocSet(), PetscMallocA(), PetscFree1(), PetscFree2(), PetscFree3(), PetscFree4(), PetscFree5(), PetscFree6(), PetscFree7()
    448: @*/
    449: PetscErrorCode PetscFreeA(int n,int lineno,const char *function,const char *filename,void *ptr0,...)
    450: {
    452:   va_list        Argp;
    453:   void           **ptr[8];
    454:   int            i;

    457:   if (n > 8) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Attempt to allocate %d objects but only up to 8 supported",n);
    458:   ptr[0] = (void**)ptr0;
    459:   va_start(Argp,ptr0);
    460:   for (i=1; i<n; i++) {
    461:     ptr[i] = va_arg(Argp,void**);
    462:   }
    463:   va_end(Argp);
    464:   if (petscmalloccoalesce) {
    465:     for (i=0; i<n; i++) {       /* Find first nonempty allocation */
    466:       if (*ptr[i]) break;
    467:     }
    468:     while (--n > i) {
    469:       *ptr[n] = NULL;
    470:     }
    471:     (*PetscTrFree)(*ptr[n],lineno,function,filename);
    472:     *ptr[n] = NULL;
    473:   } else {
    474:     while (--n >= 0) {
    475:       (*PetscTrFree)(*ptr[n],lineno,function,filename);
    476:       *ptr[n] = NULL;
    477:     }
    478:   }
    479:   return(0);
    480: }