Redis source code series (1)

Time:2021-7-19

Redis source code series memory management

Function prototypesrc/zmalloc.h

Function pointer andvoid*Pointer provides a generic mechanism

/*stringfication*/
#define __xstr(s) __str(s)
#define __str(s) #s
/*prototypes*/
void *zmalloc(size_t size);
void *zcalloc(size_t size);
void *zrealloc(void *ptr, size_t size);
void zfree(void *ptr);
char *zstrdup(const char *s);
size_t zmalloc_used_memory(void);
void zmalloc_enable_thread_safeness(void);
void zmalloc_set_oom_handler(void (*oom_handler)(size_t));
float zmalloc_get_fragmentation_ratio(size_t rss);
size_t zmalloc_get_rss(void);
#ifndef HAVE_MALLOC_SIZE
size_t zmalloc_size(void *ptr);
#endif

Function implementationsrc/zmalloc.c

Several global static quantities

/*Memory already used*/
static size_t used_memory = 0;
/*Thread safety flag global static variable*/
static int zmalloc_thread_safe = 0;
/*Lock*/
pthread_mutex_t used_memory_mutex = PTHREAD_MUTEX_INITIALIZER;

1.zlic_free

Provide originallibcMemoryfreeFunction, which containszmalloc.hPreviously defined

/* This function provide us access to the original libc free(). This is useful
 * for instance to free results obtained by backtrace_symbols(). We need
 * to define this function before including zmalloc.h that may shadow the
 * free implementation if we use jemalloc or another non standard allocator. */
void zlibc_free(void *ptr) {
    free(ptr);
}

2.PREFIX_SIZE

According to different machines, it is defined as a word size

#if defined(__sun) || defined(__sparc) || defined(__sparc__)
#define PREFIX_SIZE (sizeof(long long))
#else
#define PREFIX_SIZE (sizeof(size_t))

3.zmalloc

redisMemory application function, internal usemallocFunction implementation

/*
 * zmalloc , zcalloc ,zrealloc
 *They all applied for one more prefix_ Szie memory size, and aligned with the word length
 */
void *zmalloc(size_t size) {
    /*Size is the actual required size
     * PREFIX_ Size is a precompiled macro: depending on the machine, it is used to store the value of size*/
    void *ptr = malloc(size+PREFIX_SIZE);
    /*Error handling: calling function default_ oom*/
    if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE
    update_zmalloc_stat_alloc(zmalloc_size(ptr));
    return ptr;
#else
    /*The first word length of allocated memory is put with the value of size*/
    *((size_t*)ptr) = size;
    /*Update the global amount of memory that has been used*/
    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
    /*Shift prefix to the right_ Size the size of the space pointed to by the pointer is size*/
    //     +--------------+-----------------+
    //     | PREFIX_SIZE  | size            |
    //     +--------------+-----------------+
    //     ^              ^
    //     |              |
    //    ptr            (char*)ptr+PREFIX_SIZE
    //It is also the address that the returned pointer points to
    return (char*)ptr+PREFIX_SIZE;
#endif
}

About error handling functions,zmalloc_oom_handlerIt is actually a function pointer

/*Error handling and exit*/
static void zmalloc_default_oom(size_t size) {
    fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n",
        size);
    fflush(stderr);
    abort();
}
/*Function pointer*/
static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom;

Its value points to the defaultoom(out of memory)Processing function

As for maintaining memory size global variablesupdate_zmalloc_stat_allocIs a macro function, which is implemented as follows:

/*Make the variable used_ Memory precisely maintains the actual allocated memory*/
#define update_zmalloc_stat_alloc(__n) do { \
    /*Change to size_ t*/
    size_t _n = (__n); \
    /* 
     *Judge whether it is aligned with the word length. For 64 bit machines, check whether the memory is aligned with 8
     *If it is not aligned, a certain offset is added to make it aligned
     */
    if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \
    /*Is it necessary to ensure thread safety*/
    if (zmalloc_thread_safe) { \
        update_zmalloc_stat_add(_n); \
    } else { \
        used_memory += _n; \
    } \
} while(0)

Here are a few things to note:

  1. Macrodo{...}while(0)skill

  2. Judge whether the bit operation is consistent with the modular operation

    \[a \%(2^n)=a\&(2^n-1)
    \]

  3. Do you need a thread safe implementation and call it if necessaryupdate_zmalloc_stat_addOtherwise, it will be increased directlyused_memory

The macro used in thread safety method is implemented as follows:

#ifdef HAVE_ATOMIC
#define update_zmalloc_stat_add(__n) __sync_add_and_fetch(&used_memory, (__n))
#else
/* 
 *Thread safety method update, using mutex to ensure thread safety
 *By update_ zmalloc_ stat_ Alloc call
 *First lock, then update, and finally unlock
 */
#define update_zmalloc_stat_add(__n) do { \
    pthread_mutex_lock(&used_memory_mutex); \
    /*Thread safety*/
    used_memory += (__n); \
    pthread_mutex_unlock(&used_memory_mutex); \
} while(0)

#endif

The mutex locks used here are all derived fromPOSIX multithreading, i.epthread.h

Now that I’m hereallocationWhen there is such a mechanism, thenfreeThere will be corresponding macro to maintain the large and small amount of memory

#define update_zmalloc_stat_sub(__n) do { \
    pthread_mutex_lock(&used_memory_mutex); \
    used_memory -= (__n); \
    pthread_mutex_unlock(&used_memory_mutex); \
} while(0)
#define update_zmalloc_stat_free(__n) do { \
    size_t _n = (__n); \
    if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \
    if (zmalloc_thread_safe) { \
        update_zmalloc_stat_sub(_n); \
    } else { \
        used_memory -= _n; \
    } \
} while(0)

It should be easy to understand

4.zcalloc

Internal callcallocrealization

void *zcalloc(size_t size) {
    /*
     *Calloc is a thread safe function
     *The allocated memory size is num * size
     *And initialize to 0
     */
    void *ptr = calloc(1, size+PREFIX_SIZE);

    if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE
    update_zmalloc_stat_alloc(zmalloc_size(ptr));
    return ptr;
#else
    *((size_t*)ptr) = size;
    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
    return (char*)ptr+PREFIX_SIZE;
#endif
}

5.zrealloc

Before that, take a lookreallocfunction

Reallocate (enlarge) memory function, internal callreallocfunction

/*Reallocate memory*/
void *zrealloc(void *ptr, size_t size) {
#ifndef HAVE_MALLOC_SIZE
    void *realptr;
#endif
    size_t oldsize;
    void *newptr;

    /*Reapply a piece of memory and return it*/
    if (ptr == NULL) return zmalloc(size);
#ifdef HAVE_MALLOC_SIZE
    oldsize = zmalloc_size(ptr);
	/*Calloc re request memory*/
    newptr = realloc(ptr,size);
    if (!newptr) zmalloc_oom_handler(size);
	/*Free original memory*/
    update_zmalloc_stat_free(oldsize);
	/*Update global quantity used_ memory*/
    update_zmalloc_stat_alloc(zmalloc_size(newptr));
    return newptr;
#else
 	/*Forward prefix_ SIZE*/
    realptr = (char*)ptr-PREFIX_SIZE;
	/*The size of the original memory*/
    oldsize = *((size_t*)realptr);
	/*Reapply memory*/
    newptr = realloc(realptr,size+PREFIX_SIZE);
    if (!newptr) zmalloc_oom_handler(size);
	/*Storage size*/
    *((size_t*)newptr) = size;
	/*Free the original space*/
    update_zmalloc_stat_free(oldsize);
	/*Update global quantity used_ memory*/
    update_zmalloc_stat_alloc(size);
    return (char*)newptr+PREFIX_SIZE;
#endif
}

The rest of the implementation is easy to understand

6.zmalloc_size

/* Provide zmalloc_size() for systems where this function is not provided by
 * malloc itself, given that in that case we store a header with this
 * information as the first bytes of every allocation.
 *
 */
#ifndef HAVE_MALLOC_SIZE
size_t zmalloc_size(void *ptr) {
    /*The size of malloc's memory*/
    /*Offset one word forward*/
    void *realptr = (char*)ptr-PREFIX_SIZE;
    /*Get size*/
    size_t size = *((size_t*)realptr);
    /* Assume at least that all the allocations are padded at sizeof(long) by
     * the underlying allocator. */
    /*Memory alignment*/
    if (size&(sizeof(long)-1)) size += sizeof(long)-(size&(sizeof(long)-1));
    return size+PREFIX_SIZE;
}
#endif

7.zstrdup

Copy string function

char *zstrdup(const char *s) {
    /*One more*/
    size_t l = strlen(s)+1;
    char *p = zmalloc(l);
    memcpy(p,s,l);
    return p;
}

8. zmalloc_used_memory

The thread safety method is implemented

/*Return used_ Memory thread safety*/
size_t zmalloc_used_memory(void) {
    size_t um;

    if (zmalloc_thread_safe) {
#ifdef HAVE_ATOMIC
        um = __sync_add_and_fetch(&used_memory, 0);
#else
        /*Lock*/
        pthread_mutex_lock(&used_memory_mutex);
        um = used_memory;
        /*Unlocking*/
        pthread_mutex_unlock(&used_memory_mutex);
#endif
    }/*Ensure thread safety*/
    else {
        um = used_memory;
    }
    return um;
}

9. help functions

/*
 *Is thread safety required
 *0 means no need 
 */
void zmalloc_enable_thread_safeness(void) {
    zmalloc_thread_safe = 1;
}
/*Out of memory: out of memory
 *The default is zmalloc_ default_ oom()
 */
void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) {
    zmalloc_oom_handler = oom_handler;
}

10.zmalloc_get_rss

Returns the actual size of the current process in memory, which is related to the operating system

#if defined(HAVE_PROC_STAT)
#include 
#include 
#include 
#include 
size_t zmalloc_get_rss(void) {
    /*Sysconf is the system function to get the page size*/
    int page = sysconf(_SC_PAGESIZE);
    size_t rss;
    char buf[4096];
    char filename[256];
    int fd, count;
    char *p, *x;

	/*Save the absolute path of the stat file corresponding to the current process to the file name*/
    snprintf(filename,256,"/proc/%d/stat",getpid());
	/*Read only open stat file*/
    if ((fd = open(filename,O_RDONLY)) == -1) return 0;
    if (read(fd,buf,4096) <= 0) {
        close(fd);
        return 0;
    }
    close(fd);
	/*After reading the information, save it to buf*/
    p = buf;
    count = 23; /*  RSS is the 24th field of / proc // stat*/
    while(p && count--) {
		/*Find spaces to separate fields*/
        p = strchr(p,' ');
		/*Point to the first address of the next field*/
        if (p) p++;
    }
    if (!p) return 0;
    x = strchr(p,' ');
    if (!x) return 0;
    *x = '
#if defined(HAVE_PROC_STAT)
#include 
#include 
#include 
#include 
size_t zmalloc_get_rss(void) {
/*Sysconf is the system function to get the page size*/
int page = sysconf(_SC_PAGESIZE);
size_t rss;
char buf[4096];
char filename[256];
int fd, count;
char *p, *x;
/*Save the absolute path of the stat file corresponding to the current process to the file name*/
snprintf(filename,256,"/proc/%d/stat",getpid());
/*Read only open stat file*/
if ((fd = open(filename,O_RDONLY)) == -1) return 0;
if (read(fd,buf,4096) <= 0) {
close(fd);
return 0;
}
close(fd);
/*After reading the information, save it to buf*/
p = buf;
count = 23; /*  RSS is the 24th field of / proc // stat*/
while(p && count--) {
/*Find spaces to separate fields*/
p = strchr(p,' ');
/*Point to the first address of the next field*/
if (p) p++;
}
if (!p) return 0;
x = strchr(p,' ');
if (!x) return 0;
*x = '\0';
/*string to long long*/
rss = strtoll(p,NULL,10);
/*RSS gets the number of pages in memory, multiplied by the page size*/
rss *= page;
return rss;
}
#else
size_t zmalloc_get_rss(void) {
/* If we can't get the RSS in an OS-specific way for this system just
* return the memory usage we estimated in zmalloc()..
*
* Fragmentation will appear to be always 1 (no fragmentation)
* of course... 
*
*If it cannot be obtained through the operating system, it will return used directly_ memory.
*/
return zmalloc_used_memory();
}
#endif
/* 
* Fragmentation = RSS / allocated-bytes 
*Memory fragmentation rate
*/
float zmalloc_get_fragmentation_ratio(size_t rss) {
return (float)rss/zmalloc_used_memory();
}
'; /*string to long long*/ rss = strtoll(p,NULL,10); /*RSS gets the number of pages in memory, multiplied by the page size*/ rss *= page; return rss; } #else size_t zmalloc_get_rss(void) { /* If we can't get the RSS in an OS-specific way for this system just * return the memory usage we estimated in zmalloc().. * * Fragmentation will appear to be always 1 (no fragmentation) * of course... * *If it cannot be obtained through the operating system, it will return used directly_ memory. */ return zmalloc_used_memory(); } #endif /* * Fragmentation = RSS / allocated-bytes *Memory fragmentation rate */ float zmalloc_get_fragmentation_ratio(size_t rss) { return (float)rss/zmalloc_used_memory(); }

Recommended Today

DDD Domain Driven Design: an in-depth interpretation of anemia model and congestion model

Foreword- In order to deeply grasp and understand the core of DDD domain driven design, we can’t get around two abstract concepts – “anemia model” and “congestion model”: The transaction model is the transaction script mode* Congestion model is the domain model pattern. -Anemia model- The anemia model was first widely used in EJB2 and […]