Source code analysis of spool — reactor base of reactor module

preface

As a network framework, the core is to receive and send messages. EfficientreactorMode has always been the first choice of many network frameworks, this section mainly explainsswooleInreactormodular.

Unp learning notes: IO reuse

ReactorData structure of

• ReactorThe data structure of is more complexobjectIt’s concreteReactorThe first address of the object,ptrIt’s ownershipReactorPointer to the class of the object,
• event_numStorage of existing monitored datafdnumber,max_event_numThe maximum number of events allowed to be held,flagIs the tag bit,
• idIt is used to store the corresponding datareactorOfid，runningUsed to mark thereactorWhether it is running or not is usually set to 1 when it is created,startMarked withreactorWhether it has been started or not is generally a matter of timewaitIt is set to 1 during monitoring,onceMarksreactorWhether it only needs one-time monitoring,check_timerIndicates whether to check the scheduled task
• singal_no: every timereactorbecausefdWhen ready to return,reactorWill check thissingal_noIf the value is not empty, the corresponding signal callback function will be called
• disable_acceptMark whether to accept the new connection, this is only the main linkreactorWill be set to 0, otherreactorThreads don’t need to accept new connections, they just need to accept data
• check_signalfdIndicates whether it needs to be checkedsignalfd
• threadUsed to mark whether the currentreactorMultithreading mode or multiprocessing mode is generally used
• timeout_msecUsed to record everyreactor->waitTimeout for
• max_socketRecordedreactorThe maximum number of connections in, andmax_connectionThe results are consistent with each other;socket_listyesreactorMultithreading mode of monitoringsocket, andconnection_listbring into correspondence with;socket_arrayyesreactorMonitoring in multiprocess modefd
• handleIs the default ready callback function,write_handleIs a write ready callback function,error_handleContains error ready callback functions
• timewheel、heartbeat_interval、last_heartbeat_timeIs heartbeat detection, dedicated to eliminate idle connections
• last_malloc_trim_timeThe time of last return to the system is recorded,swooleIt will be passed on a regular basismalloc_trimFunction returns the free memory space

struct _swReactor
{
    void *object;
    void *ptr;  //reserve

    /**
     * last signal number
     */
    int singal_no;

    uint32_t event_num;
    uint32_t max_event_num;

    uint32_t check_timer :1;
    uint32_t running :1;
    uint32_t start :1;
    uint32_t once :1;

    /**
     * disable accept new connection
     */
    uint32_t disable_accept :1;

    uint32_t check_signalfd :1;

    /**
     * multi-thread reactor, cannot realloc sockets.
     */
    uint32_t thread :1;

    /**
     * reactor->wait timeout (millisecond) or -1
     */
    int32_t timeout_msec;

    uint16_t id; //Reactor ID
    uint16_t flag; //flag

    uint32_t max_socket;

#ifdef SW_USE_MALLOC_TRIM
    time_t last_malloc_trim_time;
#endif

#ifdef SW_USE_TIMEWHEEL
    swTimeWheel *timewheel;
    uint16_t heartbeat_interval;
    time_t last_heartbeat_time;
#endif

    /**
     * for thread
     */
    swConnection *socket_list;

    /**
     * for process
     */
    swArray *socket_array;

    swReactor_ handle handle[SW_ MAX_ Fdtype]; // default event
    swReactor_ handle write_ handle[SW_ MAX_ Fdtype]; // extended event 1 (usually write event)
    swReactor_ handle error_ handle[SW_ MAX_ Fdtype]; // extended event 2 (generally error event, such as socket closing)

    int (*add)(swReactor *, int fd, int fdtype);
    int (*set)(swReactor *, int fd, int fdtype);
    int (*del)(swReactor *, int fd);
    int (*wait)(swReactor *, struct timeval *);
    void (*free)(swReactor *);

    int (*setHandle)(swReactor *, int fdtype, swReactor_handle);
    swDefer_callback *defer_callback_list;
    swDefer_callback idle_task;
    swDefer_callback future_task;

    void (*onTimeout)(swReactor *);
    void (*onFinish)(swReactor *);
    void (*onBegin)(swReactor *);

    void (*enable_accept)(swReactor *);
    int (*can_exit)(swReactor *);

    int (*write)(swReactor *, int, void *, int);
    int (*close)(swReactor *, int);
    int (*defer)(swReactor *, swCallback, void *);
};

reactorCreation of

• reactorThe creation of is mainly calledswReactorEpoll_createfunction
• setHandleFunction is for listeningfdSet callback function, including read ready, write ready, error
• onFinishIt’s every callepollAfter the function returns, the callback function is invoked after processing the concrete logic.
• onTimeoutIt’s every callepollCallback function after function timeout
• writeFunction is to usereactortowardssocketInterface for sending data
• deferFunction to adddefer_callback_listThe member variable is the list of callback functions,epollFunction timeouts andonFinishIt’s going to cycledefer_callback_listCallback function inside
• socket_arrayIt’s listeningfdlist

int swReactor_create(swReactor *reactor, int max_event)
{
    int ret;
    bzero(reactor, sizeof(swReactor));

#ifdef HAVE_EPOLL
    ret = swReactorEpoll_create(reactor, max_event);

    reactor->running = 1;

    reactor->setHandle = swReactor_setHandle;

    reactor->onFinish = swReactor_onFinish;
    reactor->onTimeout = swReactor_onTimeout;

    reactor->write = swReactor_write;
    reactor->defer = swReactor_defer;
    reactor->close = swReactor_close;

    reactor->socket_array = swArray_new(1024, sizeof(swConnection));
    if (!reactor->socket_array)
    {
        swWarn("create socket array failed.");
        return SW_ERR;
    }

    return ret;
}

reactorFunction of

reactorSet file ready callback functionswReactor_setHandle

• reactorSet infdIt consists of two partsswFd_type, which indicates the type of file descriptorswEvent_typeIdentifies the read and write events of interest to the file descriptor

enum swFd_type
{
    SW_FD_TCP             = 0, //tcp socket
    SW_FD_LISTEN          = 1, //server socket
    SW_FD_CLOSE           = 2, //socket closed
    SW_FD_ERROR           = 3, //socket error
    SW_FD_UDP             = 4, //udp socket
    SW_FD_PIPE            = 5, //pipe
    SW_FD_STREAM          = 6, //stream socket
    SW_FD_WRITE           = 7, //fd can write
    SW_FD_TIMER           = 8, //timer fd
    SW_FD_AIO             = 9, //linux native aio
    SW_FD_SIGNAL          = 11, //signalfd
    SW_FD_DNS_RESOLVER    = 12, //dns resolver
    SW_FD_INOTIFY         = 13, //server socket
    SW_FD_USER            = 15, //SW_FD_USER or SW_FD_USER+n: for custom event
    SW_FD_STREAM_CLIENT   = 16, //swClient stream
    SW_FD_DGRAM_CLIENT    = 17, //swClient dgram
};

enum swEvent_type
{
    SW_EVENT_DEAULT = 256,
    SW_EVENT_READ = 1u << 9,
    SW_EVENT_WRITE = 1u << 10,
    SW_EVENT_ERROR = 1u << 11,
    SW_EVENT_ONCE = 1u << 12,
};

• swReactor_fdtypeUsed to extract data from a file descriptorswFd_type, that is, the type of file descriptor:

static sw_inline int swReactor_fdtype(int fdtype)
{
    return fdtype & (~SW_EVENT_READ) & (~SW_EVENT_WRITE) & (~SW_EVENT_ERROR);
}

• swReactor_event_read、swReactor_event_write、swReactor_event_errorThese three functions are related to each otherswFd_typeInstead, read and write events are extracted from the file descriptor

static sw_inline int swReactor_event_read(int fdtype)
{
    return (fdtype < SW_EVENT_DEAULT) || (fdtype & SW_EVENT_READ);
}

static sw_inline int swReactor_event_write(int fdtype)
{
    return fdtype & SW_EVENT_WRITE;
}

static sw_inline int swReactor_event_error(int fdtype)
{
    return fdtype & SW_EVENT_ERROR;
}

• swReactor_setHandleUsed to create a file descriptor for a file_fdtypeSet the callback function of read ready and write ready

int swReactor_setHandle(swReactor *reactor, int _fdtype, swReactor_handle handle)
{
    int fdtype = swReactor_fdtype(_fdtype);

    if (fdtype >= SW_MAX_FDTYPE)
    {
        swWarn("fdtype > SW_MAX_FDTYPE[%d]", SW_MAX_FDTYPE);
        return SW_ERR;
    }

    if (swReactor_event_read(_fdtype))
    {
        reactor->handle[fdtype] = handle;
    }
    else if (swReactor_event_write(_fdtype))
    {
        reactor->write_handle[fdtype] = handle;
    }
    else if (swReactor_event_error(_fdtype))
    {
        reactor->error_handle[fdtype] = handle;
    }
    else
    {
        swWarn("unknow fdtype");
        return SW_ERR;
    }

    return SW_OK;
}

reactoradd todeferfunction

• deferThe function is called every time the event loop ends or times out
• swReactor_deferThe function will bedefer_callback_listAdd a new callback function

static int swReactor_defer(swReactor *reactor, swCallback callback, void *data)
{
    swDefer_callback *cb = sw_malloc(sizeof(swDefer_callback));
    if (!cb)
    {
        swWarn("malloc(%ld) failed.", sizeof(swDefer_callback));
        return SW_ERR;
    }
    cb->callback = callback;
    cb->data = data;
    LL_APPEND(reactor->defer_callback_list, cb);
    return SW_OK;
}

reactorTimeout callback function

epollIf it does not return within the set time, it will return automatically. At this time, the timeout callback function will be called

static void swReactor_onTimeout(swReactor *reactor)
{
    swReactor_onTimeout_and_Finish(reactor);

    if (reactor->disable_accept)
    {
        reactor->enable_accept(reactor);
        reactor->disable_accept = 0;
    }
}

• swReactor_onTimeout_and_FinishFunction is used in the event of a timeoutfinishAnd so on
• This function will first check whether there is a timed task, if there is a timed task, it will be calledswTimer_selectExecute callback function
• The next step is to execute thedefer_callback_listMultiple callback functions of thelistIt’s a pre-defined needdeferFunctions executed
• idle_taskyesEventLoopFunction called at the end of each round of event loop used in.
• If the currentreactorCurrently inworkProcess, then callswWorker_try_to_exitFunctionevent_numIs it 0? If it is 0, then set itrunning0, stop waiting for the event to be ready
• If the currentSwooleG.servIt is empty,swReactor_emptyFunction is used to determine the currentreactorIs there any event listening? If not, it will be setrunningIs 0
• Judge whether the current time can be calledmalloc_trimFree free memory, if more thanSW_MALLOC_TRIM_INTERVAL, updatelast_malloc_trim_timeAnd callmalloc_trim

static void swReactor_onTimeout_and_Finish(swReactor *reactor)
{
    //check timer
    if (reactor->check_timer)
    {
        swTimer_select(&SwooleG.timer);
    }
    //defer callback
    swDefer_callback *cb, *tmp;
    swDefer_callback *defer_callback_list = reactor->defer_callback_list;
    reactor->defer_callback_list = NULL;
    LL_FOREACH(defer_callback_list, cb)
    {
        cb->callback(cb->data);
    }
    LL_FOREACH_SAFE(defer_callback_list, cb, tmp)
    {
        sw_free(cb);
    }
    //callback at the end
    if (reactor->idle_task.callback)
    {
        reactor->idle_task.callback(reactor->idle_task.data);
    }
#ifdef SW_COROUTINE
    //coro timeout
    if (!swIsMaster())
    {
        coro_handle_timeout();
    }
#endif
    //server worker
    swWorker *worker = SwooleWG.worker;
    if (worker != NULL)
    {
        if (SwooleWG.wait_exit == 1)
        {
            swWorker_try_to_exit();
        }
    }
    //not server, the event loop is empty
    if (SwooleG.serv == NULL && swReactor_empty(reactor))
    {
        reactor->running = 0;
    }

#ifdef SW_USE_MALLOC_TRIM
    if (SwooleG.serv && reactor->last_malloc_trim_time < SwooleG.serv->gs->now - SW_MALLOC_TRIM_INTERVAL)
    {
        malloc_trim(SW_MALLOC_TRIM_PAD);
        reactor->last_malloc_trim_time = SwooleG.serv->gs->now;
    }
#endif
}

• swReactor_emptyUsed to determine the currentreactorIs there any event that needs to be monitored
• It can be seen from the function that if the task is timedtimerIf there are waiting tasks in it, you can return false
• event_numIf it is 0, it can return true to end the event loop
• For a coroutine, you need to callcan_exitTo determine whether the event loop can be exited

int swReactor_empty(swReactor *reactor)
{
    //timer
    if (SwooleG.timer.num > 0)
    {
        return SW_FALSE;
    }

    int empty = SW_FALSE;
    //thread pool
    if (SwooleAIO.init && reactor->event_num == 1 && SwooleAIO.task_num == 0)
    {
        empty = SW_TRUE;
    }
    //no event
    else if (reactor->event_num == 0)
    {
        empty = SW_TRUE;
    }
    //coroutine
    if (empty && reactor->can_exit && reactor->can_exit(reactor))
    {
        empty = SW_TRUE;
    }
    return empty;
}

reactorEvent loop end function

• Called after each event looponFinishfunction
• This function mainly calls theswReactor_onTimeout_and_FinishBefore that, we will also check whether there is a signal trigger during the event cycle

static void swReactor_onFinish(swReactor *reactor)
{
    //check signal
    if (reactor->singal_no)
    {
        swSignal_callback(reactor->singal_no);
        reactor->singal_no = 0;
    }
    swReactor_onTimeout_and_Finish(reactor);
}

reactorEvent loop close function

• Be asocketWhen closing, thecloseFunction, the corresponding callback function isswReactor_close
• This function is used to release theswConnectionInternal application of memory and callcloseFunction to close the connection

int swReactor_close(swReactor *reactor, int fd)
{
    swConnection *socket = swReactor_get(reactor, fd);
    if (socket->out_buffer)
    {
        swBuffer_free(socket->out_buffer);
    }
    if (socket->in_buffer)
    {
        swBuffer_free(socket->in_buffer);
    }
    if (socket->websocket_buffer)
    {
        swString_free(socket->websocket_buffer);
    }
    bzero(socket, sizeof(swConnection));
    socket->removed = 1;
    swTraceLog(SW_TRACE_CLOSE, "fd=%d.", fd);
    return close(fd);
}

• swReactor_getUsed fromreactorAccording to the file descriptorswConnectionObject, due toswooleIt is generally usedreactorMultithreading mode, so basically only executionreturn &reactor->socket_list[fd];This is a sentence.
• socket_listThis list is related toconnection_listKeeping consistent is prior to applying the size formax_connectionWhat is the type ofswConnectionArray of
• socket_listPart of the data in is already connectedswConnectionSome of the objects are just emptyswConnectionAt this timeswConnection->fdIs 0

static sw_inline swConnection* swReactor_get(swReactor *reactor, int fd)
{
    if (reactor->thread)
    {
        return &reactor->socket_list[fd];
    }
    swConnection *socket = (swConnection*) swArray_alloc(reactor->socket_array, fd);
    if (socket == NULL)
    {
        return NULL;
    }
    if (!socket->active)
    {
        socket->fd = fd;
    }
    return socket;
}

reactorData writing for

• If you want to be right with asocketWriting data can’t be called directlysendFunction, because this function may be interrupted by signal (Eintr), may be temporarily unavailable (eagain), may write only part of the data, or may write successfully. Therefore,reactorA function is defined to deal with the logic of writing data
• First of all, we should make use of itswReactor_getTake out the correspondingswConnectionobject
• If the object is removedfdIt’s 0, which means thisfdThe file descriptor is not in thereactorWe’re listening inside

• If thissocketOfout_bufferIf it is empty, try to use it firstswConnection_sendfunction callsendFunction to see if all data can be sent directly

  • If you returnEINTR, then it means that it is interrupted by the signal and can be sent again
  • If you returnEAGAINThen it means that at this timesocketIt is temporarily unavailable. At this time, you need tofdThe write ready state of the file descriptor is added to thereactorAnd then copy the data to theout_bufferZhongqu
  • If the amount of data returned is less thann, indicating that only a part has been written. At this time, you need to copy the part that has not been written toout_bufferZhongqu
• Ifout_bufferIf it is not empty, it means thatsocketIf it is not writable, copy the data toout_bufferGo to China and waitreactorAfter monitoring to write ready, put theout_bufferSend it out.
• If at this timeout_bufferIf there is not enough storage space, it is necessary toswYieldLet the process sleep for a while and waitfdWrite ready status of

int swReactor_write(swReactor *reactor, int fd, void *buf, int n)
{
    int ret;
    swConnection *socket = swReactor_get(reactor, fd);
    swBuffer *buffer = socket->out_buffer;

    if (socket->fd == 0)
    {
        socket->fd = fd;
    }

    if (socket->buffer_size == 0)
    {
        socket->buffer_size = SwooleG.socket_buffer_size;
    }

    if (socket->nonblock == 0)
    {
        swoole_fcntl_set_option(fd, 1, -1);
        socket->nonblock = 1;
    }

    if (n > socket->buffer_size)
    {
        swoole_error_log(SW_LOG_WARNING, SW_ERROR_PACKAGE_LENGTH_TOO_LARGE, "data is too large, cannot exceed buffer size.");
        return SW_ERR;
    }

    if (swBuffer_empty(buffer))
    {
        if (socket->ssl_send)
        {
            goto do_buffer;
        }

        do_send:
        ret = swConnection_send(socket, buf, n, 0);

        if (ret > 0)
        {
            if (n == ret)
            {
                return ret;
            }
            else
            {
                buf += ret;
                n -= ret;
                goto do_buffer;
            }
        }
#ifdef HAVE_KQUEUE
        else if (errno == EAGAIN || errno == ENOBUFS)
#else
        else if (errno == EAGAIN)
#endif
        {
            do_buffer:
            if (!socket->out_buffer)
            {
                buffer = swBuffer_new(sizeof(swEventData));
                if (!buffer)
                {
                    swWarn("create worker buffer failed.");
                    return SW_ERR;
                }
                socket->out_buffer = buffer;
            }

            socket->events |= SW_EVENT_WRITE;

            if (socket->events & SW_EVENT_READ)
            {
                if (reactor->set(reactor, fd, socket->fdtype | socket->events) < 0)
                {
                    swSysError("reactor->set(%d, SW_EVENT_WRITE) failed.", fd);
                }
            }
            else
            {
                if (reactor->add(reactor, fd, socket->fdtype | SW_EVENT_WRITE) < 0)
                {
                    swSysError("reactor->add(%d, SW_EVENT_WRITE) failed.", fd);
                }
            }

            goto append_buffer;
        }
        else if (errno == EINTR)
        {
            goto do_send;
        }
        else
        {
            SwooleG.error = errno;
            return SW_ERR;
        }
    }
    else
    {
        append_buffer: if (buffer->length > socket->buffer_size)
        {
            if (socket->dontwait)
            {
                SwooleG.error = SW_ERROR_OUTPUT_BUFFER_OVERFLOW;
                return SW_ERR;
            }
            else
            {
                swoole_error_log(SW_LOG_WARNING, SW_ERROR_OUTPUT_BUFFER_OVERFLOW, "socket#%d output buffer overflow.", fd);
                swYield();
                swSocket_wait(fd, SW_SOCKET_OVERFLOW_WAIT, SW_EVENT_WRITE);
            }
        }

        if (swBuffer_append(buffer, buf, n) < 0)
        {
            return SW_ERR;
        }
    }
    return SW_OK;
}