本節繼續介紹PostgreSQL的後臺程式walsender,重點介紹的是呼叫棧中的函式WalSndLoop->WaitLatchOrSocket->WaitEventSetWait->WaitEventSetWaitBlock.
呼叫棧如下:
```
(gdb) bt
#0  0x00007fb6e6390903 in __epoll_wait_nocancel () from /lib64/libc.so.6
#1  0x000000000088e668 in WaitEventSetWaitBlock (set=0x10ac808, cur_timeout=29999, occurred_events=0x7ffd634441b0, 
    nevents=1) at latch.c:1048
#2  0x000000000088e543 in WaitEventSetWait (set=0x10ac808, timeout=29999, occurred_events=0x7ffd634441b0, nevents=1, 
    wait_event_info=83886092) at latch.c:1000
#3  0x000000000088dcec in WaitLatchOrSocket (latch=0x7fb6dcbfc4d4, wakeEvents=27, sock=10, timeout=29999, 
    wait_event_info=83886092) at latch.c:385
#4  0x000000000085405b in WalSndLoop (send_data=0x8547fe <xlogsendphysical>) at walsender.c:2229
#5  0x0000000000851c93 in StartReplication (cmd=0x10ab750) at walsender.c:684
#6  0x00000000008532f0 in exec_replication_command (cmd_string=0x101dd78 "START_REPLICATION 0/5D000000 TIMELINE 16")
    at walsender.c:1539
#7  0x00000000008c0170 in PostgresMain (argc=1, argv=0x1049cb8, dbname=0x1049ba8 "", username=0x1049b80 "replicator")
    at postgres.c:4178
#8  0x000000000081e06c in BackendRun (port=0x103fb50) at postmaster.c:4361
#9  0x000000000081d7df in BackendStartup (port=0x103fb50) at postmaster.c:4033
#10 0x0000000000819bd9 in ServerLoop () at postmaster.c:1706
#11 0x000000000081948f in PostmasterMain (argc=1, argv=0x1018a50) at postmaster.c:1379
#12 0x0000000000742931 in main (argc=1, argv=0x1018a50) at main.c:228
```

### 一、資料結構
**WaitEvent**
WaitEvent等待事件結構體
```
typedef struct WaitEvent
{
    //在event資料結構體中的位置
    int         pos;            /* position in the event data structure */
    //已觸發的事件
    uint32      events;         /* triggered events */
    //與該事件相關的socket fd(檔案描述符)
    pgsocket    fd;             /* socket fd associated with event */
    //在AddWaitEventToSet中提供的指標
    void       *user_data;      /* pointer provided in AddWaitEventToSet */
#ifdef WIN32
    //WIN32 是否已重置?
    bool        reset;          /* Is reset of the event required? */
#endif
} WaitEvent;

```

**WaitEventSet**
WaitEventSet等待事件集
```

/* typedef in latch.h */
//latch.h中定義的型別
struct WaitEventSet
{
    //註冊的事件數
    int         nevents;        /* number of registered events */
    //該集合中最大的事件數
    int         nevents_space;  /* maximum number of events in this set */

/*
     * Array, of nevents_space length, storing the definition of events this
     * set is waiting for.
     * 陣列,長度為nevents_space,儲存該集合中等待的事件定義
     */
    WaitEvent  *events;

/*
     * If WL_LATCH_SET is specified in any wait event, latch is a pointer to
     * said latch, and latch_pos the offset in the ->events array. This is
     * useful because we check the state of the latch before performing doing
     * syscalls related to waiting.
     * 如在等待事件中指定了WL_LATCH_SET,latch指向所述的latch,
     *   latch_pos是events陣列中的偏移.
     * 這很有用因為在執行與等待相關syscalls前可以檢查latch的狀態.
     */
    Latch      *latch;
    int         latch_pos;

/*
     * WL_EXIT_ON_PM_DEATH is converted to WL_POSTMASTER_DEATH, but this flag
     * is set so that we'll exit immediately if postmaster death is detected,
     * instead of returning.
     * WL_EXIT_ON_PM_DEATH轉換為WL_POSTMASTER_DEATH,
     *   但該標誌設定以便一旦檢測到postmaster掛掉就可以馬上退出而不是返回.
     */
    bool        exit_on_postmaster_death;

#if defined(WAIT_USE_EPOLL)
    //使用epoll
    int         epoll_fd;
    /* epoll_wait returns events in a user provided arrays, allocate once */
    //epoll_wait在使用者提供的陣列中返回事件,只需要分配一次
    struct epoll_event *epoll_ret_events;
#elif defined(WAIT_USE_POLL)
    //使用poll
    /* poll expects events to be waited on every poll() call, prepare once */
    //poll期望事件在每一次poll()呼叫時等待,只需要準備一次
    struct pollfd *pollfds;
#elif defined(WAIT_USE_WIN32)
    //WIN32
    /*
     * Array of windows events. The first element always contains
     * pgwin32_signal_event, so the remaining elements are offset by one (i.e.
     * event->pos + 1).
     * windows事件陣列.
     * 第一個元素通常儲存的是pgwin32_signal_event,
     *   因此遺留的元素是相對該元素的偏移(比如event->pos + 1)
     */
    HANDLE     *handles;
#endif
};

```

### 二、原始碼解讀
**WalSndLoop**
透過Copy處理WAL流資料的walsender程式主迴圈.
其主要邏輯如下:
1.獲取時間戳,設定相關標記
2.進入迴圈
2.1重置MyLatch,檢查中斷
2.2處理最近接收的請求或訊號
2.3檢查客戶端輸入
2.4如果從客戶端接收到CopyDone訊號,並且buffer為空,則退出迴圈
2.5執行相關處理並設定WalSndCaughtUp變數
2.6如WalSndCaughtUp為T並且沒有掛起待處理的資料
2.6.1設定狀態並判斷接收的訊號執行相關處理
2.7檢查複製是否超時以及是否需要傳送Keepalive
2.8如處於CaughtUp狀態並且仍未完成Streaming或者存在掛起的資料
   則設定等待時間,執行WaitLatchOrSocket

```

/* Main loop of walsender process that streams the WAL over Copy messages. */
//透過Copy處理WAL流資料的walsender程式主迴圈
static void
WalSndLoop(WalSndSendDataCallback send_data)
{
    /*
     * Initialize the last reply timestamp. That enables timeout processing
     * from hereon.
     * 初始化最後一個響應時間戳.
     * 在這裡開始,啟用超時處理.
     */
    last_reply_timestamp = GetCurrentTimestamp();
    waiting_for_ping_response = false;

/*
     * Loop until we reach the end of this timeline or the client requests to
     * stop streaming.
     * 迴圈,直至達到時間線的末尾或者客戶端請求停止streaming.
     */
    for (;;)
    {
        /* Clear any already-pending wakeups */
        //清除所有已掛起的wakeups.
        ResetLatch(MyLatch);
        //檢查中斷
        CHECK_FOR_INTERRUPTS();

/* Process any requests or signals received recently */
        //處理最近接收的請求或訊號
        if (ConfigReloadPending)
        {
            ConfigReloadPending = false;
            ProcessConfigFile(PGC_SIGHUP);
            SyncRepInitConfig();
        }

/* Check for input from the client */
        //檢查客戶端輸入
        ProcessRepliesIfAny();

/*
         * If we have received CopyDone from the client, sent CopyDone
         * ourselves, and the output buffer is empty, it's time to exit
         * streaming.
         * 如果從客戶端接收到CopyDone訊號,自行傳送CopyDone,
         *   同時輸出buffer是空的,是時候退出streaming了.
         */
        if (streamingDoneReceiving && streamingDoneSending &&
            !pq_is_send_pending())
            //跳出迴圈
            break;

/*
         * If we don't have any pending data in the output buffer, try to send
         * some more.  If there is some, we don't bother to call send_data
         * again until we've flushed it ... but we'd better assume we are not
         * caught up.
         * 如果在輸出緩衝區中沒有掛起的資料,嘗試傳送更多的資料.
         * 如果已存在資料,那麼在重新整理之前不需要再次呼叫send_data ...
         *   但我們最好假定沒有捕獲這些資料.
         */
        if (!pq_is_send_pending())
            //傳送資料
            send_data();
        else
            WalSndCaughtUp = false;

/* Try to flush pending output to the client */
        //嘗試重新整理掛起的輸出到客戶端
        if (pq_flush_if_writable() != 0)
            WalSndShutdown();

/* If nothing remains to be sent right now ... */
        //如果現在沒有遺留資料
        if (WalSndCaughtUp && !pq_is_send_pending())
        {
            /*
             * If we're in catchup state, move to streaming.  This is an
             * important state change for users to know about, since before
             * this point data loss might occur if the primary dies and we
             * need to failover to the standby. The state change is also
             * important for synchronous replication, since commits that
             * started to wait at that point might wait for some time.
             * 如果處於catchup狀態,切換到streaming.
             * 對於客戶端來說,讓它們知道狀態變換是很重要的,
             *   因為在此時點前如果主節點崩潰需要切換到備機時資料可能會出現丟失.
             * 對於同步複製而已,這個狀態變更也很重要,
             *   因為在這個時間點開始的commits操作可能會等待一段時間.
             */
            if (MyWalSnd->state == WALSNDSTATE_CATCHUP)
            {
                ereport(DEBUG1,
                        (errmsg("\"%s\" has now caught up with upstream server",
                                application_name)));
                //設定狀態
                WalSndSetState(WALSNDSTATE_STREAMING);
            }

/*
             * When SIGUSR2 arrives, we send any outstanding logs up to the
             * shutdown checkpoint record (i.e., the latest record), wait for
             * them to be replicated to the standby, and exit. This may be a
             * normal termination at shutdown, or a promotion, the walsender
             * is not sure which.
             * 在接收到SIGUSR2訊號時,我們將所有未完成的日誌傳送到shutdown checkpoint record
             *   (比如最後一條記錄),等待這些日誌複製到standby節點,執行完畢則退出.
             * 這可能是在關閉過程中的正常終止,也可能是升級,但walsender無法確認.
             */
            if (got_SIGUSR2)
                WalSndDone(send_data);
        }

/* Check for replication timeout. */
        //檢查複製超時
        WalSndCheckTimeOut();

/* Send keepalive if the time has come */
        //是時候傳送keepalive了
        WalSndKeepaliveIfNecessary();

/*
         * We don't block if not caught up, unless there is unsent data
         * pending in which case we'd better block until the socket is
         * write-ready.  This test is only needed for the case where the
         * send_data callback handled a subset of the available data but then
         * pq_flush_if_writable flushed it all --- we should immediately try
         * to send more.
         * 如果沒有caught up則不要阻塞,除非有未傳送的資料掛起,
         *   在這種情況下,程式最好阻塞直至socket write-ready.
         * 這個測試僅在send_data回撥函式處理可用資料的一部分但pq_flush_if_writable進行全部重新整理時需要.
         * 這時候我們應該馬上嘗試傳送更多的資料.
         */
        if ((WalSndCaughtUp && !streamingDoneSending) || pq_is_send_pending())
        {
            long        sleeptime;
            int         wakeEvents;

wakeEvents = WL_LATCH_SET | WL_EXIT_ON_PM_DEATH | WL_TIMEOUT |
                         WL_SOCKET_READABLE;

/*
             * Use fresh timestamp, not last_processed, to reduce the chance
             * of reaching wal_sender_timeout before sending a keepalive.
             * 在傳送keepalive前,
             *   使用重新整理時間戳而不是last_processed來減少超過wal_sender_timeout
             */
            sleeptime = WalSndComputeSleeptime(GetCurrentTimestamp());

if (pq_is_send_pending())
                wakeEvents |= WL_SOCKET_WRITEABLE;

/* Sleep until something happens or we time out */
            //休眠直至某些事情傳送或者超時
            (void) WaitLatchOrSocket(MyLatch, wakeEvents,
                                     MyProcPort->sock, sleeptime,
                                     WAIT_EVENT_WAL_SENDER_MAIN);
        }
    }
    return;
}

```

**WaitLatchOrSocket**
該函式與WaitLatch類似,但額外有一個socket引數,用於WL_SOCKET_*.
在WaitEventSet中新增等待事件,呼叫函式WaitEventSetWait,等待事件的發生或者超時.
```

/*
 * Like WaitLatch, but with an extra socket argument for WL_SOCKET_*
 * conditions.
 * 與WaitLatch類似,但額外有一個socket引數,用於WL_SOCKET_*
 *
 * When waiting on a socket, EOF and error conditions always cause the socket
 * to be reported as readable/writable/connected, so that the caller can deal
 * with the condition.
 * 在寫入socket時,EOF和錯誤的條件通常會導致socket被視為可讀寫和已連線,因此呼叫者可處理該條件.
 *
 * wakeEvents must include either WL_EXIT_ON_PM_DEATH for automatic exit
 * if the postmaster dies or WL_POSTMASTER_DEATH for a flag set in the
 * return value if the postmaster dies.  The latter is useful for rare cases
 * where some behavior other than immediate exit is needed.
 * wakeEvents必須包含WL_EXIT_ON_PM_DEATH用於在postmaster崩潰時的自動退出
 *   或者WL_POSTMASTER_DEATH用於在postmaster崩潰時返回標記.
 * 後者用於以下極少數的情況:需要執行某些動作而不是馬上退出.
 *
 * NB: These days this is just a wrapper around the WaitEventSet API. When
 * using a latch very frequently, consider creating a longer living
 * WaitEventSet instead; that's more efficient.
 * 注意:現在,這只是WaitEventSet API的一個包裝器.
 * 在非常頻繁的使用latch的情況下,應考慮建立一個長生命週期的WaitEventSet,這會更有效率.
 */
int
WaitLatchOrSocket(Latch *latch, int wakeEvents, pgsocket sock,
                  long timeout, uint32 wait_event_info)
{
    int         ret = 0;
    int         rc;
    WaitEvent   event;
    //建立WaitEventSet,效能上的考慮
    WaitEventSet *set = CreateWaitEventSet(CurrentMemoryContext, 3);

if (wakeEvents & WL_TIMEOUT)
        //超時
        Assert(timeout >= 0);
    else
        timeout = -1;

if (wakeEvents & WL_LATCH_SET)
        //設定了LATCH
        AddWaitEventToSet(set, WL_LATCH_SET, PGINVALID_SOCKET,
                          latch, NULL);

/* Postmaster-managed callers must handle postmaster death somehow. */
    //Postmaster-managed的呼叫者必須處理postmaster崩潰的情況.
    Assert(!IsUnderPostmaster ||
           (wakeEvents & WL_EXIT_ON_PM_DEATH) ||
           (wakeEvents & WL_POSTMASTER_DEATH));

if ((wakeEvents & WL_POSTMASTER_DEATH) && IsUnderPostmaster)
        AddWaitEventToSet(set, WL_POSTMASTER_DEATH, PGINVALID_SOCKET,
                          NULL, NULL);

if ((wakeEvents & WL_EXIT_ON_PM_DEATH) && IsUnderPostmaster)
        AddWaitEventToSet(set, WL_EXIT_ON_PM_DEATH, PGINVALID_SOCKET,
                          NULL, NULL);

if (wakeEvents & WL_SOCKET_MASK)
    {
        int         ev;

ev = wakeEvents & WL_SOCKET_MASK;
        AddWaitEventToSet(set, ev, sock, NULL, NULL);
    }

rc = WaitEventSetWait(set, timeout, &event, 1, wait_event_info);

if (rc == 0)
        ret |= WL_TIMEOUT;
    else
    {
        ret |= event.events & (WL_LATCH_SET |
                               WL_POSTMASTER_DEATH |
                               WL_SOCKET_MASK);
    }
    //釋放資源
    FreeWaitEventSet(set);

return ret;
}

```

**WaitEventSetWait**
等待加入到等待事件集合中的事件發生,或者直至超時.
迴圈等待,呼叫WaitEventSetWaitBlock,直至有事件發生.
```

/*
 * Wait for events added to the set to happen, or until the timeout is
 * reached.  At most nevents occurred events are returned.
 * 等待加入到等待事件集合中的事件發生,或者直至超時.
 * 在大多數情況下,返回nevenets已發生事件.
 *
 * If timeout = -1, block until an event occurs; if 0, check sockets for
 * readiness, but don't block; if > 0, block for at most timeout milliseconds.
 * 如果timeout = -1,阻塞直至事件發生.
 * 如果timeout = 0,檢查sockets用於讀取,但不阻塞.
 * 如果timeout > 0,阻塞直至超時.
 *
 * Returns the number of events occurred, or 0 if the timeout was reached.
 * 返回出現的實踐計數,如出現超時則返回0
 *
 * Returned events will have the fd, pos, user_data fields set to the
 * values associated with the registered event.
 * 返回的事件會包含fd,pos,user_data欄位資訊用於與註冊事件關聯.
 */
int
WaitEventSetWait(WaitEventSet *set, long timeout,
                 WaitEvent *occurred_events, int nevents,
                 uint32 wait_event_info)
{
    int         returned_events = 0;
    instr_time  start_time;
    instr_time  cur_time;
    long        cur_timeout = -1;

Assert(nevents > 0);

/*
     * Initialize timeout if requested.  We must record the current time so
     * that we can determine the remaining timeout if interrupted.
     * 如請求則初始化超時處理.
     * 必須記錄當前時間以便可以確定在中斷髮送時剩餘時間.
     */
    if (timeout >= 0)
    {
        INSTR_TIME_SET_CURRENT(start_time);
        Assert(timeout >= 0 && timeout <= INT_MAX);
        cur_timeout = timeout;
    }

pgstat_report_wait_start(wait_event_info);

#ifndef WIN32
    waiting = true;
#else
    /* Ensure that signals are serviced even if latch is already set */
    pgwin32_dispatch_queued_signals();
#endif
    while (returned_events == 0)//未有事件傳送
    {
        int         rc;

/*
         * Check if the latch is set already. If so, leave the loop
         * immediately, avoid blocking again. We don't attempt to report any
         * other events that might also be satisfied.
         * 檢查latch是否已設定.
         * 如設定,則馬上退出迴圈,避免再次阻塞.
         * 不需要嘗試報告其他可能已滿足條件的事件.
         *
         * If someone sets the latch between this and the
         * WaitEventSetWaitBlock() below, the setter will write a byte to the
         * pipe (or signal us and the signal handler will do that), and the
         * readiness routine will return immediately.
         * 如果某程式在這裡和接下來的WaitEventSetWaitBlock()之間設定了latch,
         *   設定器會寫入一個位元組到pipe中(或者發訊號給我們,訊號處理器會處理該訊號),
         *   readiness例程會馬上返回.
         *
         * On unix, If there's a pending byte in the self pipe, we'll notice
         * whenever blocking. Only clearing the pipe in that case avoids
         * having to drain it every time WaitLatchOrSocket() is used. Should
         * the pipe-buffer fill up we're still ok, because the pipe is in
         * nonblocking mode. It's unlikely for that to happen, because the
         * self pipe isn't filled unless we're blocking (waiting = true), or
         * from inside a signal handler in latch_sigusr1_handler().
         * 在Unix平臺,如果在自己的pipe中存在掛起的位元組,則會注意到何時出現的阻塞.
         * 在這種情況下只有清理pipe中的資料以避免每次使用WaitLatchOrSocket()時都必須將其清除.
         * pipe-buffer是否填滿程式是沒有問題的,因為pipe處於非阻塞模式.
         * 這不同於事件發生,因為self pipe直到阻塞(waiting = true),
         *   或者在latch_sigusr1_handler()中的訊號控制器中不會填充資料.
         *
         * On windows, we'll also notice if there's a pending event for the
         * latch when blocking, but there's no danger of anything filling up,
         * as "Setting an event that is already set has no effect.".
         * 在Windows平臺,我們同時會注意到對於阻塞時,latch上是否存在掛起的事件,
         *   但對於填充來說沒有任何的問題,這可以視為"設定已設定的事件是不會有問題的".
         *
         * Note: we assume that the kernel calls involved in latch management
         * will provide adequate synchronization on machines with weak memory
         * ordering, so that we cannot miss seeing is_set if a notification
         * has already been queued.
         * 注意:假定涉及latch管理的核心呼叫將在記憶體順序較差的機器上提供足夠的同步保護,
         *   這樣在通知已入佇列時不會錯過is_set標記.
         */
        if (set->latch && set->latch->is_set)
        {
            //已設定latch
            occurred_events->fd = PGINVALID_SOCKET;
            occurred_events->pos = set->latch_pos;
            occurred_events->user_data =
                set->events[set->latch_pos].user_data;//使用者資料
            occurred_events->events = WL_LATCH_SET;
            occurred_events++;
            returned_events++;
            //退出迴圈
            break;
        }

/*
         * Wait for events using the readiness primitive chosen at the top of
         * this file. If -1 is returned, a timeout has occurred, if 0 we have
         * to retry, everything >= 1 is the number of returned events.
         * 等待使用在該檔案最前面選擇的原始readiness方法的事件發生.
         * 如果返回-1,意味著超時發生,如為0,只能再次重試,>= 1是返回的events數量.
         */
        rc = WaitEventSetWaitBlock(set, cur_timeout,
                                   occurred_events, nevents);

if (rc == -1)
            //超時
            break;              /* timeout occurred */
        else
            //事件數
            returned_events = rc;

/* If we're not done, update cur_timeout for next iteration */
        //如為完成,更新cur_timeout已備下次迴圈迭代
        if (returned_events == 0 && timeout >= 0)
        {
            INSTR_TIME_SET_CURRENT(cur_time);
            INSTR_TIME_SUBTRACT(cur_time, start_time);
            cur_timeout = timeout - (long) INSTR_TIME_GET_MILLISEC(cur_time);
            if (cur_timeout <= 0)
                break;
        }
    }
#ifndef WIN32
    waiting = false;
#endif

pgstat_report_wait_end();

return returned_events;
}

```

**WaitEventSetWaitBlock**
使用linux's epoll_wait(2)等待.
呼叫epoll_wait方法,如發生事件,遍歷events執行相關處理.
```
 
#if defined(WAIT_USE_EPOLL)

/*
 * Wait using linux's epoll_wait(2).
 * 使用linux's epoll_wait(2)等待.
 *
 * This is the preferable wait method, as several readiness notifications are
 * delivered, without having to iterate through all of set->events. The return
 * epoll_event struct contain a pointer to our events, making association
 * easy.
 * 這是一種更好的等待方法,隨著多個readiness通知的交付,而不需要透過迭代所有的set->events.
 * 返回的epoll_event結構體儲存了指向事件的指標,讓關聯變更簡單.
 */
static inline int
WaitEventSetWaitBlock(WaitEventSet *set, int cur_timeout,
                      WaitEvent *occurred_events, int nevents)
{
    int         returned_events = 0;
    int         rc;
    WaitEvent  *cur_event;
    struct epoll_event *cur_epoll_event;

/* Sleep */
    //休眠
    rc = epoll_wait(set->epoll_fd, set->epoll_ret_events,
                    nevents, cur_timeout);

/* Check return code */
    //檢查返回程式碼
    if (rc < 0)
    {
        /*  is okay, otherwise complain */
        //驗證EINTR是否ok,否則報錯
        if (errno != EINTR)
        {
            waiting = false;
            ereport(ERROR,
                    (errcode_for_socket_access(),
                     errmsg("epoll_wait() failed: %m")));
        }
        return 0;
    }
    else if (rc == 0)
    {
        /* timeout exceeded */
        //超時
        return -1;
    }

/*
     * At least one event occurred, iterate over the returned epoll events
     * until they're either all processed, or we've returned all the events
     * the caller desired.
     * 至少有一個事件發生了,迭代返回的epoll事件,直至全部被處理或者已返回撥用者鎖指定的所有事件
     */
    for (cur_epoll_event = set->epoll_ret_events;
         cur_epoll_event < (set->epoll_ret_events + rc) &&
         returned_events < nevents;
         cur_epoll_event++)
    {
        /* epoll's data pointer is set to the associated WaitEvent */
        //epoll的資料指標設定為關聯WaitEvent
        cur_event = (WaitEvent *) cur_epoll_event->data.ptr;

occurred_events->pos = cur_event->pos;
        occurred_events->user_data = cur_event->user_data;
        occurred_events->events = 0;

if (cur_event->events == WL_LATCH_SET &&
            cur_epoll_event->events & (EPOLLIN | EPOLLERR | EPOLLHUP))
        {
            //------------- 出現事件
            /* There's data in the self-pipe, clear it. */
            //在self-pipe中存在資料,清除之
            drainSelfPipe();

if (set->latch->is_set)
            {
                occurred_events->fd = PGINVALID_SOCKET;
                occurred_events->events = WL_LATCH_SET;
                occurred_events++;
                returned_events++;
            }
        }
        else if (cur_event->events == WL_POSTMASTER_DEATH &&
                 cur_epoll_event->events & (EPOLLIN | EPOLLERR | EPOLLHUP))
        {
            //------------- postmaster掛了
            /*
             * We expect an EPOLLHUP when the remote end is closed, but
             * because we don't expect the pipe to become readable or to have
             * any errors either, treat those cases as postmaster death, too.
             *
             * Be paranoid about a spurious event signalling the postmaster as
             * being dead.  There have been reports about that happening with
             * older primitives (select(2) to be specific), and a spurious
             * WL_POSTMASTER_DEATH event would be painful. Re-checking doesn't
             * cost much.
             */
            if (!PostmasterIsAliveInternal())
            {
                if (set->exit_on_postmaster_death)
                    proc_exit(1);
                occurred_events->fd = PGINVALID_SOCKET;
                occurred_events->events = WL_POSTMASTER_DEATH;
                occurred_events++;
                returned_events++;
            }
        }
        else if (cur_event->events & (WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE))
        {
            //--------- socket可讀寫
            Assert(cur_event->fd != PGINVALID_SOCKET);

if ((cur_event->events & WL_SOCKET_READABLE) &&
                (cur_epoll_event->events & (EPOLLIN | EPOLLERR | EPOLLHUP)))
            {
                /* data available in socket, or EOF */
                //socket已有可用資料,或者已達末尾(EOF)
                occurred_events->events |= WL_SOCKET_READABLE;
            }

if ((cur_event->events & WL_SOCKET_WRITEABLE) &&
                (cur_epoll_event->events & (EPOLLOUT | EPOLLERR | EPOLLHUP)))
            {
                /* writable, or EOF */
                //可寫或者EOF
                occurred_events->events |= WL_SOCKET_WRITEABLE;
            }

if (occurred_events->events != 0)
            {
                occurred_events->fd = cur_event->fd;
                occurred_events++;
                returned_events++;
            }
        }
    }

return returned_events;
}

#elif defined(WAIT_USE_POLL)

/*
 * Wait using poll(2).
 *
 * This allows to receive readiness notifications for several events at once,
 * but requires iterating through all of set->pollfds.
 */
static inline int
WaitEventSetWaitBlock(WaitEventSet *set, int cur_timeout,
                      WaitEvent *occurred_events, int nevents)
{
    int         returned_events = 0;
    int         rc;
    WaitEvent  *cur_event;
    struct pollfd *cur_pollfd;

/* Sleep */
    rc = poll(set->pollfds, set->nevents, (int) cur_timeout);

/* Check return code */
    if (rc < 0)
    {
        /* EINTR is okay, otherwise complain */
        if (errno != EINTR)
        {
            waiting = false;
            ereport(ERROR,
                    (errcode_for_socket_access(),
                     errmsg("poll() failed: %m")));
        }
        return 0;
    }
    else if (rc == 0)
    {
        /* timeout exceeded */
        return -1;
    }

for (cur_event = set->events, cur_pollfd = set->pollfds;
         cur_event < (set->events + set->nevents) &&
         returned_events < nevents;
         cur_event++, cur_pollfd++)
    {
        /* no activity on this FD, skip */
        if (cur_pollfd->revents == 0)
            continue;

occurred_events->pos = cur_event->pos;
        occurred_events->user_data = cur_event->user_data;
        occurred_events->events = 0;

if (cur_event->events == WL_LATCH_SET &&
            (cur_pollfd->revents & (POLLIN | POLLHUP | POLLERR | POLLNVAL)))
        {
            /* There's data in the self-pipe, clear it. */
            drainSelfPipe();

if (set->latch->is_set)
            {
                occurred_events->fd = PGINVALID_SOCKET;
                occurred_events->events = WL_LATCH_SET;
                occurred_events++;
                returned_events++;
            }
        }
        else if (cur_event->events == WL_POSTMASTER_DEATH &&
                 (cur_pollfd->revents & (POLLIN | POLLHUP | POLLERR | POLLNVAL)))
        {
            /*
             * We expect an POLLHUP when the remote end is closed, but because
             * we don't expect the pipe to become readable or to have any
             * errors either, treat those cases as postmaster death, too.
             *
             * Be paranoid about a spurious event signalling the postmaster as
             * being dead.  There have been reports about that happening with
             * older primitives (select(2) to be specific), and a spurious
             * WL_POSTMASTER_DEATH event would be painful. Re-checking doesn't
             * cost much.
             */
            if (!PostmasterIsAliveInternal())
            {
                if (set->exit_on_postmaster_death)
                    proc_exit(1);
                occurred_events->fd = PGINVALID_SOCKET;
                occurred_events->events = WL_POSTMASTER_DEATH;
                occurred_events++;
                returned_events++;
            }
        }
        else if (cur_event->events & (WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE))
        {
            int         errflags = POLLHUP | POLLERR | POLLNVAL;

Assert(cur_event->fd >= PGINVALID_SOCKET);

if ((cur_event->events & WL_SOCKET_READABLE) &&
                (cur_pollfd->revents & (POLLIN | errflags)))
            {
                /* data available in socket, or EOF */
                occurred_events->events |= WL_SOCKET_READABLE;
            }

if ((cur_event->events & WL_SOCKET_WRITEABLE) &&
                (cur_pollfd->revents & (POLLOUT | errflags)))
            {
                /* writeable, or EOF */
                occurred_events->events |= WL_SOCKET_WRITEABLE;
            }

if (occurred_events->events != 0)
            {
                occurred_events->fd = cur_event->fd;
                occurred_events++;
                returned_events++;
            }
        }
    }
    return returned_events;
}

#elif defined(WAIT_USE_WIN32)

/*
 * Wait using Windows' WaitForMultipleObjects().
 *
 * Unfortunately this will only ever return a single readiness notification at
 * a time.  Note that while the official documentation for
 * WaitForMultipleObjects is ambiguous about multiple events being "consumed"
 * with a single bWaitAll = FALSE call,
 * https://blogs.msdn.microsoft.com/oldnewthing/20150409-00/?p=44273 confirms
 * that only one event is "consumed".
 */
static inline int
WaitEventSetWaitBlock(WaitEventSet *set, int cur_timeout,
                      WaitEvent *occurred_events, int nevents)
{
    int         returned_events = 0;
    DWORD       rc;
    WaitEvent  *cur_event;

/* Reset any wait events that need it */
    for (cur_event = set->events;
         cur_event < (set->events + set->nevents);
         cur_event++)
    {
        if (cur_event->reset)
        {
            WaitEventAdjustWin32(set, cur_event);
            cur_event->reset = false;
        }

/*
         * Windows does not guarantee to log an FD_WRITE network event
         * indicating that more data can be sent unless the previous send()
         * failed with WSAEWOULDBLOCK.  While our caller might well have made
         * such a call, we cannot assume that here.  Therefore, if waiting for
         * write-ready, force the issue by doing a dummy send().  If the dummy
         * send() succeeds, assume that the socket is in fact write-ready, and
         * return immediately.  Also, if it fails with something other than
         * WSAEWOULDBLOCK, return a write-ready indication to let our caller
         * deal with the error condition.
         */
        if (cur_event->events & WL_SOCKET_WRITEABLE)
        {
            char        c;
            WSABUF      buf;
            DWORD       sent;
            int         r;

buf.buf = &c;
            buf.len = 0;

r = WSASend(cur_event->fd, &buf, 1, &sent, 0, NULL, NULL);
            if (r == 0 || WSAGetLastError() != WSAEWOULDBLOCK)
            {
                occurred_events->pos = cur_event->pos;
                occurred_events->user_data = cur_event->user_data;
                occurred_events->events = WL_SOCKET_WRITEABLE;
                occurred_events->fd = cur_event->fd;
                return 1;
            }
        }
    }

/*
     * Sleep.
     *
     * Need to wait for ->nevents + 1, because signal handle is in [0].
     */
    rc = WaitForMultipleObjects(set->nevents + 1, set->handles, FALSE,
                                cur_timeout);

/* Check return code */
    if (rc == WAIT_FAILED)
        elog(ERROR, "WaitForMultipleObjects() failed: error code %lu",
             GetLastError());
    else if (rc == WAIT_TIMEOUT)
    {
        /* timeout exceeded */
        return -1;
    }

if (rc == WAIT_OBJECT_0)
    {
        /* Service newly-arrived signals */
        pgwin32_dispatch_queued_signals();
        return 0;               /* retry */
    }

/*
     * With an offset of one, due to the always present pgwin32_signal_event,
     * the handle offset directly corresponds to a wait event.
     */
    cur_event = (WaitEvent *) &set->events[rc - WAIT_OBJECT_0 - 1];

occurred_events->pos = cur_event->pos;
    occurred_events->user_data = cur_event->user_data;
    occurred_events->events = 0;

if (cur_event->events == WL_LATCH_SET)
    {
        if (!ResetEvent(set->latch->event))
            elog(ERROR, "ResetEvent failed: error code %lu", GetLastError());

if (set->latch->is_set)
        {
            occurred_events->fd = PGINVALID_SOCKET;
            occurred_events->events = WL_LATCH_SET;
            occurred_events++;
            returned_events++;
        }
    }
    else if (cur_event->events == WL_POSTMASTER_DEATH)
    {
        /*
         * Postmaster apparently died.  Since the consequences of falsely
         * returning WL_POSTMASTER_DEATH could be pretty unpleasant, we take
         * the trouble to positively verify this with PostmasterIsAlive(),
         * even though there is no known reason to think that the event could
         * be falsely set on Windows.
         */
        if (!PostmasterIsAliveInternal())
        {
            if (set->exit_on_postmaster_death)
                proc_exit(1);
            occurred_events->fd = PGINVALID_SOCKET;
            occurred_events->events = WL_POSTMASTER_DEATH;
            occurred_events++;
            returned_events++;
        }
    }
    else if (cur_event->events & WL_SOCKET_MASK)
    {
        WSANETWORKEVENTS resEvents;
        HANDLE      handle = set->handles[cur_event->pos + 1];

Assert(cur_event->fd);

occurred_events->fd = cur_event->fd;

ZeroMemory(&resEvents, sizeof(resEvents));
        if (WSAEnumNetworkEvents(cur_event->fd, handle, &resEvents) != 0)
            elog(ERROR, "failed to enumerate network events: error code %u",
                 WSAGetLastError());
        if ((cur_event->events & WL_SOCKET_READABLE) &&
            (resEvents.lNetworkEvents & FD_READ))
        {
            /* data available in socket */
            occurred_events->events |= WL_SOCKET_READABLE;

/*------
             * WaitForMultipleObjects doesn't guarantee that a read event will
             * be returned if the latch is set at the same time.  Even if it
             * did, the caller might drop that event expecting it to reoccur
             * on next call.  So, we must force the event to be reset if this
             * WaitEventSet is used again in order to avoid an indefinite
             * hang.  Refer https://msdn.microsoft.com/en-us/library/windows/desktop/ms741576(v=vs.85).aspx
             * for the behavior of socket events.
             *------
             */
            cur_event->reset = true;
        }
        if ((cur_event->events & WL_SOCKET_WRITEABLE) &&
            (resEvents.lNetworkEvents & FD_WRITE))
        {
            /* writeable */
            occurred_events->events |= WL_SOCKET_WRITEABLE;
        }
        if ((cur_event->events & WL_SOCKET_CONNECTED) &&
            (resEvents.lNetworkEvents & FD_CONNECT))
        {
            /* connected */
            occurred_events->events |= WL_SOCKET_CONNECTED;
        }
        if (resEvents.lNetworkEvents & FD_CLOSE)
        {
            /* EOF/error, so signal all caller-requested socket flags */
            occurred_events->events |= (cur_event->events & WL_SOCKET_MASK);
        }

if (occurred_events->events != 0)
        {
            occurred_events++;
            returned_events++;
        }
    }

return returned_events;
}
#endif

```

### 三、跟蹤分析
在主節點上用gdb跟蹤postmaster,在PostgresMain上設定斷點後啟動standby節點,進入斷點
```
[xdb@localhost ~]$ ps -ef|grep postgres
xdb       1376     1  1 14:16 pts/0    00:00:00 /appdb/xdb/pg11.2/bin/postgres
[xdb@localhost ~]$ gdb -p 1376
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-100.el7
...
(gdb) set follow-fork-mode child
(gdb) b WalSndLoop
Breakpoint 1 at 0x853e63: file walsender.c, line 2111.
(gdb) c
Continuing.
[New process 1450]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
[Switching to Thread 0x7f17cfa9a8c0 (LWP 1450)]

Breakpoint 1, WalSndLoop (send_data=0x8547fe <xlogsendphysical>) at walsender.c:2111
2111        last_reply_timestamp = GetCurrentTimestamp();
(gdb) 
```
獲取時間戳,設定相關標記
```
(gdb) n
2112        waiting_for_ping_response = false;
(gdb) p last_reply_timestamp
$1 = 606818445090174
(gdb) 
```
重置MyLatch
```
(gdb) n
2124            if (!PostmasterIsAlive())
(gdb) 
2128            ResetLatch(MyLatch);
(gdb) p MyLatch
$2 = (struct Latch *) 0x7f17c46994d4
(gdb) p *MyLatch
$3 = {is_set = 1, is_shared = true, owner_pid = 1465}
(gdb) n
2130            CHECK_FOR_INTERRUPTS();
(gdb) p *MyLatch
$4 = {is_set = 0, is_shared = true, owner_pid = 1465}
(gdb) 
```
處理最近接收到的訊號
```
(gdb) n
2133            if (ConfigReloadPending)
(gdb) 
2141            ProcessRepliesIfAny();
(gdb) 
[Inferior 2 (process 1465) exited normally]
(gdb) 
```
程式退出,新產生了程式1466
```
xdb       1466  1376  0 16:41 ?        00:00:00 postgres: walsender replicator 192.168.26.26(40516) streaming 0/5D032830
```
跟蹤1466程式
```
(gdb) attach 1466
Attaching to program: /appdb/xdb/pg11.2/bin/postgres, process 1466
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done.
[Thread debugging using libthread_db enabled]
...
```
執行SQL
```
testdb=# drop table t1;
DROP TABLE
```
接收到訊號SIGUSR1,檢視呼叫棧如下
```
Program received signal SIGUSR1, User defined signal 1.
0x00007f17cde2d903 in __epoll_wait_nocancel () from /lib64/libc.so.6
(gdb) bt
#0  0x00007f17cde2d903 in __epoll_wait_nocancel () from /lib64/libc.so.6
#1  0x000000000088e668 in WaitEventSetWaitBlock (set=0x296e7c8, cur_timeout=29999, occurred_events=0x7fffed781d00, 
    nevents=1) at latch.c:1048
```
DONE!

### 四、參考資料
[PG Source Code](https://doxygen.postgresql.org)</xlogsendphysical></xlogsendphysical>

PostgreSQL 原始碼解讀（155）- 後臺程式#7（walsender#3）

相關文章