MySQL·原始碼分析·InnoDB非同步IO工作流程
之前的一篇核心月報InnoDB IO子系統 中介紹了InnoDB IO子系統中包含的同步IO以及非同步IO。本篇文章將從原始碼層面剖析一下InnoDB IO子系統中,資料頁的同步IO以及非同步IO請求的具體實現過程。
在MySQL5.6中,InnoDB的非同步IO主要是用來處理預讀以及對資料檔案的寫請求的。而對於正常的頁面資料讀取則是通過同步IO進行的。到底二者在程式碼層面上的實現過程有什麼樣的區別? 接下來我們將以Linux native io的執行過程為主線,對IO請求的執行過程進行梳理。
重點資料結構
- os_aio_array_t
/** 用來記錄某一類(ibuf,log,read,write)非同步IO(aio)請求的陣列型別。每一個非同步IO請求都會在型別對應的陣列中註冊一個innodb
aio物件。*/
os_aio_array_t {
os_ib_mutex_t mutex; // 主要用來控制非同步read/write執行緒的併發操作。對於ibuf,log型別,由於只有一個執行緒,所以不存在併發操作問題
os_event_t not_full; // 一個條件變數event,用來通知等待獲取slot的執行緒是否os_aio_array_t陣列有空閒的slot供aio請求
os_event_t is_empty; // 條件變數event,用來通知IO執行緒os_aio_array_t陣列是否有pening的IO請求。
ulint n_slots; // 陣列容納的IO請求數。= 執行緒數 * 每個segment允許pending的請求數(256)
ulint n_segments; // 允許獨立wait的segment數,即某種型別的IO的允許最大執行緒數
ulint cur_seg; /* IO請求會按照round robin的方式分配到不同的segment中,該變數指示下一個IO請求可以分配的segment */
ulint n_reserved; // 已經Pending的IO請求數
os_aio_slot_t* slots; // 用來記錄具體的每個IO請求物件的陣列,也即n_segments 個執行緒共用n_slots個槽位來存放pending io請求
#ifdef __WIN__
HANDLE* handles;
/*!< Pointer to an array of OS native
event handles where we copied the
handles from slots, in the same
order. This can be used in
WaitForMultipleObjects; used only in
Windows */
#endif __WIN__
#if defined(LINUX_NATIVE_AIO)
io_context_t* aio_ctx; // aio上下文的陣列,每個segment擁有獨立的一個aio上下文陣列,用來記錄以及完成的IO請求上下文
struct io_event* aio_events; // 該陣列用來記錄已經完成的IO請求事件。非同步IO通過設定事件通知IO執行緒處理完成的IO請求
struct iocb** pending; // 用來記錄pending的aio請求
ulint* count; // 該陣列記錄了每個segment對應的pending aio請求數量
#endif /* LINUX_NATIV_AIO */
}
- os_aio_slot_t
// os_aio_array_t陣列中用來記錄一個非同步IO(aio)請求的物件
os_aio_slot_t {
ibool is_read; /*!< TRUE if a read operation */
ulint pos; // os_aio_array_t陣列中所在的位置
ibool reserved; // TRUE表示該Slot已經被別的IO請求佔用了
time_t reservation_time; // 佔用的時間
ulint len; // io請求的長度
byte* buf; // 資料讀取或者需要寫入的buffer,通常指向buffer pool的一個頁面,壓縮頁面有特殊處理
ulint type; /* 請求型別,即讀還是寫IO請求 */
os_offset_t offset; /*!< file offset in bytes */
os_file_t file; /*!< file where to read or write */
const char* name; /*!< 需要讀取的檔案及路徑資訊 */
ibool io_already_done; /* TRUE表示IO已經完成了
fil_node_t* message1; /* 該aio操作的innodb檔案描述符(f_node_t)*/
void* message2; /* 用來記錄完成IO請求所對應的具體buffer pool bpage頁 */
#ifdef WIN_ASYNC_IO
HANDLE handle; /*!< handle object we need in the
OVERLAPPED struct */
OVERLAPPED control; /*!< Windows control block for the
aio request */
#elif defined(LINUX_NATIVE_AIO)
struct iocb control; /* 該slot使用的aio請求控制塊iocb */
int n_bytes; /* 讀寫bytes */
int ret; /* AIO return code */
#endif /* WIN_ASYNC_IO */
}
流程圖
原始碼分析
- 物理資料頁操作入口函式os_aio_func
ibool
os_aio_func(
/*========*/
ulint type, /* IO型別,READ還是WRITE IO */
ulint mode, /* 這裡表示是否使用SIMULATED aio執行非同步IO請求 */
const char* name, /* IO需要開啟的tablespace路徑+名稱 */
os_file_t file, /* IO操作的檔案 */
void* buf, // 資料讀取或者需要寫入的buffer,通常指向buffer pool的一個頁面,壓縮頁面有特殊處理
os_offset_t offset, /*!< in: file offset where to read or write */
ulint n, /* 讀取或寫入位元組數 */
fil_node_t* message1, /* 該aio操作的innodb檔案描述符(f_node_t),只對非同步IO起作用 */
void* message2, /* 用來記錄完成IO請求所對應的具體buffer pool bpage頁,只對非同步IO起作用 */
ibool should_buffer, // 是否需要快取aio請求,該變數主要對預讀起作用
ibool page_encrypt,
/*!< in: Whether to encrypt */
ulint page_size)
/*!< in: Page size */
{
...
wake_later = mode & OS_AIO_SIMULATED_WAKE_LATER;
mode = mode & (~OS_AIO_SIMULATED_WAKE_LATER);
if (mode == OS_AIO_SYNC
#ifdef WIN_ASYNC_IO
&& !srv_use_native_aio
#endif /* WIN_ASYNC_IO */
) {
/* This is actually an ordinary synchronous read or write:
no need to use an i/o-handler thread. NOTE that if we use
Windows async i/o, Windows does not allow us to use
ordinary synchronous os_file_read etc. on the same file,
therefore we have built a special mechanism for synchronous
wait in the Windows case.
Also note that the Performance Schema instrumentation has
been performed by current os_aio_func()`s wrapper function
pfs_os_aio_func(). So we would no longer need to call
Performance Schema instrumented os_file_read() and
os_file_write(). Instead, we should use os_file_read_func()
and os_file_write_func() */
/* 這裡如果是同步IO,並且native io沒有開啟的情況下,直接使用os_file_read/write函式進行讀取,
不需要經過IO執行緒進行處理 */
if (type == OS_FILE_READ) {
if (page_encrypt) {
return(os_file_read_decrypt_page(file, buf, offset, n, page_size));
} else {
return(os_file_read_func(file, buf, offset, n));
}
}
ut_ad(!srv_read_only_mode);
ut_a(type == OS_FILE_WRITE);
if (page_encrypt) {
return(os_file_write_encrypt_page(name, file, buf, offset, n, page_size));
} else {
return(os_file_write_func(name, file, buf, offset, n));
}
}
try_again:
switch (mode) {
// 根據訪問型別,定位IO請求陣列
case OS_AIO_NORMAL:
if (type == OS_FILE_READ) {
array = os_aio_read_array;
} else {
ut_ad(!srv_read_only_mode);
array = os_aio_write_array;
}
break;
case OS_AIO_IBUF:
ut_ad(type == OS_FILE_READ);
/* Reduce probability of deadlock bugs in connection with ibuf:
do not let the ibuf i/o handler sleep */
wake_later = FALSE;
if (srv_read_only_mode) {
array = os_aio_read_array;
}
break;
case OS_AIO_LOG:
if (srv_read_only_mode) {
array = os_aio_read_array;
} else {
array = os_aio_log_array;
}
break;
case OS_AIO_SYNC:
array = os_aio_sync_array;
#if defined(LINUX_NATIVE_AIO)
/* In Linux native AIO we don`t use sync IO array. */
ut_a(!srv_use_native_aio);
#endif /* LINUX_NATIVE_AIO */
break;
default:
ut_error;
array = NULL; /* Eliminate compiler warning */
}
// 阻塞為當前IO請求申請一個用來執行非同步IO的slot
slot = os_aio_array_reserve_slot(type, array, message1, message2, file,
name, buf, offset, n, page_encrypt, page_size);
DBUG_EXECUTE_IF("simulate_slow_aio",
{
os_thread_sleep(1000000);
}
);
if (type == OS_FILE_READ) {
if (srv_use_native_aio) {
os_n_file_reads++;
os_bytes_read_since_printout += n;
#ifdef WIN_ASYNC_IO
// 這裡是Windows用來處理非同步IO讀請求
ret = ReadFile(file, buf, (DWORD) n, &len,
&(slot->control));
#elif defined(LINUX_NATIVE_AIO)
// 這裡是Linux來處理native io
if (!os_aio_linux_dispatch(array, slot, should_buffer)) {
goto err_exit;
#endif /* WIN_ASYNC_IO */
} else {
if (!wake_later) {
// 喚醒simulated aio處理執行緒
os_aio_simulated_wake_handler_thread(
os_aio_get_segment_no_from_slot(
array, slot));
}
}
} else if (type == OS_FILE_WRITE) {
ut_ad(!srv_read_only_mode);
if (srv_use_native_aio) {
os_n_file_writes++;
#ifdef WIN_ASYNC_IO
// 這裡是Windows用來處理非同步IO寫請求
ret = WriteFile(file, buf, (DWORD) n, &len,
&(slot->control));
#elif defined(LINUX_NATIVE_AIO)
// 這裡是Linux來處理native io
if (!os_aio_linux_dispatch(array, slot, false)) {
goto err_exit;
}
#endif /* WIN_ASYNC_IO */
} else {
if (!wake_later) {
// 喚醒simulated aio處理執行緒
os_aio_simulated_wake_handler_thread(
os_aio_get_segment_no_from_slot(
array, slot));
}
}
} else {
ut_error;
}
...
}
- 負責通知Linux核心執行native IO請求的函式os_aio_linux_dispatch
static
ibool
os_aio_linux_dispatch(
/*==================*/
os_aio_array_t* array, /* IO請求函式 */
os_aio_slot_t* slot, /* 申請好的slot */
ibool should_buffer) // 是否需要快取aio 請求,該變數主要對預讀起作用
{
...
/* Find out what we are going to work with.
The iocb struct is directly in the slot.
The io_context is one per segment. */
// 每個segment包含的slot個數,Linux下每個segment包含256個slot
slots_per_segment = array->n_slots / array->n_segments;
iocb = &slot->control;
io_ctx_index = slot->pos / slots_per_segment;
if (should_buffer) {
/* 這裡也可以看到aio請求快取只對讀請求起作用 */
ut_ad(array == os_aio_read_array);
ulint n;
ulint count;
os_mutex_enter(array->mutex);
/* There are array->n_slots elements in array->pending, which is divided into
* array->n_segments area of equal size. The iocb of each segment are
* buffered in its corresponding area in the pending array consecutively as
* they come. array->count[i] records the number of buffered aio requests in
* the ith segment.*/
n = io_ctx_index * slots_per_segment
+ array->count[io_ctx_index];
array->pending[n] = iocb;
array->count[io_ctx_index] ++;
count = array->count[io_ctx_index];
os_mutex_exit(array->mutex);
// 如果當前segment的slot都已經被佔用了,就需要提交一次非同步aio請求
if (count == slots_per_segment) {
os_aio_linux_dispatch_read_array_submit(); //no cover line
}
// 否則就直接返回
return (TRUE);
}
// 直接提交IO請求到核心
ret = io_submit(array->aio_ctx[io_ctx_index], 1, &iocb);
...
}
- IO執行緒負責監控aio請求的主函式fil_aio_wait
void
fil_aio_wait(
/*=========*/
ulint segment) /*!< in: the number of the segment in the aio
array to wait for */
{
ibool ret;
fil_node_t* fil_node;
void* message;
ulint type;
ut_ad(fil_validate_skip());
if (srv_use_native_aio) { // 使用native io
srv_set_io_thread_op_info(segment, "native aio handle");
#ifdef WIN_ASYNC_IO
ret = os_aio_windows_handle( // Window監控入口
segment, 0, &fil_node, &message, &type);
#elif defined(LINUX_NATIVE_AIO)
ret = os_aio_linux_handle( // Linux native io監控入口
segment, &fil_node, &message, &type);
#else
ut_error;
ret = 0; /* Eliminate compiler warning */
#endif /* WIN_ASYNC_IO */
} else {
srv_set_io_thread_op_info(segment, "simulated aio handle");
ret = os_aio_simulated_handle( // Simulated aio監控入口
segment, &fil_node, &message, &type);
}
ut_a(ret);
if (fil_node == NULL) {
ut_ad(srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS);
return;
}
srv_set_io_thread_op_info(segment, "complete io for fil node");
mutex_enter(&fil_system->mutex);
// 到這裡表示至少有一個IO請求已經完成,該函式設定狀態資訊
fil_node_complete_io(fil_node, fil_system, type);
mutex_exit(&fil_system->mutex);
ut_ad(fil_validate_skip());
/* Do the i/o handling */
/* IMPORTANT: since i/o handling for reads will read also the insert
buffer in tablespace 0, you have to be very careful not to introduce
deadlocks in the i/o system. We keep tablespace 0 data files always
open, and use a special i/o thread to serve insert buffer requests. */
if (fil_node->space->purpose == FIL_TABLESPACE) { // 資料檔案讀寫IO
srv_set_io_thread_op_info(segment, "complete io for buf page");
// IO請求完成後,這裡處理buffer pool對應的bpage相關的一些狀態資訊並根據checksum驗證資料的正確性
buf_page_io_complete(static_cast<buf_page_t*>(message));
} else { // 日誌檔案的讀寫IO
srv_set_io_thread_op_info(segment, "complete io for log");
log_io_complete(static_cast<log_group_t*>(message));
}
}
#endif /* UNIV_HOTBACKUP */
- IO執行緒負責處理native IO請求的函式os_aio_linux_handle
ibool
os_aio_linux_handle(ulint global_seg, // 屬於哪個segment
fil_node_t**message1, /* 該aio操作的innodb檔案描述符(f_node_t)*/
void** message2, /* 用來記錄完成IO請求所對應的具體buffer pool bpage頁 */
ulint* type){ // 讀or寫IO
// 根據global_seg獲得該aio 的os_aio_array_t陣列,並返回對應的segment
segment = os_aio_get_array_and_local_segment(&array, global_seg);
n = array->n_slots / array->n_segments; //獲得一個執行緒可監控的io event數
/* Loop until we have found a completed request. */
for (;;) {
ibool any_reserved = FALSE;
os_mutex_enter(array->mutex);
for (i = 0; i < n; ++i) { // 遍歷該執行緒所發起的所有aio請求
slot = os_aio_array_get_nth_slot(
array, i + segment * n);
if (!slot->reserved) { // 該slot是否被佔用
continue;
} else if (slot->io_already_done) { // IO請求已經完成,可以通知主執行緒返回資料了
/* Something for us to work on. */
goto found;
} else {
any_reserved = TRUE;
}
}
os_mutex_exit(array->mutex);
// 到這裡說明沒有找到一個完成的io,則再去collect
os_aio_linux_collect(array, segment, n);
found: // 找到一個完成的io,將內容返回
*message1 = slot->message1;
*message2 = slot->message2; // 返回完成IO所對應的bpage頁
*type = slot->type;
if (slot->ret == 0 && slot->n_bytes == (long) slot->len) {
if (slot->page_encrypt
&& slot->type == OS_FILE_READ) {
os_decrypt_page(slot->buf, slot->len, slot->page_size, FALSE);
}
ret = TRUE;
} else {
errno = -slot->ret;
/* os_file_handle_error does tell us if we should retry
this IO. As it stands now, we don`t do this retry when
reaping requests from a different context than
the dispatcher. This non-retry logic is the same for
windows and linux native AIO.
We should probably look into this to transparently
re-submit the IO. */
os_file_handle_error(slot->name, "Linux aio");
ret = FALSE;
}
os_mutex_exit(array->mutex);
os_aio_array_free_slot(array, slot);
return(ret);
}
- 等待native IO請求完成os_aio_linux_collect
os_aio_linux_collect(os_aio_array_t* array,
ulint segment,
ulint seg_size){
events = &array->aio_events[segment * seg_size]; // 定位segment所對應的io event的陣列位置
/* 獲得該執行緒的aio上下文陣列 */
io_ctx = array->aio_ctx[segment];
/* Starting point of the segment we will be working on. */
start_pos = segment * seg_size;
/* End point. */
end_pos = start_pos + seg_size;
retry:
/* Initialize the events. The timeout value is arbitrary.
We probably need to experiment with it a little. */
memset(events, 0, sizeof(*events) * seg_size);
timeout.tv_sec = 0;
timeout.tv_nsec = OS_AIO_REAP_TIMEOUT;
ret = io_getevents(io_ctx, 1, seg_size, events, &timeout); // 阻塞等待該IO執行緒所監控的任一IO請求完成
if (ret > 0) { // 有IO請求完成
for (i = 0; i < ret; i++) {
// 記錄完成IO的請求資訊到對應的os_aio_slot_t 物件
os_aio_slot_t* slot;
struct iocb* control;
control = (struct iocb*) events[i].obj; // 獲得完成的aio的iocb,即提交這個aio請求的iocb
ut_a(control != NULL);
slot = (os_aio_slot_t*) control->data; // 通過data獲得這個aio iocb所對應的os_aio_slot_t
/* Some sanity checks. */
ut_a(slot != NULL);
ut_a(slot->reserved);
os_mutex_enter(array->mutex);
slot->n_bytes = events[i].res; // 將該io執行的結果儲存到slot裡
slot->ret = events[i].res2;
slot->io_already_done = TRUE; // 標誌該io已經完成了,這個標誌也是外層判斷的條件
os_mutex_exit(array->mutex);
}
return;
}
…
}
綜上重點對InnoDB navtive IO讀寫資料檔案從原始碼角度進行了分析,有興趣的讀者也可以繼續瞭解InnoDB自帶的simulated IO的實現過程,原理雷同native IO,只是在實現方式上自己進行了處理。本篇文章對InnoDB IO請求的執行流程進行了梳理,對重點資料結構以及函式進行了分析,希望對讀者日後進行原始碼閱讀及修改有所幫助。
相關文章
- InnoDB IO路徑原始碼分析原始碼
- 深入OKHttp原始碼分析(一)----同步和非同步請求流程和原始碼分析HTTP原始碼非同步
- Mysql工作流程分析MySql
- java同步非阻塞IOJava
- Java 非阻塞 IO 和非同步 IOJava非同步
- InnoDB儲存引擎——非同步IO儲存引擎非同步
- MySQL InnoDB update流程MySql
- IO模式和IO多路複用(阻塞IO、非阻塞IO、同步IO、非同步IO等概念)模式非同步
- IO - 同步 非同步 阻塞 非阻塞的區別非同步
- MySQL 引擎特性:InnoDB IO 子系統MySql
- WindowManager呼叫流程原始碼分析原始碼
- 執行流程原始碼分析原始碼
- Lettuce同步命令原始碼分析原始碼
- android IO Prefetch原始碼分析Android原始碼
- 網路IO之阻塞、非阻塞、同步、非同步總結非同步
- 談談對不同I/O模型的理解 (阻塞/非阻塞IO,同步/非同步IO)模型非同步
- IO通訊模型(二)同步非阻塞模式NIO(NonBlocking IO)模型模式BloC
- MySQL InnoDB Update和Crash Recovery流程MySql
- MySQL 5.5 原始碼安裝流程MySql原始碼
- MySQL 引擎特性:InnoDB 同步機制MySql
- Flutter啟動流程原始碼分析Flutter原始碼
- View繪製流程原始碼分析View原始碼
- 原始碼分析Retrofit請求流程原始碼
- Mybatis執行流程原始碼分析MyBatis原始碼
- apiserver原始碼分析——啟動流程APIServer原始碼
- Activity啟動流程原始碼分析原始碼
- Spark job分配流程原始碼分析Spark原始碼
- Java IO原始碼分析(三)——PipedOutputStream和PipedInputStreamJava原始碼
- Java IO 之 FileInputStream & FileOutputStream 原始碼分析Java原始碼
- Okhttp同步請求原始碼分析HTTP原始碼
- 影片聊天原始碼,同步、非同步示例程式碼分析原始碼非同步
- MYSQL原始碼分析1MySql原始碼
- Mysql原始碼分析2MySql原始碼
- RxJava2原始碼分析(一):基本流程分析RxJava原始碼
- 如何解讀 Java IO、NIO 中的同步阻塞與同步非阻塞?Java
- Composer 工作原理 [原始碼分析]原始碼
- Struts2 原始碼分析-----工作原理分析原始碼
- Dart 原始碼分析:深入理解 dart:io HttpClientDart原始碼HTTPclient