以下程式碼基於swoole4.4.5-alpha, php7.1.26
我們按照執行流程去逐步分析swoole協程的實現, php程式是這樣的:
<?php
go(function (){
Co::sleep(1);
echo "a";
});
echo "c";go實際上是swoole_coroutine_create的別名:
PHP_FALIAS(go, swoole_coroutine_create, arginfo_swoole_coroutine_create);首先會執行zif_swoole_coroutine_create去建立協程:
// 真正執行的函式
PHP_FUNCTION(swoole_coroutine_create)
{
...
// 解析引數
ZEND_PARSE_PARAMETERS_START(1, -1)
Z_PARAM_FUNC(fci, fci_cache)
Z_PARAM_VARIADIC('*', fci.params, fci.param_count)
ZEND_PARSE_PARAMETERS_END_EX(RETURN_FALSE);
...
long cid = PHPCoroutine::create(&fci_cache, fci.param_count, fci.params);
if (sw_likely(cid > 0))
{
RETURN_LONG(cid);
}
else
{
RETURN_FALSE;
}
}
long PHPCoroutine::create(zend_fcall_info_cache *fci_cache, uint32_t argc, zval *argv)
{
...
// 儲存匿名函式引數和執行結構
php_coro_args php_coro_args;
php_coro_args.fci_cache = fci_cache;
php_coro_args.argv = argv;
php_coro_args.argc = argc;
save_task(get_task()); // 儲存php棧到當前task
// 建立coroutine
return Coroutine::create(main_func, (void*) &php_coro_args);
}php_coro_args是用來儲存回撥函式資訊的結構:
// 儲存回撥函式的結構體
struct php_coro_args
{
zend_fcall_info_cache *fci_cache; // 匿名函式資訊
zval *argv; // 引數
uint32_t argc; // 引數數量
};php_corutine::get_task()用來獲取當前正在執行的任務, 第一次執行時, 獲取的是初始化好的main_task:
php_coro_task PHPCoroutine::main_task = {0};
// 獲取當前的task, 沒有則是主task
static inline php_coro_task* get_task()
{
php_coro_task *task = (php_coro_task *) Coroutine::get_current_task();
return task ? task : &main_task;
}
static inline void* get_current_task()
{
return sw_likely(current) ? current->get_task() : nullptr;
}
inline void* get_task()
{
return task;
}save_task會將當前php棧資訊儲存到當前使用的task上, 當前使用的是main_task, 所以這些資訊會被儲存在main_task上:
void PHPCoroutine::save_task(php_coro_task *task)
{
save_vm_stack(task); // 儲存php棧
...
}
inline void PHPCoroutine::save_vm_stack(php_coro_task *task)
{
task->bailout = EG(bailout);
task->vm_stack_top = EG(vm_stack_top); // 當前棧頂
task->vm_stack_end = EG(vm_stack_end); // 棧底
task->vm_stack = EG(vm_stack); // 整個棧結構
task->vm_stack_page_size = EG(vm_stack_page_size);
task->error_handling = EG(error_handling);
task->exception_class = EG(exception_class);
task->exception = EG(exception);
}php_coro_task這個結構用來儲存當前任務的php棧:
struct php_coro_task
{
JMP_BUF *bailout; // 內部異常使用
zval *vm_stack_top; // 棧頂
zval *vm_stack_end; // 棧底
zend_vm_stack vm_stack; // 執行棧
size_t vm_stack_page_size;
zend_execute_data *execute_data;
zend_error_handling_t error_handling;
zend_class_entry *exception_class;
zend_object *exception;
zend_output_globals *output_ptr;
/* for array_walk non-reentrancy */
php_swoole_fci *array_walk_fci;
swoole::Coroutine *co; // 屬於哪個coroutine
std::stack<php_swoole_fci *> *defer_tasks;
long pcid;
zend_object *context;
int64_t last_msec;
zend_bool enable_scheduler;
};儲存完當前php棧就可以開始建立coroutine了:
static inline long create(coroutine_func_t fn, void* args = nullptr)
{
return (new Coroutine(fn, args))->run();
}
Coroutine(coroutine_func_t fn, void *private_data) :
ctx(stack_size, fn, private_data) // 預設stack size 2M
{
cid = ++last_cid; // 分配協程id
coroutines[cid] = this; // 當前物件指標儲存在全域性的corutines靜態屬性上
if (sw_unlikely(count() > peak_num)) // 更新峰值
{
peak_num = count();
}
}首先, 會建立一個ctx物件, context物件主要用來管理c棧
#define SW_DEFAULT_C_STACK_SIZE (2 *1024 * 1024)
size_t Coroutine::stack_size = SW_DEFAULT_C_STACK_SIZE;
ctx(stack_size, fn, private_data)
Context::Context(size_t stack_size, coroutine_func_t fn, void* private_data) :
fn_(fn), stack_size_(stack_size), private_data_(private_data)
{
end_ = false; // 標記協程是否已經執行完成
swap_ctx_ = nullptr;
stack_ = (char*) sw_malloc(stack_size_); // 分配一塊記憶體儲存c棧, 預設2M
...
void* sp = (void*) ((char*) stack_ + stack_size_); // 計算出棧頂地址即最高地址
ctx_ = make_fcontext(sp, stack_size_, (void (*)(intptr_t))&context_func); // 構建上下文
}make_fcontext函式是boost.context庫中提供的,由彙編編寫,不同平臺有不同實現,我們這裡使用的是make_x86_64_sysv_elf_gas.S這個檔案:
傳參使用的暫存器依次是rdi、rsi、rdx、rcx、r8、r9
make_fcontext:
/* first arg of make_fcontext() == top of context-stack */
/* rax = sp */
movq %rdi, %rax
/* shift address in RAX to lower 16 byte boundary */
/* rax = rax & -16 => rax = rax & (~0x10000 + 1) => rax = rax - rax%16, 其實就是按16對齊*/
andq $-16, %rax
/* reserve space for context-data on context-stack */
/* size for fc_mxcsr .. RIP + return-address for context-function */
/* on context-function entry: (RSP -0x8) % 16 == 0 */
/*lea是“load effective address”的縮寫，
簡單的說，lea指令可以用來將一個記憶體地址直接賦給目的運算元，
例如：lea eax,[ebx+8]就是將ebx+8這個值直接賦給eax，而不是把ebx+8處的記憶體地址裡的資料賦給eax。
而mov指令則恰恰相反，例如：mov eax,[ebx+8]則是把記憶體地址為ebx+8處的資料賦給eax。*/
/* rax = rax - 0x48, 預留0x48個位元組 */
leaq -0x48(%rax), %rax
/* third arg of make_fcontext() == address of context-function */
/* context_func函式地址放在rax+0x38處*/
movq %rdx, 0x38(%rax)
/* save MMX control- and status-word */
stmxcsr (%rax)
/* save x87 control-word */
fnstcw 0x4(%rax)
/* compute abs address of label finish */
/*
https://sourceware.org/binutils/docs/as/i386_002dMemory.html
The x86-64 architecture adds an RIP (instruction pointer relative) addressing.
This addressing mode is specified by using ‘rip’ as a base register. Only constant offsets are valid. For example:
AT&T: ‘1234(%rip)’, Intel: ‘[rip + 1234]’
Points to the address 1234 bytes past the end of the current instruction.
AT&T: ‘symbol(%rip)’, Intel: ‘[rip + symbol]’
Points to the symbol in RIP relative way, this is shorter than the default absolute addressing.
*/
/* rcx = finish */
leaq finish(%rip), %rcx
/* save address of finish as return-address for context-function */
/* will be entered after context-function returns */
/* finish函式地址放在rax+0x40處 */
movq %rcx, 0x40(%rax)
/*return rax*/
ret /* return pointer to context-data */
finish:
/* exit code is zero */
xorq %rdi, %rdi
/* exit application */
call _exit@PLT
hltmake_fcontext函式執行完之後, 用來儲存上下文的記憶體佈局是這樣:
/****************************************************************************************
* |<- ctx_
---------------------------------------------------------------------------------- *
* | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | *
* ---------------------------------------------------------------------------------- *
* | 0x0 | 0x4 | 0x8 | 0xc | 0x10 | 0x14 | 0x18 | 0x1c | *
* ---------------------------------------------------------------------------------- *
* | fc_mxcsr|fc_x87_cw| | | | *
* ---------------------------------------------------------------------------------- *
* ---------------------------------------------------------------------------------- *
* | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | *
* ---------------------------------------------------------------------------------- *
* | 0x20 | 0x24 | 0x28 | 0x2c | 0x30 | 0x34 | 0x38 | 0x3c | *
* ---------------------------------------------------------------------------------- *
* | | | | context_func | *
* ---------------------------------------------------------------------------------- *
* ---------------------------------------------------------------------------------- *
* | 16 | 17 | | *
* ---------------------------------------------------------------------------------- *
* | 0x40 | 0x44 | | *
* ---------------------------------------------------------------------------------- *
* | finish | | *
* ---------------------------------------------------------------------------------- *
* *
****************************************************************************************/Coroutine物件被例項化完之後開始執行run方法, run方法會將上一個執行了相關方法的Coroutine物件存入origin中, 並把current置為當前物件:
static sw_co_thread_local Coroutine* current;
Coroutine *origin;
inline long run()
{
long cid = this->cid;
origin = current; // orign儲存原來的物件
current = this; // current置為當前物件
ctx.swap_in(); // 換入
...
}接下來是切換c棧的核心方法, swap_in和swap_out, 底層也是由boost.context庫提供的, 先來看換入:
bool Context::swap_in()
{
jump_fcontext(&swap_ctx_, ctx_, (intptr_t) this, true);
return true;
}
// jump_x86_64_sysv_elf_gas.S
jump_fcontext:
/* 當前暫存器壓入棧, 注意, rbp上面實際上還有一個rip, 因為call jump_fcontext 等價於 push rip, jmp jump_fcontext. */
/* rip儲存著下一條要執行的指令, 在這裡就是jump_fcontext之後的return true */
pushq %rbp /* save RBP */
pushq %rbx /* save RBX */
pushq %r15 /* save R15 */
pushq %r14 /* save R14 */
pushq %r13 /* save R13 */
pushq %r12 /* save R12 */
/* prepare stack for FPU */
leaq -0x8(%rsp), %rsp
/* test for flag preserve_fpu */
cmp $0, %rcx
je 1f
/* save MMX control- and status-word */
stmxcsr (%rsp)
/* save x87 control-word */
fnstcw 0x4(%rsp)
1:
/* store RSP (pointing to context-data) in RDI */
/* *swap_ctx_ = rsp, 儲存棧頂位置 */
movq %rsp, (%rdi)
/* restore RSP (pointing to context-data) from RSI */
/* rsp = ctx_, 這裡將當前執行棧指向了剛剛透過make_fcontext構建出來的棧 */
movq %rsi, %rsp
/* test for flag preserve_fpu */
cmp $0, %rcx
je 2f
/* restore MMX control- and status-word */
ldmxcsr (%rsp)
/* restore x87 control-word */
fldcw 0x4(%rsp)
2:
/* prepare stack for FPU */
leaq 0x8(%rsp), %rsp
/* 將暫存器恢復從新棧上壓入的值, 這次執行時這裡還都是空的 */
popq %r12 /* restrore R12 */
popq %r13 /* restrore R13 */
popq %r14 /* restrore R14 */
popq %r15 /* restrore R15 */
popq %rbx /* restrore RBX */
popq %rbp /* restrore RBP */
/* restore return-address */
/* r8 = make_fcontext(往上看看make_fcontext結束後的記憶體佈局圖) */
popq %r8
/* use third arg as return-value after jump */
/* rax = this */
movq %rdx, %rax
/* use third arg as first arg in context function */
/* rdi = this */
movq %rdx, %rdi
/* indirect jump to context */
/* 執行context_func */
jmp *%r8jump_fcontext執行完之後原來的棧記憶體佈局是這樣:
/****************************************************************************************
* |<-swap_ctx_ *
* ---------------------------------------------------------------------------------- *
* | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | *
* ---------------------------------------------------------------------------------- *
* | 0x0 | 0x4 | 0x8 | 0xc | 0x10 | 0x14 | 0x18 | 0x1c | *
* ---------------------------------------------------------------------------------- *
* | fc_mxcsr|fc_x87_cw| R12 | R13 | R14 | *
* ---------------------------------------------------------------------------------- *
* ---------------------------------------------------------------------------------- *
* | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | *
* ---------------------------------------------------------------------------------- *
* | 0x20 | 0x24 | 0x28 | 0x2c | 0x30 | 0x34 | 0x38 | 0x3c | *
* ---------------------------------------------------------------------------------- *
* | R15 | RBX | RBP | RIP/return true | *
* ---------------------------------------------------------------------------------- *
* *
****************************************************************************************/context_func有一個引數, jump_fcontext執行完後往rdi寫入的this將作為引數給contextfunc使用, fn, private_data_是構造ctx時傳入的引數:
void Context::context_func(void *arg)
{
Context *_this = (Context *) arg;
_this->fn_(_this->private_data_); // main_func(php_coro_args)
_this->end_ = true;
_this->swap_out();
}main_func會為當前協程分配一個新的執行棧, 並將其與剛剛例項化好的Coroutine繫結, 然後執行協程的回撥函式:
void PHPCoroutine::main_func(void *arg)
{
...
// 在EG上建立一個新的vmstack, 用於執行go()裡的回撥函式, 之前的執行棧已經被儲存在main_task上了
vm_stack_init();
call = (zend_execute_data *) (EG(vm_stack_top));
task = (php_coro_task *) EG(vm_stack_top);
EG(vm_stack_top) = (zval *) ((char *) call + PHP_CORO_TASK_SLOT * sizeof(zval)); // 為task預留位置
call = zend_vm_stack_push_call_frame(call_info, func, argc, object_or_called_scope); // 為引數分配棧空間
EG(bailout) = NULL;
EG(current_execute_data) = call;
EG(error_handling) = EH_NORMAL;
EG(exception_class) = NULL;
EG(exception) = NULL;
save_vm_stack(task); // 儲存vmstack到當前task上
record_last_msec(task); // 記錄時間
task->output_ptr = NULL;
task->array_walk_fci = NULL;
task->co = Coroutine::get_current(); // 記錄當前coroutine
task->co->set_task((void *) task); // coroutine與當前task繫結
task->defer_tasks = nullptr;
task->pcid = task->co->get_origin_cid(); // 記錄上一個協程id
task->context = nullptr;
task->enable_scheduler = 1;
if (EXPECTED(func->type == ZEND_USER_FUNCTION))
{
...
// 初始化execute_data
zend_init_func_execute_data(call, &func->op_array, retval);
// 執行協程裡的使用者函式
zend_execute_ex(EG(current_execute_data));
}
...
}接下來就是執行使用者回撥函式生成的opcode了, 執行到Co::sleep(1)時會呼叫System::sleep(seconds), 這裡面會為當前coroutine註冊一個定時事件, 回撥函式是sleep_timeout:
int System::sleep(double sec)
{
Coroutine* co = Coroutine::get_current_safe(); // 獲取當前coroutine
if (swoole_timer_add((long) (sec * 1000), SW_FALSE, sleep_timeout, co) == NULL) // 為當前couroutine新增一個定時事件
{
return -1;
}
co->yield(); // 切換
return 0;
}
// 定時事件註冊的回撥
static void sleep_timeout(swTimer *timer, swTimer_node *tnode)
{
((Coroutine *) tnode->data)->resume();
}yield函式負責php棧和c棧的切換
void Coroutine::yield()
{
SW_ASSERT(current == this || on_bailout != nullptr);
state = SW_CORO_WAITING; // 協程狀態變為waiting
if (sw_likely(on_yield))
{
on_yield(task); // php棧切換
}
current = origin; // 切換當前協程到上一個
ctx.swap_out(); // c棧切換
}先來看php棧的切換, on_yield是初始化時已經註冊好的函式
void PHPCoroutine::init()
{
Coroutine::set_on_yield(on_yield);
Coroutine::set_on_resume(on_resume);
Coroutine::set_on_close(on_close);
}
void PHPCoroutine::on_yield(void *arg)
{
php_coro_task *task = (php_coro_task *) arg; // 當前task
php_coro_task *origin_task = get_origin_task(task); // 獲取上一個task
save_task(task); // 儲存當前任務
restore_task(origin_task); // 恢復上一個任務
}拿到上一個task就可以透過上面儲存的執行資訊恢復EG了, 程式很簡單, 只要把vmstack和current_execute_data換回來就可以了:
void PHPCoroutine::restore_task(php_coro_task *task)
{
restore_vm_stack(task);
...
}
inline void PHPCoroutine::restore_vm_stack(php_coro_task *task)
{
EG(bailout) = task->bailout;
EG(vm_stack_top) = task->vm_stack_top;
EG(vm_stack_end) = task->vm_stack_end;
EG(vm_stack) = task->vm_stack;
EG(vm_stack_page_size) = task->vm_stack_page_size;
EG(current_execute_data) = task->execute_data;
EG(error_handling) = task->error_handling;
EG(exception_class) = task->exception_class;
EG(exception) = task->exception;
...
}這個時候php棧執行狀態已經恢復到剛剛呼叫go()函式時的狀態了(main_task), 再看看c棧切換是怎麼處理的:
bool Context::swap_out()
{
jump_fcontext(&ctx_, swap_ctx_, (intptr_t) this, true);
return true;
}回憶一下swap_in函式, swap_ctx_儲存著執行swap_in時的rsp, ctx_儲存著透過make_fcontext初始化好的棧頂位置, 再來看一遍jump_fcontext執行:
// jump_x86_64_sysv_elf_gas.S
jump_fcontext:
/* 當前暫存器壓入棧, 注意, rbp上面實際上還有一個rip, 因為call jump_fcontext 等價於 push rip, jmp jump_fcontext. */
/* rip儲存著下一條要執行的指令, 在這裡就是swap_out裡jump_fcontext之後的return true */
pushq %rbp /* save RBP */
pushq %rbx /* save RBX */
pushq %r15 /* save R15 */
pushq %r14 /* save R14 */
pushq %r13 /* save R13 */
pushq %r12 /* save R12 */
/* prepare stack for FPU */
leaq -0x8(%rsp), %rsp
/* test for flag preserve_fpu */
cmp $0, %rcx
je 1f
/* save MMX control- and status-word */
stmxcsr (%rsp)
/* save x87 control-word */
fnstcw 0x4(%rsp)
1:
/* store RSP (pointing to context-data) in RDI */
/* *ctx_ = rsp, 儲存棧頂位置 */
movq %rsp, (%rdi)
/* restore RSP (pointing to context-data) from RSI */
/* rsp = swap_ctx_, 這裡將當前執行棧指向了之前執行swap_in時的rsp */
movq %rsi, %rsp
/* test for flag preserve_fpu */
cmp $0, %rcx
je 2f
/* restore MMX control- and status-word */
ldmxcsr (%rsp)
/* restore x87 control-word */
fldcw 0x4(%rsp)
2:
/* prepare stack for FPU */
leaq 0x8(%rsp), %rsp
/* 將暫存器恢復到執行swap_in時的狀態 */
popq %r12 /* restrore R12 */
popq %r13 /* restrore R13 */
popq %r14 /* restrore R14 */
popq %r15 /* restrore R15 */
popq %rbx /* restrore RBX */
popq %rbp /* restrore RBP */
/* restore return-address */
/* r8 = Context::swap_in::return true */
popq %r8
/* use third arg as return-value after jump */
/* rax = this */
movq %rdx, %rax
/* use third arg as first arg in context function */
/* rdi = this */
movq %rdx, %rdi
/* indirect jump to context */
/* 接著上一次swap_in的位置繼續執行 */
jmp *%r8這個時候php和c棧都已經恢復到執行swap_in的狀態, 程式碼一路返回到zif_swoole_coroutine_create執行完畢:
bool Context::swap_in()
{
jump_fcontext(&swap_ctx_, ctx_, (intptr_t) this, true);
return true; // 從這裡開始繼續執行, 回到之前呼叫它的函式
}
inline long run()
{
...
ctx.swap_in(); // 返回
check_end(); // 檢查協程是否已經執行完畢, 執行完畢需要做清理
return cid;
}
static inline long create(coroutine_func_t fn, void* args = nullptr)
{
return (new Coroutine(fn, args))->run();
}
long PHPCoroutine::create(zend_fcall_info_cache *fci_cache, uint32_t argc, zval *argv)
{
...
return Coroutine::create(main_func, (void*) &php_coro_args);
}
PHP_FUNCTION(swoole_coroutine_create)
{
...
long cid = PHPCoroutine::create(&fci_cache, fci.param_count, fci.params);
...
RETURN_LONG(cid); // 返回協程id
}因為execute_data已經切換回main_task上的主協程opcode了, 所以下一條opcode是 'echo "a"', 相當於把sleep後面的程式碼跳過了
<?php
go(function (){
Co::sleep(1);
echo "a";
});
echo "c"; // 從這裡開始繼續執行等到一定時機, 定時器會呼叫sleep函式註冊的回撥函式sleep_timeout(呼叫時機後面會介紹), 喚醒協程繼續運轉:
// 定時事件註冊的回撥
static void sleep_timeout(swTimer *timer, swTimer_node *tnode)
{
((Coroutine *) tnode->data)->resume();
}
// 恢復整個執行環境
void Coroutine::resume()
{
...
state = SW_CORO_RUNNING; // 協程狀態改為進行中
if (sw_likely(on_resume))
{
on_resume(task); // 恢復php執行狀態
}
origin = current;
current = this;
ctx.swap_in(); // 恢復c棧
...
}
// 恢復task
void PHPCoroutine::on_resume(void *arg)
{
php_coro_task *task = (php_coro_task *) arg;
php_coro_task *current_task = get_task();
save_task(current_task); // 儲存當前任務
restore_task(task); // 恢復任務
record_last_msec(task); // 記錄時間
}zend_vm會讀取到之後的opcode 'echo "a"', 繼續執行
<?php
go(function (){
Co::sleep(1);
echo "a"; // 從這裡開始繼續執行
});
echo "c";當前回撥中的opcode被全部執行完畢之後, PHPCoroutine::main_func還會把之前註冊的defer執行一遍, 順序是FILO, 然後清理資源
void PHPCoroutine::main_func(void *arg)
{
...
if (EXPECTED(func->type == ZEND_USER_FUNCTION))
{
...
// 協程回撥函式執行完畢, 返回
zend_execute_ex(EG(current_execute_data));
}
if (task->defer_tasks)
{
std::stack<php_swoole_fci *> *tasks = task->defer_tasks;
while (!tasks->empty())
{
php_swoole_fci *defer_fci = tasks->top();
tasks->pop(); // FILO
// 呼叫defer註冊的函式
if (UNEXPECTED(sw_zend_call_function_anyway(&defer_fci->fci, &defer_fci->fci_cache) != SUCCESS))
{
...
}
}
}
// resources release
...
}main_func執行完回到Context::context_func方法, 把當前協程標記為已結束, 再做一次swap_out回到剛剛swap_in的地方, 也就是resume方法, 之後去檢查喚醒的協程有沒有執行完畢, 檢查只需要判斷end_屬性
void Context::context_func(void *arg)
{
Context *_this = (Context *) arg;
_this->fn_(_this->private_data_); // main_func(closure)返回
_this->end_ = true; // 當前協程標記為已結束
_this->swap_out(); // 切換回main c棧
}
void Coroutine::resume()
{
...
ctx.swap_in(); // 切換回這裡
check_end(); // 檢查協程是否已經結束
}
inline void check_end()
{
if (ctx.is_end())
{
close();
}
}
inline bool is_end()
{
return end_;
}close方法會清理為這個協程建立的vm_stack, 同時切回到main_task, 這時c棧和php棧都已經切換回主協程
void Coroutine::close()
{
...
state = SW_CORO_END; // 狀態改為已結束
if (on_close)
{
on_close(task);
}
current = origin;
coroutines.erase(cid); // 移除當前協程
delete this;
}
void PHPCoroutine::on_close(void *arg)
{
php_coro_task *task = (php_coro_task *) arg;
php_coro_task *origin_task = get_origin_task(task);
vm_stack_destroy(); // 銷燬vm_stack
restore_task(origin_task); // 還原main_task
}本作品採用《CC 協議》，轉載必須註明作者和本文連結