基於程式池的多程式伺服器通訊

程式碼來源：遊雙的linux高效能伺服器程式設計

主程式建立一個程式池，當有客戶端傳送請求時，主程式選擇一個程式與該客戶端進行通訊，為了簡單期間，去掉了一些訊號處理程式碼，其中訊號的用法可以參考基於多程式的網路聊天程式

伺服器程式碼如下：

#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <assert.h>
#include <stdio.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <fcntl.h>
#include <stdlib.h>
#include <sys/epoll.h>
#include <signal.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include<iostream>
using namespace std;

/* 描述一個子程式的類 */
class process
{
	public:
		process():m_pid(-1){}
		pid_t m_pid;//子程式的PID
		int m_pipefd[2];//父子程式通訊的管道
};

template<typename T>
class processpool
{
	public:
		/* 單例模式,以保證最多建立一個processpoll例項，這是程式正確處理訊號的必要條件 */
		static processpool<T>* create(int listenfd,int process_number = 8)
		{
			if(!m_instance)m_instance = new processpool<T>(listenfd,process_number);
			return m_instance;
		}
		~processpool()
		{
			delete[] m_sub_process;
		}
		void run();
	private:
		processpool(int listenfd,int process_number = 8);
		void setup_sig_pipe();
		void run_parent();
		void run_child();

		static const int MAX_PROCESS_NUMBRE = 16;//程式池允許的最大子程式數量
		static const int USER_PRE_PROCESS = 65535;//每個程式最多能處理的客戶數量
		static const int MAX_EVENT_NUMBRE = 10000;//epoll最多能處理的事件數
		int m_process_number;//程式池中的程式總數
		int m_idx;//子程式在程式池中的序號
		int m_epollfd;
		int m_listenfd;//監聽socket
		bool m_stop;
		process* m_sub_process;//儲存所有子程式的描述資訊
		static processpool<T>* m_instance;//程式池靜態實力
};
template<typename T> processpool<T>* processpool<T>::m_instance = NULL;

static int setnonblock(int fd)
{
	int old_flag = fcntl(fd,F_GETFL);
	assert(fcntl(fd,F_SETFL,old_flag | O_NONBLOCK) >= 0);
	return old_flag;
}

static void addfd(int epollfd,int fd)
{
	epoll_event ee;
	ee.data.fd = fd;
	ee.events = EPOLLIN | EPOLLET;
	epoll_ctl(epollfd,EPOLL_CTL_ADD,fd,&ee);
	setnonblock(fd);
}

static void removefd(int epollfd,int fd)
{
	epoll_ctl(epollfd,EPOLL_CTL_DEL,fd,0);
	close(fd);
}

void addsig(int sig,void(handler)(int),bool restart = true)
{
	struct sigaction sa;
	memset(&sa,'\0',sizeof(sa));
	sa.sa_handler = handler;
	if(restart)sa.sa_flags | SA_RESTART;
	sigfillset(&sa.sa_mask);
	assert(sigaction(sig,&sa,NULL) != -1);
}

/* 建立一個程式池，listenfd是監聽套接字，由父程式負責監聽，然後通知子程式處理 */
template<typename T>
processpool<T>::processpool(int listenfd,int process_number):m_listenfd(listenfd),m_process_number(process_number),m_idx(-1),m_stop(false)
{
	assert(process_number > 0 && process_number <= MAX_PROCESS_NUMBRE);
	m_sub_process = new process[process_number];
	int i;
	/* 建立process_number個子程式，並建立他們和父程式之間的通訊管道 */
	for(i = 0;i < process_number ; ++i)
	{
		int ret = socketpair(AF_UNIX,SOCK_STREAM,0,m_sub_process[i].m_pipefd);
		assert( ret == 0);
		m_sub_process[i].m_pid = fork();
		assert(m_sub_process[i].m_pid >= 0);
		if(m_sub_process[i].m_pid > 0)//父程式
		{
			close(m_sub_process[i].m_pipefd[1]);
			continue;//繼續建立子程式
		}
		else//子程式
		{
			close(m_sub_process[i].m_pipefd[0]);
			m_idx = i;//自己在程式池的下標
			break;
		}
	}
}

/* 父程式中的m_idx == -1,子程式中的m_idx為程式池的下標 */
template<typename T>
void processpool<T>::run()
{
	if(m_idx != -1) run_child();
	else run_parent();
}

/* 統一事件源 */
template<typename T>
void processpool<T>::setup_sig_pipe()
{
	m_epollfd = epoll_create(1000);
	assert(m_epollfd != -1);

	addsig(SIGPIPE,SIG_IGN);//防止向關閉的客戶端寫資料導致伺服器程式的終止
}

/* 父程式的執行函式 */
template<typename T>
void processpool<T>::run_parent()
{
	setup_sig_pipe();
	addfd(m_epollfd,m_listenfd);//父程式監聽m_listenfd
	epoll_event events[MAX_EVENT_NUMBRE];
	int sub_process_counter = 0;
	char new_conn = 'c';
	int ret = -1;
	while( !m_stop)
	{
		int number = epoll_wait(m_epollfd,events,MAX_EVENT_NUMBRE,-1);
		if((number < 0) && (errno != EINTR))
		{
			printf("epoll failure\n");
			break;
		}
		int i;
		for(i = 0;i < number;i++)
		{
			int sockfd = events[i].data.fd;
			//如果有新連線到來，就選擇一個子程式進行處理
			if(sockfd == m_listenfd)
			{
				int index = sub_process_counter;
				do
				{
					if(m_sub_process[index].m_pid != -1)break;
					index = (index + 1) % m_process_number;
				}
				while(index != sub_process_counter);
				if(m_sub_process[index].m_pid == -1)//沒有子程式
				{
					m_stop =  true;
					break;
				}
				sub_process_counter = (index + 1) % m_process_number;
				send(m_sub_process[index].m_pipefd[0],&new_conn,sizeof(new_conn),0);
				printf("send request to child %d\n",index);
			}
		}
	}
	close(m_epollfd);
}

/*  子程式的執行函式 */ 
template<typename T>
void processpool<T>::run_child()
{
	setup_sig_pipe();
	int pipefd = m_sub_process[m_idx].m_pipefd[1];//每個子程式通過其在程式池中的序號找到與父程式的通訊管道
	addfd(m_epollfd,pipefd);
	epoll_event events[MAX_EVENT_NUMBRE];
	T* users = new T[USER_PRE_PROCESS];//子程式處理的客戶端物件池
	int ret;

	while( !m_stop)
	{
		int number = epoll_wait(m_epollfd,events,MAX_EVENT_NUMBRE,-1);
		if(number < 0 && errno != EINTR)
		{
			printf("epoll failure\n");
			break;
		}
		int i;
		for(i = 0;i < number;++i)
		{
			int sockfd = events[i].data.fd;
			//父程式傳送來的新連線的通知
			if(sockfd == pipefd && events[i].events & EPOLLIN)
			{
				char client;
				ret = recv(sockfd,&client,sizeof(client),0);
				if((ret < 0 && errno != EAGAIN )||ret == 0)continue;
				struct sockaddr_in client_address;
				socklen_t client_length = sizeof(client_address);
				int connfd = accept(m_listenfd,(sockaddr*)&client_address,&client_length);
				if(connfd < 0)
				{
					printf("errno is : %d \n",errno);
					continue;
				}
				addfd(m_epollfd,connfd);
				users[connfd].init(m_epollfd,connfd,client_address);//呼叫模板T的初始化方法
			}
			else if(events[i].events & EPOLLIN)//客戶端發來資料
			{
				users[sockfd].process();
			}
		}
	}
	delete[] users;
	users = NULL;
	close(pipefd);
	close(m_epollfd);
}

//回射類
class echo_conn
{
	public:
		void init(int epollfd,int sockfd,const sockaddr_in& client_addr)
		{
			m_epollfd = epollfd;
			m_sockfd = sockfd;
			m_address = client_addr;
			memset(m_buf,'\0',BUFFER_SIZE);
			m_read_idx = 0;
		}
		void process()
		{
			while(true)
			{
				int ret = recv(m_sockfd,m_buf,BUFFER_SIZE-1,0);
				if(ret < 0)
				{
					if(errno != EAGAIN)removefd(m_epollfd,m_sockfd);
					break;
				}
				else if(ret == 0)//客戶端關閉
				{
					removefd(m_epollfd,m_sockfd);
					break;
				}
				else 
				{
					printf("client data is : %s\n",m_buf); 
					send(m_sockfd,m_buf,ret,0);
				}
			}
		}
	private:
		static const int BUFFER_SIZE = 1024;
		static int m_epollfd;
		int m_sockfd;
		sockaddr_in m_address;
		char m_buf[BUFFER_SIZE];
		int m_read_idx;//標記讀緩衝區已經讀入的客戶資料的最後一個位元組的下一個位置
};
int echo_conn::m_epollfd = 1;

int main(int argc,char* argv[])
{
	if(argc != 3)
	{
		printf("usage %s server_ip server_port \n",basename(argv[0]));
		return -1;
	}

	struct sockaddr_in address;
	memset(&address,0,sizeof(address));
	address.sin_family = AF_INET;
	inet_pton(AF_INET,argv[1],&address.sin_addr);
	address.sin_port = htons(atoi(argv[2]));
	int listenfd = socket(AF_INET,SOCK_STREAM,0);
	assert(listenfd != -1);
	int use = 1;
	int ret = setsockopt(listenfd,SOL_SOCKET,SO_REUSEADDR,&use,sizeof(use));
	assert(ret == 0);
	ret = bind(listenfd,(const sockaddr*)&address,sizeof(address));
	assert(ret != -1);
	ret = listen(listenfd,100);
	assert(ret != -1);
	
	processpool<echo_conn>* pool = processpool<echo_conn>::create(listenfd);//建立程式池
	if(pool)
	{
		pool -> run();//父子程式返回後都會呼叫該方法
		delete pool;
	}

	close(listenfd);
	return 0;
}

客戶端程式碼如下：

#define _GNU_SOURCE 1
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <assert.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <poll.h>
#include <fcntl.h>

#define BUFFER_SIZE 64

int main( int argc, char* argv[] )
{
    if( argc <= 2 )
    {
        printf( "usage: %s ip_address port_number\n", basename( argv[0] ) );
        return 1;
    }
    const char* ip = argv[1];
    int port = atoi( argv[2] );

    struct sockaddr_in server_address;
    bzero( &server_address, sizeof( server_address ) );
    server_address.sin_family = AF_INET;
    inet_pton( AF_INET, ip, &server_address.sin_addr );
    server_address.sin_port = htons( port );

    int sockfd = socket( PF_INET, SOCK_STREAM, 0 );
    assert( sockfd >= 0 );
    if ( connect( sockfd, ( struct sockaddr* )&server_address, sizeof( server_address ) ) < 0 )
    {
        printf( "connection failed\n" );
        close( sockfd );
        return 1;
    }

    pollfd fds[2];
    fds[0].fd = 0;
    fds[0].events = POLLIN;
    fds[0].revents = 0;
    fds[1].fd = sockfd;
    fds[1].events = POLLIN | POLLRDHUP;
    fds[1].revents = 0;
    char read_buf[BUFFER_SIZE];
    int pipefd[2];
    int ret = pipe( pipefd );
    assert( ret != -1 );

    while( 1 )
    {
        ret = poll( fds, 2, -1 );
        if( ret < 0 )
        {
            printf( "poll failure\n" );
            break;
        }

        if( fds[1].revents & POLLRDHUP )
        {
            printf( "server close the connection\n" );
            break;
        }
        else if( fds[1].revents & POLLIN )
        {
            memset( read_buf, '\0', BUFFER_SIZE );
            int len = recv( fds[1].fd, read_buf, BUFFER_SIZE-1, 0 );
			int i;
		    for(i = 0;i<len;i++)printf("%c",read_buf[i]);
        }

        if( fds[0].revents & POLLIN )
        {
            ret = splice( 0, NULL, pipefd[1], NULL, 32768, SPLICE_F_MORE | SPLICE_F_MOVE );
            ret = splice( pipefd[0], NULL, sockfd, NULL, 32768, SPLICE_F_MORE | SPLICE_F_MOVE );
        }
    }
    
    close( sockfd );
    return 0;
}