linux下打滿網路卡的頻寬和影響打滿網路卡的原因

1.打滿網路卡頻寬的定義

網路傳輸場景很多，咱們這裡用一個最常見的場景：

客戶端傳送資料，服務端接收資料，所以這裡給一個打滿的定義：

使用監控軟體發現客戶端的上行和服務端的下載頻寬用滿

2.測試軟體

客戶端：

// Client side C program to demonstrate Socket
// programming
#include <arpa/inet.h>
#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>
#include <errno.h>
#include <stdlib.h>
#include <sched.h>
#define PORT 8080

#define BUFF_SIZE (4*1024)
#define BUFF_SEND_SIZE (2*1024)


void loop_barrier(int loop_count)
{
        static count = 1;
        double sum = 0.0f;
        for(int i = 0 ; i < loop_count; i++)
                sum = sum * 1223491.3f * (double)(9401.f + i) * count++;
        
}


int main(int argc, char const* argv[])
{
    short port = atoi(argv[1]);
    int buff_send_size = atoi(argv[2]);
    int usleep_time = atoi(argv[3]);
    int loop_count = atoi(argv[4]);
    printf("%d %d %d %d\n", port, buff_send_size, usleep_time, loop_count);
    int status, valread, client_fd;
    struct sockaddr_in serv_addr;
    //char* hello = "Hello from client adsadfasdfasdfasdf0001234";
    char* hello = malloc(buff_send_size);
    for(int i = 0; i < buff_send_size; i++)
        hello[i] = '0' + i % 10;

    char buffer[BUFF_SIZE] = { 0 };
    if ((client_fd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
        printf("\n Socket creation error \n");
        return -1;
    }

    serv_addr.sin_family = AF_INET;
    //serv_addr.sin_port = htons(PORT);
    serv_addr.sin_port = htons(port);

    // Convert IPv4 and IPv6 addresses from text to binary
    // form
    if (inet_pton(AF_INET, "10.10.3.93", &serv_addr.sin_addr)
        <= 0) {
        printf(
            "\nInvalid address/ Address not supported \n");
        return -1;
    }

    if ((status
        = connect(client_fd, (struct sockaddr*)&serv_addr,
                sizeof(serv_addr)))
        < 0) {
        printf("\nConnection Failed \n");
        return -1;
    }

    while(1)
    {
        //usleep(usleep_time);
        //sched_yield();
        loop_barrier(loop_count);
        send(client_fd, hello, strlen(hello), 0);
        continue;
        //printf("[send] %s\n", hello);
        valread = read(client_fd, buffer, sizeof(buffer)); // subtract 1 for the null
        if (valread <= 0)
        {
            printf("recv %d %s", valread, errno);
            exit(-2);    
        }
        buffer[valread] = '\0';
        //printf("[recv] %s\n", buffer);
        //sleep(1);
    }

    // closing the connected socket
    close(client_fd);
    return 0;
}

測試命令：

nohup ./client 8080 2048 0 1000 &（連線服務端埠8080，一次傳送buffer是2048位元組，usleep 0，算術計算loop迴圈1000）
nohup ./client 8081 2048 0 1000 &
nohup ./client 8082 2048 0 1000 &

服務端：

// Server side C program to demonstrate Socket
// programming
#include <netinet/in.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>
#include <errno.h>
#include <sched.h> 

#define PORT 8080

#define BUFF_SIZE (4*1024)

void loop_barrier(int loop_count)
{
    static count = 1;
    double sum = 0.0f;
    for(int i = 0 ; i < loop_count; i++)
        sum = sum * 1223491.3f * (double)(9401.f + i) * count++;
        
}

int main(int argc, char const* argv[])
{
    short port = atoi(argv[1]);
    int usleep_time = atoi(argv[2]);
    int loop_count = atoi(argv[3]);
    printf("%d %d %d\n",port, usleep_time, loop_count);
    int server_fd, new_socket;
    ssize_t valread;
    struct sockaddr_in address;
    int opt = 1;
    socklen_t addrlen = sizeof(address);
    char buffer[BUFF_SIZE] = { 0 };
    char* hello = "Hello from server asdfasdfsadfasdfasdf009999999999";

    // Creating socket file descriptor
    if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
        perror("socket failed");
        exit(EXIT_FAILURE);
    }

    // Forcefully attaching socket to the port 8080
    if (setsockopt(server_fd, SOL_SOCKET,
                SO_REUSEADDR | SO_REUSEPORT, &opt,
                sizeof(opt))) {
        perror("setsockopt");
        exit(EXIT_FAILURE);
    }
    address.sin_family = AF_INET;
    address.sin_addr.s_addr = INADDR_ANY;
    //address.sin_port = htons(PORT);
    address.sin_port = htons(port);

    // Forcefully attaching socket to the port 8080
    if (bind(server_fd, (struct sockaddr*)&address,
            sizeof(address))
        < 0) {
        perror("bind failed");
        exit(EXIT_FAILURE);
    }
    if (listen(server_fd, 3) < 0) {
        perror("listen");
        exit(EXIT_FAILURE);
    }
    if ((new_socket
        = accept(server_fd, (struct sockaddr*)&address,
                &addrlen))
        < 0) {
        perror("accept");
        exit(EXIT_FAILURE);
    }

    while(1)
    {
        valread = read(new_socket, buffer,
                sizeof(buffer)); // subtract 1 for the null
        //usleep(usleep_time);
        //sched_yield();  
        loop_barrier(loop_count);
        if (valread <= 0)
                {
                        printf("recv %d %s", valread, errno);
                        exit(-2);
                }
        continue;
        buffer[valread] = '\0';
    //    printf("[recv] %s\n", buffer);
        send(new_socket, hello, strlen(hello), 0);
    //    printf("[send] %s\n", hello);
    //    sleep(1);
    }
    // terminator at the end
    //printf("%s\n", buffer);
    //send(new_socket, hello, strlen(hello), 0);
    //printf("Hello message sent\n");

    // closing the connected socket
    close(new_socket);
    // closing the listening socket
    close(server_fd);
    return 0;
}

測試命令：

nohup ./server 8080 0 1000 &（監聽埠8080，usleep 0，算術計算loop迴圈1000）
nohup ./server 8081 0 1000 &
nohup ./server 8082 0 1000 &

3.測試結果：

說明：因為是在一臺宿主機進行測試，所以其實不走宿主機的網路卡，走的是虛擬機器的網路卡，但是對網路敏感度更高，對比實驗更有說服力：

usleep 100 結果:300Mb
usleep 10 結果:600Mb
usleep 1 結果:690Mb
empty loop 結果:20Gb
usleep 0 結果:708Mb (deactivate_task呼叫)
sched_yield 結果:10Gb
loop 10000 calculate multipie_hard 結果: 2.7Gb (用了static 避免走快取)
loop 1000 calculate multipie_hard 結果: 8.2Gb (用了static 避免走快取)
loop 100 calculate multipie_hard 結果: 15Gb (用了static 避免走快取)

4.初步分析結論：

empty loop ： 很容易就把網路頻寬打滿。

usleep 0 ：網路頻寬完全利用不上，猜測是deactivate_task等呼叫，引發更大的耗時。

sched_yield ：出讓cpu，網路頻寬也能打的很高。

loop 10000 calculate multipie_hard : 就算進行很耗時的cpu計算，網路頻寬依然可以打的很高。

5.總結：

所以網路卡頻寬打滿，不僅僅是和其他操作耗時有關，還跟作業系統的一些進行核心態切換的api有關（例如usleep）