SimpleEventBuilder/testing/is_epoll_really_thsafe/event_builder_epoll.cxx

#include <cerrno>
#include <cstddef>
#include <cinttypes>
#include <cstdint>
#include <cstdio>     // for fprintf()
#include <functional>
#include <iostream>
#include <queue>
#include <ratio>
#include <unistd.h>    // for close(), read()
#include <sys/epoll.h> // for epoll_create1(), epoll_ctl(), struct epoll_event
#include <cstring>    // for strncmp

//my addition to the online guide
#include <csignal>
#include <cstdlib>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <sys/socket.h>

#include <fcntl.h>
#include <utility>
#include <vector>
#include <chrono>
#include <fstream>

#include <thread>
#include <mutex>
#include <atomic>
#include <array>


#define MAX_EVENTS 20000

int makeSocket() {
    int sockfd;
    if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
        perror("socket failed");
        exit(EXIT_FAILURE);
    }
    return sockfd;
}

void bindSocketPort(int server_fd, int port) {
    struct sockaddr_in localAddr;
    localAddr.sin_family = AF_INET;
    localAddr.sin_addr.s_addr = INADDR_ANY;
    localAddr.sin_port = htons(port);

    if (bind(server_fd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) {
        perror("bind failed");
        exit(EXIT_FAILURE);
    }
    printf("FD %d bound to port %d\n", server_fd, port);
}

void startListening(int server_fd) {
    if (listen(server_fd, 20000) < 0) {
        perror("listen");
        exit(EXIT_FAILURE);
    }
    printf("FD %d listening to new connections\n", server_fd);
}

int acceptConnection(int server_fd) {
    int client_fd;
    struct sockaddr_in remoteAddr;

    size_t addrlen = sizeof(remoteAddr);
    if ((client_fd = accept(server_fd, (struct sockaddr *)&remoteAddr, (socklen_t *)&addrlen)) < 0) {
        perror("accept");
        exit(EXIT_FAILURE);
    } else {
        int flags = fcntl(client_fd, F_GETFL);
        fcntl(client_fd, F_SETFL, flags | O_NONBLOCK);
    }
    printf("Connection from host %s, port %d, FD %d\n", inet_ntoa(remoteAddr.sin_addr), ntohs(remoteAddr.sin_port), client_fd);
    return client_fd;
}

void acceptConnectionEpollStyle(int server_fd, int &efd) {
    struct sockaddr_in new_remoteAddr;
    int addrlen = sizeof(struct sockaddr_in);

    while (true) {
        int conn_sock = accept(server_fd, (struct sockaddr*)&new_remoteAddr, (socklen_t*)&addrlen);

        if (conn_sock == -1) {
            // All incoming connections have been processed
            if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) {
                break;
            } else {
                perror("accept");
                break;
            }
        }

        // make new connection non-blocking
        int flags = fcntl(conn_sock, F_GETFL, 0);
        fcntl(conn_sock, F_SETFL, flags | O_NONBLOCK);

        // monitor new connection for read events, always in edge triggered
        struct epoll_event event;
        event.events = EPOLLIN | EPOLLEXCLUSIVE;//| EPOLLET;
        event.data.fd = conn_sock;

        // Allow epoll to monitor the new connection
        if (epoll_ctl(efd, EPOLL_CTL_ADD, conn_sock, &event) == -1) {
            perror("epoll_ctl: conn_sock");
            break;
        }
        printf("Accepted epoll style connection from %s:%d from fd: %d\n", inet_ntoa(new_remoteAddr.sin_addr), ntohs(new_remoteAddr.sin_port), conn_sock);

    }
}

void term_handler(int signal) {
    printf("Terminated, received SIGNAL %d", signal);
    exit(EXIT_SUCCESS);
}

std::atomic<uint64_t> grandtotal_kb;

std::queue<std::pair<int, uint64_t>> data_queue;
std::mutex queue_mutex;


void thradizable(int &epoll_fd, int &master_socket, const int &th_flag, const int thread_index) {
    epoll_event events[MAX_EVENTS];
    uint64_t bytes_read = 0;
    uint64_t kBytes_read = 0;

    while (true) {
        if (th_flag == 1) {
            grandtotal_kb += kBytes_read;
            break;
        }
        // Time measurements
        ///auto start = std::chrono::high_resolution_clock::now();

        // Returns only the sockets for which there are events
        //printf("Before wait\n");
        int nfds = epoll_wait(epoll_fd, events, MAX_EVENTS, -1);
        //printf("After wait\n");
        if (nfds == -1) {
            perror("epoll_wait");
            exit(EXIT_FAILURE);
        }

        

        // Iterate on the sockets having events
        for (int i = 0; i < nfds; i++) {
            //printf("Tot fds = %d reading from %d\n", nfds, i);
            int fd = events[i].data.fd;
            if (fd == master_socket) {
                // If the activity is on the master socket, than it's a new connection request
                acceptConnectionEpollStyle(master_socket, epoll_fd);

            } else if ((events[i].events & EPOLLERR) || (events[i].events & EPOLLHUP) || (!(events[i].events & EPOLLIN))) {
                // Than the client connection is closed, so I close it
                printf("Closing %d", fd);
                close(fd);
            } else {
                uint64_t part = grandtotal_kb;
                //printf("Ev trig th %d with tot b %lu\n", thread_index, part);
                // Than we received data from one of the monitored sockets
                uint64_t buffer[8];
                int valread = 0;
                //while (valread != EAGAIN) {
                    std::unique_lock<std::mutex> lk(queue_mutex);
                    valread = recv(fd, reinterpret_cast<char*>(buffer), sizeof(buffer), 0);
                    if (valread > 0) {
                        //printf("[RICEVUTO]\t FROM %d\n", fd);

                        int opt_incr = (valread % 8) ? 1 : 0;
                        for (int q_i = 0; q_i < (valread/8) + opt_incr; q_i++) {
                            data_queue.push(std::pair<int, uint64_t>(thread_index, buffer[q_i]));
                        }
                        grandtotal_kb += valread;
                        

                        /*
                        bytes_read += valread;
                        int kilos = 0;
                        if ((kilos = bytes_read / 1024) > 0) {
                            kBytes_read += kilos;
                            bytes_read -= (kilos * 1024);
                            //printf("reade bites %lu", bytes_read);
                        }*/
                    }
                    lk.unlock();
                //}

            }
        }

        ///auto end = std::chrono::high_resolution_clock::now();
        ///double time_taken = std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count();
        //time taken in milliseconds
        ///
        /*time_taken *= 1e-6;
        total_time_taken += time_taken;

        if (total_time_taken > 3e4) {
            times.push_back(total_time_taken);
            tot_received_data.push_back(kBytes_read);
            break;
        }*/
        ///
    }

}


std::vector<uint64_t> popped;


void printer_thread() {
    while (1) {
        std::unique_lock<std::mutex> lk(queue_mutex);
        if (!data_queue.empty()) {
            std::pair<int, uint64_t> element = data_queue.front();
            data_queue.pop();
            popped.push_back(element.second);
            //printf("Element: %lu from thread %d\n", element.second, element.first);

            if (element.second == 999) {
                int fail_flag = 0;
                for (int i = 1; i < 999; i++) {
                    if (popped[i] == i) continue;
                    else fail_flag = 1;
                }
                
                if (fail_flag == 1) printf("FAILURE\n");
                else printf("success\n");
            }
        }
        lk.unlock();
        usleep(10);
    }
}


int main(int argc, char const *argv[]) {

    signal(SIGTERM, term_handler);


    if (argc != 2) {
        printf("Usage: %s portNumber \n", argv[0]);
        exit(EXIT_FAILURE);
    }
    int port = atoi(argv[1]);
    printf("Start socket port %d\n", port);
 
    int master_socket;
    const int opt = 1;


    master_socket = makeSocket();

    //set master socket to allow multiple connections , 
    //this is just a good habit, it will work without this 
    if( setsockopt(master_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, 
          sizeof(opt)) < 0 )  
    {  
        perror("setsockopt");  
        exit(EXIT_FAILURE);  
    }

    bindSocketPort(master_socket, port);
    startListening(master_socket);

    int flags = fcntl(master_socket, F_GETFL, 0);
    fcntl(master_socket, F_SETFL, flags | O_NONBLOCK);

    epoll_event ev, events[MAX_EVENTS];
    std::array<uint64_t, MAX_EVENTS> kBytes_read_on_descr;

    //The atomic here is used as a flag to tell the thread to stop
    std::vector<std::thread> vThreads;

    //create the epoll instance
    int epoll_fd = epoll_create1(0);
    if (epoll_fd == -1) {
        printf("Failed to create epoll file descriptor\n");
        exit(EXIT_FAILURE);
    }

    ev.data.fd = master_socket;
    // Reading events with edge triggered mode
    ev.events = EPOLLIN | EPOLLEXCLUSIVE;//| EPOLLET;

    // Allowing epoll to monitor the master_socket
    if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, master_socket, &ev) == -1){
        perror("epoll_ctl");
        exit(EXIT_FAILURE);
    }
    
    std::array<std::thread, 2> vT;
    std::array<int, 2> thread_flags;
    
    for (int t_i = 0; t_i < 2; t_i++) {
        thread_flags[t_i] = 0;
        vT[t_i] = std::thread(thradizable, std::ref(epoll_fd), std::ref(master_socket), std::cref(thread_flags.at(t_i)), t_i);
    }

    std::thread printer(printer_thread);


    printer.join();





    if (close(epoll_fd)) {
        printf("Failed to close epoll file descriptor");
        exit(EXIT_FAILURE);
    }

    return 0;
}