#include #include #include #include #include #include // for fprintf() #include #include #include #include #include // for close(), read() #include // for epoll_create1(), epoll_ctl(), struct epoll_event #include // for strncmp //my addition to the online guide #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fragment_dataformat.h" #include "full_event_format.h" #include "TimeoutException.h" #include "ControlledQueue.h" #define MAX_EVENTS 1024 #define MAX_QUEUE_SIZE 1000000 #define READER_THREADS 6 // That's a buffer size of 64 kB, to maximize performance without it being too big, according to testing #define BUFFER_SIZE_WORDS 16384 int min_fd = MAX_EVENTS + 1; int max_fd = 0; int exp_num = 0; std::mutex conn_mutex; std::condition_variable conn_cv; int makeSocket() { int sockfd; if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) { perror("socket failed"); exit(EXIT_FAILURE); } return sockfd; } void bindSocketPort(int server_fd, int port) { struct sockaddr_in localAddr; localAddr.sin_family = AF_INET; localAddr.sin_addr.s_addr = INADDR_ANY; localAddr.sin_port = htons(port); if (bind(server_fd, (struct sockaddr *)&localAddr, sizeof(localAddr)) < 0) { perror("bind failed"); exit(EXIT_FAILURE); } printf("FD %d bound to port %d\n", server_fd, port); } void startListening(int server_fd) { if (listen(server_fd, 20000) < 0) { perror("listen"); exit(EXIT_FAILURE); } printf("FD %d listening to new connections\n", server_fd); } int acceptConnection(int server_fd) { int client_fd; struct sockaddr_in remoteAddr; size_t addrlen = sizeof(remoteAddr); if ((client_fd = accept(server_fd, (struct sockaddr *)&remoteAddr, (socklen_t *)&addrlen)) < 0) { perror("accept"); exit(EXIT_FAILURE); } else { int flags = fcntl(client_fd, F_GETFL); fcntl(client_fd, F_SETFL, flags | O_NONBLOCK); } printf("Connection from host %s, port %d, FD %d\n", inet_ntoa(remoteAddr.sin_addr), ntohs(remoteAddr.sin_port), client_fd); return client_fd; } void acceptConnectionEpollStyle(int server_fd, int &efd) { struct sockaddr_in new_remoteAddr; int addrlen = sizeof(struct sockaddr_in); while (true) { int conn_sock = accept(server_fd, (struct sockaddr*)&new_remoteAddr, (socklen_t*)&addrlen); if (conn_sock == -1) { // All incoming connections have been processed if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) { break; } else { perror("accept"); break; } } // make new connection non-blocking int flags = fcntl(conn_sock, F_GETFL, 0); fcntl(conn_sock, F_SETFL, flags | O_NONBLOCK); // monitor new connection for read events, always in edge triggered struct epoll_event event; event.events = EPOLLIN | EPOLLEXCLUSIVE;//| EPOLLET; event.data.fd = conn_sock; // Allow epoll to monitor the new connection if (epoll_ctl(efd, EPOLL_CTL_ADD, conn_sock, &event) == -1) { perror("epoll_ctl: conn_sock"); break; } printf("Accepted epoll style connection from %s:%d from fd: %d\n", inet_ntoa(new_remoteAddr.sin_addr), ntohs(new_remoteAddr.sin_port), conn_sock); if (max_fd < conn_sock) max_fd = conn_sock; if (min_fd > conn_sock) min_fd = conn_sock; } } void term_handler(int signal) { printf("Terminated, received SIGNAL %d", signal); exit(EXIT_SUCCESS); } std::array queues_mutexes; void thradizable(int &epoll_fd, int &master_socket, std::array& queues_array, const int &th_flag, const int thread_index) { epoll_event events[MAX_EVENTS]; while (true) { if (th_flag == 1) break; // Time measurements ///auto start = std::chrono::high_resolution_clock::now(); // Returns only the sockets for which there are events //printf("Before wait\n"); int nfds = epoll_wait(epoll_fd, events, MAX_EVENTS, -1); //printf("After wait\n"); if (nfds == -1) { perror("epoll_wait"); exit(EXIT_FAILURE); } // Iterate on the sockets having events for (int i = 0; i < nfds; i++) { //printf("Tot fds = %d reading from %d\n", nfds, i); int fd = events[i].data.fd; if (fd == master_socket) { // If the activity is on the master socket, than it's a new connection request std::unique_lock conn_lk(conn_mutex); acceptConnectionEpollStyle(master_socket, epoll_fd); conn_lk.unlock(); conn_cv.notify_all(); } else if ((events[i].events & EPOLLERR) || (events[i].events & EPOLLHUP) || (!(events[i].events & EPOLLIN))) { // Than the client connection is closed, so I close it printf("Closing %d", fd); close(fd); } else { //printf("Ev trig th %d with tot b %lu\n", thread_index, part); // Than we received data from one of the monitored sockets uint32_t buffer[BUFFER_SIZE_WORDS]; //while (valread != EAGAIN) { std::unique_lock lk(queues_mutexes[fd]); ssize_t valread = recv(fd, reinterpret_cast(buffer), sizeof(buffer), 0); if (valread > 0) { //printf("[RICEVUTO]\t FROM %d\n", fd); int opt_incr = (valread % 4) ? 1 : 0; for (int q_i = 0; q_i < (valread/4) + opt_incr; q_i++) { queues_array[fd].put(buffer[q_i]); } /* bytes_read += valread; int kilos = 0; if ((kilos = bytes_read / 1024) > 0) { kBytes_read += kilos; bytes_read -= (kilos * 1024); //printf("reade bites %lu", bytes_read); }*/ } lk.unlock(); //} } } ///auto end = std::chrono::high_resolution_clock::now(); ///double time_taken = std::chrono::duration_cast(end - start).count(); //time taken in milliseconds /// /*time_taken *= 1e-6; total_time_taken += time_taken; if (total_time_taken > 3e4) { times.push_back(total_time_taken); tot_received_data.push_back(kBytes_read); break; }*/ /// } } std::vector times; std::vector timed_means; double tot_time_taken = 0; uint32_t tot_size_full_event = 0; uint32_t num_events_stored = 0; uint32_t fragments_lost = 0; int fragments_per_event = max_fd - min_fd + 1; void builder_thread (std::array &queues_array, uint32_t &runNumber) { uint32_t counter = 0; while (1) { auto start = std::chrono::high_resolution_clock::now(); if ( (max_fd < min_fd) || (exp_num <= 0) || ((max_fd - min_fd + 1) != exp_num)) { std::unique_lock conn_lk(conn_mutex); while ( (max_fd < min_fd) || (exp_num <= 0) || ((max_fd - min_fd + 1) != exp_num)) { conn_cv.wait(conn_lk); } conn_lk.unlock(); } //printf("sono fuori bro"); FullEvent fullEvent; fullEvent.headerSize = 5; fullEvent.runNumber = runNumber; fullEvent.eventNumber = counter; fullEvent.fragmentsArray = new Fragment[max_fd - min_fd + 1]; uint32_t fullEventPayloadSize = 0; double sumsize = 0; for (int i = min_fd; i <= max_fd; i++){ //std::unique_lock lk(queues_mutexes[i]); int inCurrentEventNumber = 0; while (inCurrentEventNumber != 1) { try { uint32_t starter = queues_array[i].get(); if (starter == FRAGMENT_HEADER_MARKER) { uint32_t headerSize = queues_array[i].get(); uint32_t fragmentSize = queues_array[i].get(); uint32_t* buffer = new uint32_t[fragmentSize]; buffer[0] = starter; buffer[1] = headerSize; buffer[2] = fragmentSize; for (int j = 3; j < fragmentSize; j++) { buffer[j] = queues_array[i].get(); } if (getDetectorEventNumber(buffer) < fullEvent.eventNumber) { continue; } else if (getDetectorEventNumber(buffer) == fullEvent.eventNumber) { Fragment& fragment = fullEvent.fragmentsArray[i - min_fd]; decode_fragment(buffer, fragment); inCurrentEventNumber = 1; fullEventPayloadSize += fragment.header.fragmentSize; } else { printf("È successo un cazzo di casino"); } delete [] buffer; } } catch (const TimeoutException& ex) { //printf("Ho catchato\n"); inCurrentEventNumber = 1; Fragment& fragment = fullEvent.fragmentsArray[i - min_fd]; Header& header = fragment.header; header.sourceIdentifier = i - min_fd; header.runNumber = runNumber; header.detectorEventNumber = counter; header.numberOfStatusElements = 1; uint32_t firstStatusElement = 0x0; firstStatusElement |= TIMEOUT_ERROR; header.statusElementsArray = new uint32_t[header.numberOfStatusElements]; header.statusElementsArray[0] = firstStatusElement; header.headerSize = 7 + header.numberOfStatusElements; header.fragmentSize = header.headerSize; //fragment.header = header; //fullEvent.fragmentsArray[i - min_fd] = fragment; fullEventPayloadSize += fragment.header.fragmentSize; fragments_lost++; } } sumsize += queues_array[i].size(); //lk.unlock(); } double mean = sumsize / queues_array.size(); //timed_means.push_back(mean); fullEvent.eventSize = fullEvent.headerSize + fullEventPayloadSize; //printFullEvent(fullEvent); num_events_stored = counter; counter++; auto end = std::chrono::high_resolution_clock::now(); double time_taken = std::chrono::duration_cast(end - start).count(); time_taken *= 1e-9; tot_time_taken += time_taken; //times.push_back(tot_time_taken); tot_size_full_event += fullEvent.eventSize; //printf("timee %f", tot_time_taken); if (tot_time_taken >= 30) break; } } int main(int argc, char const *argv[]) { signal(SIGTERM, term_handler); if (argc != 5) { printf("Usage: %s portNumber runNumber numClients timeout_ms\n", argv[0]); exit(EXIT_FAILURE); } int port = atoi(argv[1]); uint32_t runNumber = atoi(argv[2]); exp_num = atoi(argv[3]); int timeout_ms = atoi(argv[4]); printf("You chose parameters: pnum %d runNum %d numClients %d timeout %d", port, runNumber, exp_num, timeout_ms); printf("Start socket port %d\n", port); int master_socket; const int opt = 1; master_socket = makeSocket(); //set master socket to allow multiple connections , //this is just a good habit, it will work without this if( setsockopt(master_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(opt)) < 0 ) { perror("setsockopt"); exit(EXIT_FAILURE); } if( setsockopt(master_socket, IPPROTO_TCP, TCP_NODELAY, (char *)&opt, sizeof(opt)) < 0 ) { perror("setsockopt"); exit(EXIT_FAILURE); } bindSocketPort(master_socket, port); startListening(master_socket); int flags = fcntl(master_socket, F_GETFL, 0); fcntl(master_socket, F_SETFL, flags | O_NONBLOCK); epoll_event ev, events[MAX_EVENTS]; std::array kBytes_read_on_descr; //The atomic here is used as a flag to tell the thread to stop std::vector vThreads; //create the epoll instance int epoll_fd = epoll_create1(0); if (epoll_fd == -1) { printf("Failed to create epoll file descriptor\n"); exit(EXIT_FAILURE); } ev.data.fd = master_socket; // Reading events with edge triggered mode ev.events = EPOLLIN | EPOLLEXCLUSIVE;//| EPOLLET; // Allowing epoll to monitor the master_socket if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, master_socket, &ev) == -1){ perror("epoll_ctl"); exit(EXIT_FAILURE); } std::array vT; std::array thread_flags; // Creating the data structure and initialization with max size and timeout ez win std::array queues_array; for (auto& queue : queues_array) { queue.init(MAX_QUEUE_SIZE, timeout_ms * 1000); } for (int t_i = 0; t_i < READER_THREADS; t_i++) { thread_flags[t_i] = 0; vT[t_i] = std::thread(thradizable, std::ref(epoll_fd), std::ref(master_socket), std::ref(queues_array), std::cref(thread_flags.at(t_i)), t_i); //vT[t_i].detach(); } std::thread bdth(builder_thread, std::ref(queues_array), std::ref(runNumber)); //ADD CONSUMER THREAD bdth.join(); //for (double mean : timed_means) { // printf("%.0f ", mean); //} //sleep(30); printf("fragments lost %d\n", fragments_lost); printf("tot ev %d\n", num_events_stored); std::ofstream outfile; //Test on queue size over time //outfile.open("queue_occup_1kmean_20ms_timeout50ms_500clients.csv"); //outfile << "times;means;\n"; //auto iter_times = times.begin(); //auto iter_sizes = timed_means.begin(); //for ( ; iter_times != times.end() && iter_sizes != timed_means.end(); (++iter_times, ++iter_sizes)) { // outfile << *iter_times << ";" << *iter_sizes << ";\n"; //} outfile.open("mean_4_std_0_meantime_0ms_stddevtime_0ms_NODELAY.csv", std::ios_base::app); outfile << timeout_ms << ";" << tot_time_taken << ";" << tot_size_full_event * 4 << ";" << fragments_lost/static_cast(fragments_per_event * num_events_stored) << ";\n"; if (close(epoll_fd)) { printf("Failed to close epoll file descriptor"); exit(EXIT_FAILURE); } return 0; }