Index: example/io/cat.c
===================================================================
--- example/io/cat.c	(revision f0d67e5abea7221dccd4183dd58ccc6cdbee32fc)
+++ example/io/cat.c	(revision f0d67e5abea7221dccd4183dd58ccc6cdbee32fc)
@@ -0,0 +1,78 @@
+/*
+This is a simple "cat" example that uses io_uring in IORING_SETUP_IOPOLL mode.
+It demonstrates the bare minimum needed to use io_uring in polling mode.
+It uses liburing for simplicity.
+*/
+
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <fcntl.h>
+#include <liburing.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+struct io_uring ring;
+
+__attribute__((aligned(1024))) char data[1024];
+
+int main(int argc,  char * argv[]) {
+      if(argc != 2) {
+            printf("usage:   %s FILE - prints file to console.\n", argv[0]);
+            return 1;
+      }
+
+      int fd = open(argv[1], O_DIRECT);
+      if(fd < 0) {
+            printf("Could not open file %s.\n", argv[1]);
+            return 2;
+      }
+
+      /* prep the array */
+      struct iovec iov = { data, 1024 };
+
+      /* init liburing */
+      io_uring_queue_init(256, &ring, IORING_SETUP_IOPOLL);
+
+      /* declare required structs */
+      struct io_uring_sqe * sqe;
+      struct io_uring_cqe * cqe;
+
+      /* get an sqe and fill in a READV operation */
+      sqe = io_uring_get_sqe(&ring);
+      io_uring_prep_readv(sqe, fd, &iov, 1, 0);
+      // io_uring_prep_read(sqe, fd, data, 1024, 0);
+
+      sqe->user_data = (uint64_t)(uintptr_t)data;
+
+      /* tell the kernel we have an sqe ready for consumption */
+      io_uring_submit(&ring);
+
+      /* wait for the sqe to complete */
+      int ret = io_uring_wait_cqe(&ring, &cqe);
+
+      /* read and process cqe event */
+      if(ret == 0) {
+            char * out = (char *)(uintptr_t)cqe->user_data;
+            signed int len = cqe->res;
+            io_uring_cqe_seen(&ring, cqe);
+
+            if(len > 0) {
+                  printf("%.*s", len, out);
+            }
+            else if( len < 0 ) {
+                  fprintf(stderr, "readv/read returned error : %s\n", strerror(-len));
+            }
+      }
+      else {
+            printf("%d\n", ret);
+            io_uring_cqe_seen(&ring, cqe);
+      }
+
+      io_uring_queue_exit(&ring);
+
+      close(fd);
+}
Index: example/io/filereader.c
===================================================================
--- example/io/filereader.c	(revision f0d67e5abea7221dccd4183dd58ccc6cdbee32fc)
+++ example/io/filereader.c	(revision f0d67e5abea7221dccd4183dd58ccc6cdbee32fc)
@@ -0,0 +1,122 @@
+/*
+This is a file reading example that users io_uring in non-blocking mode.
+It demonstrates the bare minimum needed to use io_uring.
+It also optionally pre-registers the file descriptors (and a pipe, just to show it works).
+It uses liburing for simplicity.
+*/
+
+
+#include <errno.h>
+#include <fcntl.h>
+#include <liburing.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+int main(int argc, char * argv[]) {
+	if(argc != 3 && argc != 4) {
+            printf("usage:   %s FILE TIMES [fixed] - read FILE from disk TIMES times\n", argv[0]);
+            return EXIT_FAILURE;
+      }
+
+	bool fixed = false;
+	if(argc == 4) {
+		fixed = 0 == strcmp(argv[3], "fixed");
+	}
+
+      int times = atoi( argv[2] );
+      if(times <= 0) {
+            printf("Invalid number of times %d (from %s).\n", times, argv[2]);
+            return EXIT_FAILURE;
+      }
+
+      int fd = open(argv[1], 0);
+      if(fd < 0) {
+            printf("Could not open file %s.\n", argv[1]);
+            return EXIT_FAILURE;
+      }
+
+	int rfd = fd;
+
+	/* prep the array */
+      char data[100];
+      struct iovec iov = { data, 100 };
+
+	/* init liburing */
+	struct io_uring ring;
+      io_uring_queue_init(256, &ring, 0);
+
+	int pipefds[2];
+	if(fixed) {
+		int ret = pipe(pipefds);
+		if( ret < 0 ) {
+			printf("Pipe Error : %s\n", strerror( errno ));
+			return EXIT_FAILURE;
+		}
+		rfd = 0;
+		int fds[] = {
+			fd, pipefds[0], pipefds[1]
+		};
+		int cnt = sizeof(fds) / sizeof(fds[0]);
+		printf("Registering %d files as fixed\n", cnt);
+		ret = io_uring_register_files(&ring, fds, cnt);
+		if( ret < 0 ) {
+			printf("Register Error : %s\n", strerror( -ret ));
+			return EXIT_FAILURE;
+		}
+	}
+
+      /* declare required structs */
+	printf("Reading %s(%d) %d times\n", argv[1], fd, times);
+	size_t counter = 0;
+	for(int i = 0; i < times; i++) {
+		/* get an sqe and fill in a READV operation */
+	      struct io_uring_sqe * sqe = io_uring_get_sqe(&ring);
+		io_uring_prep_readv(sqe, rfd, &iov, 1, 0);
+		if(fixed) {
+			sqe->flags = IOSQE_FIXED_FILE;
+		}
+
+		/* tell the kernel we have an sqe ready for consumption */
+      	io_uring_submit(&ring);
+
+		/* poll the cq and count how much polling we did */
+		while(true) {
+			struct io_uring_cqe * cqe = NULL;
+			/* wait for the sqe to complete */
+			int ret = io_uring_wait_cqe_nr(&ring, &cqe, 0);
+
+			/* read and process cqe event */
+			switch(ret) {
+			case 0:
+				if( cqe->res < 0 ) {
+					printf("Completion Error : %s\n", strerror( -cqe->res ));
+					return EXIT_FAILURE;
+				}
+				io_uring_cqe_seen(&ring, cqe);
+				goto LOOP;
+			case -EAGAIN:
+				counter++;
+				break;
+			default:
+				printf("Wait Error : %s\n", strerror( -ret ));
+				return EXIT_FAILURE;
+			}
+		}
+
+		LOOP:;
+	}
+
+	printf("%zu\n", counter);
+
+      io_uring_queue_exit(&ring);
+
+      close(fd);
+
+	if(fixed) {
+		close(pipefds[0]);
+		close(pipefds[1]);
+	}
+}
Index: example/io/simple/client.c
===================================================================
--- example/io/simple/client.c	(revision f0d67e5abea7221dccd4183dd58ccc6cdbee32fc)
+++ example/io/simple/client.c	(revision f0d67e5abea7221dccd4183dd58ccc6cdbee32fc)
@@ -0,0 +1,85 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <netdb.h>
+#include <unistd.h>
+
+#include <sys/types.h>
+#include <sys/socket.h>
+
+int main(int argc, char * argv[]) {
+      if(argc != 2) {
+            printf("usage:    %s portnumber\n", argv[0]);
+            exit( EXIT_FAILURE );
+      }
+      int port = atoi(argv[1]);
+      if(port < 1) {
+            printf("Invalid port : %d (from %s)\n", port, argv[1]);
+            exit( EXIT_FAILURE );
+      }
+
+      int sock = socket(AF_INET, SOCK_STREAM, 0);
+      if(sock < 0) {
+            perror( "socket" );
+            exit( EXIT_FAILURE );
+      }
+
+      struct hostent * server = gethostbyname("localhost");
+      if(server == NULL) {
+            perror("localhost not found");
+            exit( EXIT_FAILURE );
+      }
+
+      struct sockaddr_in serv_addr;
+      memset(&serv_addr, 0, sizeof(serv_addr));
+      serv_addr.sin_family = AF_INET;
+      memcpy(&serv_addr.sin_addr.s_addr, server->h_addr, server->h_length);
+      serv_addr.sin_port = htons(port);
+
+      int ret = connect(sock, (struct sockaddr *)&serv_addr, sizeof(serv_addr));
+      if(ret < 0) {
+            perror( "connect" );
+            exit( EXIT_FAILURE );
+      }
+
+      char buffer[256];
+      struct iovec iov = { buffer, 0 };
+      struct msghdr msg;
+      msg.msg_name = NULL;
+      msg.msg_namelen = 0;
+      msg.msg_control = NULL;
+      msg.msg_controllen = 0;
+      msg.msg_iov = &iov;
+      msg.msg_iovlen = 1;
+      msg.msg_flags = 0;
+
+
+      int rd;
+      while(0 != (rd = read(STDIN_FILENO, buffer, 256))) {
+            if(rd < 0) {
+                  perror( "read" );
+                  exit( EXIT_FAILURE );
+            }
+
+            iov.iov_len = rd;
+            int sent = sendmsg(sock, &msg, 0);
+            if( sent < 0 ) {
+                  perror( "read" );
+                  exit( EXIT_FAILURE );
+            }
+
+            if(sent != rd) {
+                  printf("Expected to send %d bytes, sent %d\n", rd, sent);
+                  exit( EXIT_FAILURE );
+            }
+      }
+
+      ret = close(sock);
+      if(ret < 0) {
+            perror( "close" );
+            exit( EXIT_FAILURE );
+      }
+
+      exit( EXIT_SUCCESS );
+}
Index: example/io/simple/server.c
===================================================================
--- example/io/simple/server.c	(revision f0d67e5abea7221dccd4183dd58ccc6cdbee32fc)
+++ example/io/simple/server.c	(revision f0d67e5abea7221dccd4183dd58ccc6cdbee32fc)
@@ -0,0 +1,143 @@
+/*
+This is a simple server that users io_uring in blocking mode.
+It demonstrates the bare minimum needed to use io_uring.
+It uses liburing for simplicity.
+*/
+
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <unistd.h>
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+
+#include <liburing.h>
+
+struct io_uring ring;
+
+char data[256];
+struct iovec iov = { data, 256 };
+struct msghdr msg = { (void *)"", 0, &iov, 1, NULL, 0, 0 };
+static void async_read(int sock) {
+	/* get an sqe and fill in a READ operation */
+      struct io_uring_sqe * sqe = io_uring_get_sqe(&ring);
+      io_uring_prep_recvmsg(sqe, sock, &msg, 0);
+      sqe->user_data = 0;
+
+      /* tell the kernel we have an sqe ready for consumption */
+      int ret = io_uring_submit(&ring);
+      assert(ret == 1);
+}
+
+int main(int argc, char *argv[]) {
+	if(argc != 2) {
+            printf("usage:    %s portnumber\n", argv[0]);
+            exit( EXIT_FAILURE );
+      }
+      int port = atoi(argv[1]);
+      if(port < 1) {
+            printf("Invalid port : %d (from %s)\n", port, argv[1]);
+            exit( EXIT_FAILURE );
+      }
+
+	int sock = socket(AF_INET, SOCK_STREAM, 0);
+	if(sock < 0) {
+		perror( "socket" );
+		exit( EXIT_FAILURE );
+	}
+
+	struct sockaddr_in serv_addr;
+      memset(&serv_addr, 0, sizeof(serv_addr));
+      serv_addr.sin_family = AF_INET;
+      serv_addr.sin_addr.s_addr = INADDR_ANY;
+      serv_addr.sin_port = htons(port);
+
+	int ret = bind(sock, (struct sockaddr *) &serv_addr, sizeof(serv_addr));
+	if(ret < 0) {
+		perror( "bind" );
+		exit( EXIT_FAILURE );
+	}
+
+
+     	listen(sock,1);
+
+	struct sockaddr_in cli_addr;
+     	__socklen_t clilen = sizeof(cli_addr);
+	int newsock = accept(sock, (struct sockaddr *) &cli_addr, &clilen);
+     	if (newsock < 0) {
+		perror( "accept" );
+		exit( EXIT_FAILURE );
+	}
+
+	io_uring_queue_init( 16, &ring, 0 );
+
+	async_read( newsock );
+
+	while(1) {
+		struct io_uring_cqe * cqe;
+		struct __kernel_timespec ts = { 2, 0 };
+		// int ret = io_uring_wait_cqes( &ring, &cqe, 1, &ts, NULL); // Requires Linux 5.4
+		int ret = io_uring_wait_cqe( &ring, &cqe );
+
+		if( ret < 0 ) {
+                  printf( "Main Loop Error : %s\n", strerror(-ret) );
+			close( sock );
+                  exit( EXIT_FAILURE );
+            }
+
+		switch(cqe->user_data) {
+                  // Read completed
+                  case 0:
+                        // If it is the end of file we are done
+                        if( cqe->res == 0 ) {
+                              goto END;
+                        }
+
+				if( cqe->res < 0 ) {
+					perror( "Main Loop Error" );
+					close( sock );
+					exit( EXIT_FAILURE );
+				}
+
+				printf("'%.*s'\n", cqe->res, data);
+
+				async_read( newsock );
+
+                        // otherwise prepare a new read
+                        break;
+                  // Wait timed out, time to print
+			// Requires Linux 5.4
+                  case LIBURING_UDATA_TIMEOUT:
+                  	printf(".");
+                        break;
+                  // Problem
+                  default:
+                        printf("Unexpected user data : %llu", cqe->user_data);
+                        exit( EXIT_FAILURE );
+            }
+
+     		io_uring_cqe_seen( &ring, cqe );
+	}
+END:
+
+	io_uring_queue_exit( &ring );
+
+	ret = close(newsock);
+      if(ret < 0) {
+            perror( "close new" );
+            exit( EXIT_FAILURE );
+      }
+
+	ret = close(sock);
+      if(ret < 0) {
+            perror( "close old" );
+            exit( EXIT_FAILURE );
+      }
+
+	return 0;
+}
Index: example/io/simple/server.cfa
===================================================================
--- example/io/simple/server.cfa	(revision f0d67e5abea7221dccd4183dd58ccc6cdbee32fc)
+++ example/io/simple/server.cfa	(revision f0d67e5abea7221dccd4183dd58ccc6cdbee32fc)
@@ -0,0 +1,137 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <unistd.h>
+
+extern "C" {
+	#include <sys/types.h>
+	#include <sys/socket.h>
+	#include <netinet/in.h>
+}
+
+#include <time.hfa>
+#include <thread.hfa>
+
+//----------
+monitor Printer {};
+
+void heartbeat( Printer & mutex ) {
+	fprintf(stderr, ".");
+}
+
+void message( Printer & mutex, char * msg, size_t len ) {
+	fprintf(stderr, "'%.*s'", len, msg);
+}
+
+void status( Printer & mutex, const char * st ) {
+	fprintf(stderr, "%s\n", st);
+}
+
+void error( Printer & mutex, const char * msg, int error) {
+	fprintf(stderr, "%s - %s\n", msg, strerror(error));
+}
+
+Printer printer;
+
+//----------
+thread HeartBeat {};
+
+void ^?{}( HeartBeat & mutex ) {}
+
+void main( HeartBeat & this ) {
+	while(true) {
+		waitfor( ^?{} : this ) { break; }
+		or else{
+			sleep( 5`s );
+			heartbeat( printer );
+		}
+	}
+}
+
+//----------
+extern ssize_t cfa_recvmsg(int sockfd, struct msghdr *msg, int flags);
+extern int cfa_accept4(int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags);
+extern int cfa_close(int fd);
+
+//----------
+thread Server { int port; };
+void main( Server & this ) {
+	char data[256];
+	struct iovec iov = { data, 256 };
+	struct msghdr msg = { "", 0, &iov, 1, NULL, 0, 0 };
+
+	int sock = socket(AF_INET, SOCK_STREAM, 0);
+	if(sock < 0) {
+		error( printer, "socket", -sock);
+		exit( EXIT_FAILURE );
+	}
+
+	status( printer, "Socket created" );
+
+	struct sockaddr_in serv_addr;
+      memset(&serv_addr, 0, sizeof(serv_addr));
+      serv_addr.sin_family = AF_INET;
+      serv_addr.sin_addr.s_addr = INADDR_ANY;
+      serv_addr.sin_port = htons(this.port);
+
+	int ret = bind(sock, (struct sockaddr *) &serv_addr, sizeof(serv_addr));
+	if(ret < 0) {
+		error( printer, "bind", -ret);
+		exit( EXIT_FAILURE );
+	}
+
+	status( printer, "Socket bound" );
+
+     	listen(sock,1);
+
+	struct sockaddr_in cli_addr;
+     	__socklen_t clilen = sizeof(cli_addr);
+	int newsock = cfa_accept4(sock, (struct sockaddr *) &cli_addr, &clilen, 0);
+     	if (newsock < 0) {
+		error( printer, "accept", -newsock);
+		exit( EXIT_FAILURE );
+	}
+
+	status( printer, "Socket accepted, looping" );
+
+	while(1) {
+		int res = cfa_recvmsg(newsock, &msg, 0);
+		if(res == 0) break;
+		if(res < 0) {
+			error( printer, "recvmsg", -res);
+			exit( EXIT_FAILURE );
+		}
+
+		message(printer, data, res);
+	}
+
+	ret = cfa_close(newsock);
+      if(ret < 0) {
+            error( printer, "close new", -ret);
+            exit( EXIT_FAILURE );
+      }
+
+	ret = cfa_close(sock);
+      if(ret < 0) {
+            error( printer, "close old", -ret);
+            exit( EXIT_FAILURE );
+      }
+}
+
+//----------
+int main(int argc, char * argv []) {
+	if(argc != 2) {
+            printf("usage:    %s portnumber\n", argv[0]);
+            exit( EXIT_FAILURE );
+      }
+      int port = atoi(argv[1]);
+      if(port < 1) {
+            printf("Invalid port : %d (from %s)\n", port, argv[1]);
+            exit( EXIT_FAILURE );
+      }
+
+	HeartBeat heartbeat;
+	Server server = { port };
+	// while(true);
+}
Index: example/io/simple/server_epoll.c
===================================================================
--- example/io/simple/server_epoll.c	(revision f0d67e5abea7221dccd4183dd58ccc6cdbee32fc)
+++ example/io/simple/server_epoll.c	(revision f0d67e5abea7221dccd4183dd58ccc6cdbee32fc)
@@ -0,0 +1,177 @@
+/*
+Similar to the server in servier.c, this is a simple server
+that instead uses epoll to block.
+It opens the door to have several polling user-thread per cluster.
+It uses liburing for simplicity.
+*/
+
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <unistd.h>
+
+#include <sys/epoll.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+
+#include <liburing.h>
+
+#define MAX_EVENTS 10
+struct epoll_event ev, events[MAX_EVENTS];
+
+struct io_uring ring;
+
+char data[256];
+struct iovec iov = { data, 256 };
+struct msghdr msg = { (void *)"", 0, &iov, 1, NULL, 0, 0 };
+static void async_read(int sock) {
+	/* get an sqe and fill in a READ operation */
+      struct io_uring_sqe * sqe = io_uring_get_sqe(&ring);
+      io_uring_prep_recvmsg(sqe, sock, &msg, 0);
+      sqe->user_data = 0;
+
+      /* tell the kernel we have an sqe ready for consumption */
+      int ret = io_uring_submit(&ring);
+      assert(ret == 1);
+}
+
+int main(int argc, char *argv[]) {
+	if(argc != 2) {
+            printf("usage:    %s portnumber\n", argv[0]);
+            exit( EXIT_FAILURE );
+      }
+      int port = atoi(argv[1]);
+      if(port < 1) {
+            printf("Invalid port : %d (from %s)\n", port, argv[1]);
+            exit( EXIT_FAILURE );
+      }
+
+	int sock = socket(AF_INET, SOCK_STREAM, 0);
+	if(sock < 0) {
+		perror( "socket" );
+		exit( EXIT_FAILURE );
+	}
+
+	struct sockaddr_in serv_addr;
+      memset(&serv_addr, 0, sizeof(serv_addr));
+      serv_addr.sin_family = AF_INET;
+      serv_addr.sin_addr.s_addr = INADDR_ANY;
+      serv_addr.sin_port = htons(port);
+
+	int ret = bind(sock, (struct sockaddr *) &serv_addr, sizeof(serv_addr));
+	if(ret < 0) {
+		perror( "bind" );
+		exit( EXIT_FAILURE );
+	}
+
+
+     	listen(sock,1);
+
+	struct sockaddr_in cli_addr;
+     	__socklen_t clilen = sizeof(cli_addr);
+	int newsock = accept(sock, (struct sockaddr *) &cli_addr, &clilen);
+     	if (newsock < 0) {
+		perror( "accept" );
+		exit( EXIT_FAILURE );
+	}
+
+	io_uring_queue_init( 16, &ring, 0 );
+
+      int epollfd = epoll_create1(0);
+      if (epollfd == -1) {
+            perror("epoll_create1");
+            exit(EXIT_FAILURE);
+      }
+
+      ev.events = EPOLLIN | EPOLLONESHOT;
+      ev.data.u64 = (uint64_t)&ring;
+      if (epoll_ctl(epollfd, EPOLL_CTL_ADD, ring.ring_fd, &ev) == -1) {
+            perror("epoll_ctl: first");
+            exit(EXIT_FAILURE);
+      }
+
+
+	async_read( newsock );
+
+	while(1) {
+            BLOCK:
+            int nfds = epoll_wait(epollfd, events, MAX_EVENTS, -1);
+            if (nfds == -1) {
+                  perror("epoll_wait");
+                  exit(EXIT_FAILURE);
+            }
+
+
+		while(1) {
+                  struct io_uring_cqe * cqe;
+                  int ret = io_uring_peek_cqe( &ring, &cqe );
+
+                  if( ret < 0 ) {
+                        if(-ret == EAGAIN) {
+                              if (epoll_ctl(epollfd, EPOLL_CTL_MOD, ring.ring_fd, &ev) == -1) {
+                                    perror("epoll_ctl: loop");
+                                    exit(EXIT_FAILURE);
+                              }
+                              goto BLOCK;
+                        }
+                        printf( "Main Loop Error : %s\n", strerror(-ret) );
+                        close( sock );
+                        exit( EXIT_FAILURE );
+                  }
+
+                  switch(cqe->user_data) {
+                        // Read completed
+                        case 0:
+                              // If it is the end of file we are done
+                              if( cqe->res == 0 ) {
+                                    goto END;
+                              }
+
+                              if( cqe->res < 0 ) {
+                                    perror( "Main Loop Error" );
+                                    close( sock );
+                                    exit( EXIT_FAILURE );
+                              }
+
+                              printf("'%.*s'\n", cqe->res, data);
+
+                              async_read( newsock );
+
+                              // otherwise prepare a new read
+                              break;
+                        // Wait timed out, time to print
+                        // Requires Linux 5.4
+                        case LIBURING_UDATA_TIMEOUT:
+                              printf(".");
+                              break;
+                        // Problem
+                        default:
+                              printf("Unexpected user data : %llu", cqe->user_data);
+                              exit( EXIT_FAILURE );
+                  }
+
+                  io_uring_cqe_seen( &ring, cqe );
+            }
+	}
+END:
+
+	io_uring_queue_exit( &ring );
+
+	ret = close(newsock);
+      if(ret < 0) {
+            perror( "close new" );
+            exit( EXIT_FAILURE );
+      }
+
+	ret = close(sock);
+      if(ret < 0) {
+            perror( "close old" );
+            exit( EXIT_FAILURE );
+      }
+
+	return 0;
+}