Changeset af7acb9

Timestamp:

May 8, 2020, 11:53:29 AM (3 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

arm-eh, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

04bc1c0

Parents:

6ec07e5 (diff), e68d092 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' of plg.uwaterloo.ca:software/cfa/cfa-cc

Files:

• benchmark/io/readv.cfa (modified) (6 diffs)
• doc/bibliography/pl.bib (modified) (1 diff)
• libcfa/prelude/defines.hfa.in (modified) (1 diff)
• libcfa/src/concurrency/io.cfa (modified) (17 diffs)
• libcfa/src/concurrency/kernel.cfa (modified) (4 diffs)
• libcfa/src/concurrency/kernel.hfa (modified) (4 diffs)
• libcfa/src/concurrency/kernel_private.hfa (modified) (2 diffs)
• libcfa/src/exception.hfa (added)
• libcfa/src/heap.cfa (modified) (18 diffs)
• libcfa/src/iostream.cfa (modified) (4 diffs)
• libcfa/src/startup.cfa (modified) (2 diffs)
• src/Parser/parser.yy (modified) (2 diffs)
• tests/exceptions/.expect/data-except.txt (added)
• tests/exceptions/conditional.cfa (modified) (5 diffs)
• tests/exceptions/data-except.cfa (added)
• tests/exceptions/except-mac.hfa (deleted)
• tests/exceptions/finally.cfa (modified) (3 diffs)
• tests/exceptions/interact.cfa (modified) (11 diffs)
• tests/exceptions/resume.cfa (modified) (8 diffs)
• tests/exceptions/terminate.cfa (modified) (8 diffs)
• tests/manipulatorsOutput1.cfa (modified) (6 diffs)
• tests/manipulatorsOutput3.cfa (modified) (4 diffs)

benchmark/io/readv.cfa

@@ -16 +16 @@
 #include <thread.hfa>
 #include <time.hfa>
-
-#if !defined(HAVE_LINUX_IO_URING_H)
-#warning no io uring
-#endif
 
 extern bool traceHeapOn();
@@ -53 +49 @@
         while(__atomic_load_n(&run, __ATOMIC_RELAXED)) {
                 int r = cfa_preadv2(fd, &iov, 1, 0, 0);
-                if(r < 0) abort(strerror(-r));
+                if(r < 0) abort("%s\n", strerror(-r));
 
                 __atomic_fetch_add( &count, 1, __ATOMIC_SEQ_CST );
@@ -63 +59 @@
         unsigned long int nthreads = 2;
         unsigned long int nprocs   = 1;
-
-        printf("Setting local\n");
-        setlocale(LC_NUMERIC, "");
+        int flags = 0;
 
         arg_loop:
         for(;;) {
                 static struct option options[] = {
-                        {"duration",  required_argument, 0, 'd'},
-                        {"nthreads",  required_argument, 0, 't'},
-                        {"nprocs",    required_argument, 0, 'p'},
-                        {"bufsize",   required_argument, 0, 'b'},
+                        {"duration",   required_argument, 0, 'd'},
+                        {"nthreads",   required_argument, 0, 't'},
+                        {"nprocs",     required_argument, 0, 'p'},
+                        {"bufsize",    required_argument, 0, 'b'},
+                        {"userthread", no_argument      , 0, 'u'},
                         {0, 0, 0, 0}
                 };
 
                 int idx = 0;
-                int opt = getopt_long(argc, argv, "d:t:p:b:", options, &idx);
+                int opt = getopt_long(argc, argv, "d:t:p:b:u", options, &idx);
 
                 const char * arg = optarg ? optarg : "";
@@ -115 +110 @@
                                 }
                                 break;
+                        case 'u':
+                                flags |= CFA_CLUSTER_IO_POLLER_USER_THREAD;
+                                break;
                         // Other cases
                         default: /* ? */
@@ -135 +133 @@
         }
 
-        printf("Running %lu threads over %lu processors for %lf seconds\n", nthreads, nprocs, duration);
+        printf("Running %lu threads, reading %lu bytes each, over %lu processors for %lf seconds\n", nthreads, buflen, nprocs, duration);
 
         {
                 Time start, end;
-                cluster cl = { "IO Cluster" };
+                cluster cl = { "IO Cluster", flags };
                 the_cluster = &cl;
                 #if !defined(__CFA_NO_STATISTICS__)
@@ -161 +159 @@
                         }
                 }
-                printf("Took %ld ms\n", (end - start)`ms);
-                printf("Total reads:      %'zu\n", count);
-                printf("Reads per second: %'lf\n", ((double)count) / (end - start)`s);
+                printf("Took %'ld ms\n", (end - start)`ms);
+                printf("Total reads      : %'15zu\n", count);
+                printf("Reads per second : %'18.2lf\n", ((double)count) / (end - start)`s);
+                printf("Total read size  : %'15zu\n", buflen * count);
+                printf("Bytes per second : %'18.2lf\n", ((double)count * buflen) / (end - start)`s);
         }
 

doc/bibliography/pl.bib

@@ -4759 +4759 @@
     contributer = {pabuhr@plg},
     author      = {Gregory R. Andrews},
-    title       = {A Method for Solving Synronization Problems},
+    title       = {A Method for Solving Synchronization Problems},
     journal     = scp,
     volume      = 13,

libcfa/prelude/defines.hfa.in

@@ -19 +19 @@
 #undef HAVE_PWRITEV2
 
-// #define __CFA_IO_POLLING_USER__
-// #define __CFA_IO_POLLING_KERNEL__
+#undef __CFA_NO_STATISTICS__

libcfa/src/concurrency/io.cfa

@@ -20 +20 @@
 
 #if !defined(HAVE_LINUX_IO_URING_H)
-        void __kernel_io_startup( cluster &, bool ) {
+        void __kernel_io_startup( cluster &, int, bool ) {
                 // Nothing to do without io_uring
         }
@@ -86 +86 @@
         #endif
 
-        #if defined(__CFA_IO_POLLING_USER__)
-                void ?{}( __io_poller_fast & this, struct cluster & cltr ) {
-                        this.ring = &cltr.io;
-                        (this.thrd){ "Fast I/O Poller", cltr };
-                }
-                void ^?{}( __io_poller_fast & mutex this );
-        void main( __io_poller_fast & this );
-        static inline $thread * get_thread( __io_poller_fast & this ) { return &this.thrd; }
-                void ^?{}( __io_poller_fast & mutex this ) {}
-        #endif
+        // Fast poller user-thread
+        // Not using the "thread" keyword because we want to control
+        // more carefully when to start/stop it
+        struct __io_poller_fast {
+                struct __io_data * ring;
+                bool waiting;
+                $thread thrd;
+        };
+
+        void ?{}( __io_poller_fast & this, struct cluster & cltr ) {
+                this.ring = cltr.io;
+                this.waiting = true;
+                (this.thrd){ "Fast I/O Poller", cltr };
+        }
+        void ^?{}( __io_poller_fast & mutex this );
+        void main( __io_poller_fast & this );
+        static inline $thread * get_thread( __io_poller_fast & this ) { return &this.thrd; }
+        void ^?{}( __io_poller_fast & mutex this ) {}
+
+        struct __submition_data {
+                // Head and tail of the ring (associated with array)
+                volatile uint32_t * head;
+                volatile uint32_t * tail;
+
+                // The actual kernel ring which uses head/tail
+                // indexes into the sqes arrays
+                uint32_t * array;
+
+                // number of entries and mask to go with it
+                const uint32_t * num;
+                const uint32_t * mask;
+
+                // Submission flags (Not sure what for)
+                uint32_t * flags;
+
+                // number of sqes not submitted (whatever that means)
+                uint32_t * dropped;
+
+                // Like head/tail but not seen by the kernel
+                volatile uint32_t alloc;
+                volatile uint32_t ready;
+
+                __spinlock_t lock;
+
+                // A buffer of sqes (not the actual ring)
+                struct io_uring_sqe * sqes;
+
+                // The location and size of the mmaped area
+                void * ring_ptr;
+                size_t ring_sz;
+
+                // Statistics
+                #if !defined(__CFA_NO_STATISTICS__)
+                        struct {
+                                struct {
+                                        volatile unsigned long long int val;
+                                        volatile unsigned long long int cnt;
+                                        volatile unsigned long long int block;
+                                } submit_avg;
+                        } stats;
+                #endif
+        };
+
+        struct __completion_data {
+                // Head and tail of the ring
+                volatile uint32_t * head;
+                volatile uint32_t * tail;
+
+                // number of entries and mask to go with it
+                const uint32_t * mask;
+                const uint32_t * num;
+
+                // number of cqes not submitted (whatever that means)
+                uint32_t * overflow;
+
+                // the kernel ring
+                struct io_uring_cqe * cqes;
+
+                // The location and size of the mmaped area
+                void * ring_ptr;
+                size_t ring_sz;
+
+                // Statistics
+                #if !defined(__CFA_NO_STATISTICS__)
+                        struct {
+                                struct {
+                                        unsigned long long int val;
+                                        unsigned long long int slow_cnt;
+                                        unsigned long long int fast_cnt;
+                                } completed_avg;
+                        } stats;
+                #endif
+        };
+
+        struct __io_data {
+                struct __submition_data submit_q;
+                struct __completion_data completion_q;
+                uint32_t ring_flags;
+                int cltr_flags;
+                int fd;
+                semaphore submit;
+                volatile bool done;
+                struct {
+                        struct {
+                                void * stack;
+                                pthread_t kthrd;
+                        } slow;
+                        __io_poller_fast fast;
+                        __bin_sem_t sem;
+                } poller;
+        };
 
 //=============================================================================================
 // I/O Startup / Shutdown logic
 //=============================================================================================
-        void __kernel_io_startup( cluster & this, bool main_cluster ) {
+        void __kernel_io_startup( cluster & this, int io_flags, bool main_cluster ) {
+                this.io = malloc();
+
                 // Step 1 : call to setup
                 struct io_uring_params params;
@@ -113 +216 @@
 
                 // Step 2 : mmap result
-                memset(&this.io, 0, sizeof(struct io_ring));
-                struct io_uring_sq & sq = this.io.submit_q;
-                struct io_uring_cq & cq = this.io.completion_q;
+                memset( this.io, 0, sizeof(struct __io_data) );
+                struct __submition_data  & sq = this.io->submit_q;
+                struct __completion_data & cq = this.io->completion_q;
 
                 // calculate the right ring size
@@ -193 +296 @@
 
                 // Update the global ring info
-                this.io.flags = params.flags;
-                this.io.fd    = fd;
-                this.io.done  = false;
-                (this.io.submit){ min(*sq.num, *cq.num) };
+                this.io->ring_flags = params.flags;
+                this.io->cltr_flags = io_flags;
+                this.io->fd         = fd;
+                this.io->done       = false;
+                (this.io->submit){ min(*sq.num, *cq.num) };
 
                 // Initialize statistics
                 #if !defined(__CFA_NO_STATISTICS__)
-                        this.io.submit_q.stats.submit_avg.val = 0;
-                        this.io.submit_q.stats.submit_avg.cnt = 0;
-                        this.io.completion_q.stats.completed_avg.val = 0;
-                        this.io.completion_q.stats.completed_avg.cnt = 0;
+                        this.io->submit_q.stats.submit_avg.val   = 0;
+                        this.io->submit_q.stats.submit_avg.cnt   = 0;
+                        this.io->submit_q.stats.submit_avg.block = 0;
+                        this.io->completion_q.stats.completed_avg.val = 0;
+                        this.io->completion_q.stats.completed_avg.slow_cnt = 0;
+                        this.io->completion_q.stats.completed_avg.fast_cnt = 0;
                 #endif
 
@@ -212 +318 @@
 
         void __kernel_io_finish_start( cluster & this ) {
-                #if defined(__CFA_IO_POLLING_USER__)
+                if( this.io->cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ) {
                         __cfadbg_print_safe(io_core, "Kernel I/O : Creating fast poller for cluter %p\n", &this);
-                        (this.io.poller.fast){ this };
-                        __thrd_start( this.io.poller.fast, main );
-                #endif
+                        (this.io->poller.fast){ this };
+                        __thrd_start( this.io->poller.fast, main );
+                }
 
                 // Create the poller thread
                 __cfadbg_print_safe(io_core, "Kernel I/O : Creating slow poller for cluter %p\n", &this);
-                this.io.poller.slow.stack = __create_pthread( &this.io.poller.slow.kthrd, __io_poller_slow, &this );
+                this.io->poller.slow.stack = __create_pthread( &this.io->poller.slow.kthrd, __io_poller_slow, &this );
         }
 
@@ -226 +332 @@
                 __cfadbg_print_safe(io_core, "Kernel I/O : Stopping pollers for cluster\n", &this);
                 // Notify the poller thread of the shutdown
-                __atomic_store_n(&this.io.done, true, __ATOMIC_SEQ_CST);
+                __atomic_store_n(&this.io->done, true, __ATOMIC_SEQ_CST);
 
                 // Stop the IO Poller
                 sigval val = { 1 };
-                pthread_sigqueue( this.io.poller.slow.kthrd, SIGUSR1, val );
-                #if defined(__CFA_IO_POLLING_USER__)
-                        post( this.io.poller.sem );
-                #endif
+                pthread_sigqueue( this.io->poller.slow.kthrd, SIGUSR1, val );
+                post( this.io->poller.sem );
 
                 // Wait for the poller thread to finish
-                pthread_join( this.io.poller.slow.kthrd, 0p );
-                free( this.io.poller.slow.stack );
+                pthread_join( this.io->poller.slow.kthrd, 0p );
+                free( this.io->poller.slow.stack );
 
                 __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller stopped for cluster\n", &this);
 
-                #if defined(__CFA_IO_POLLING_USER__)
+                if( this.io->cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ) {
+                        with( this.io->poller.fast ) {
+                                /* paranoid */ verify( waiting ); // The thread shouldn't be in a system call
+                                /* paranoid */ verify( this.procs.head == 0p || &this == mainCluster );
+                                /* paranoid */ verify( this.idles.head == 0p || &this == mainCluster );
+
+                                // We need to adjust the clean-up based on where the thread is
+                                if( thrd.preempted != __NO_PREEMPTION ) {
+
+                                        // This is the tricky case
+                                        // The thread was preempted and now it is on the ready queue
+                                        /* paranoid */ verify( thrd.state == Active );           // The thread better be in this state
+                                        /* paranoid */ verify( thrd.next == 1p );                // The thread should be the last on the list
+                                        /* paranoid */ verify( this.ready_queue.head == &thrd ); // The thread should be the only thing on the list
+
+                                        // Remove the thread from the ready queue of this cluster
+                                        this.ready_queue.head = 1p;
+                                        thrd.next = 0p;
+
+                                        // Fixup the thread state
+                                        thrd.state = Blocked;
+                                        thrd.preempted = __NO_PREEMPTION;
+
+                                        // Pretend like the thread was blocked all along
+                                }
+                                // !!! This is not an else if !!!
+                                if( thrd.state == Blocked ) {
+
+                                        // This is the "easy case"
+                                        // The thread is parked and can easily be moved to active cluster
+                                        verify( thrd.curr_cluster != active_cluster() || thrd.curr_cluster == mainCluster );
+                                        thrd.curr_cluster = active_cluster();
+
                         // unpark the fast io_poller
-                        unpark( &this.io.poller.fast.thrd __cfaabi_dbg_ctx2 );
-
-                        ^(this.io.poller.fast){};
+                                        unpark( &thrd __cfaabi_dbg_ctx2 );
+                                }
+                                else {
+
+                                        // The thread is in a weird state
+                                        // I don't know what to do here
+                                        abort("Fast poller thread is in unexpected state, cannot clean-up correctly\n");
+                                }
+
+                        }
+
+                        ^(this.io->poller.fast){};
 
                         __cfadbg_print_safe(io_core, "Kernel I/O : Fast poller stopped for cluster\n", &this);
-                #endif
+                }
         }
 
@@ -259 +404 @@
                 #if !defined(__CFA_NO_STATISTICS__)
                         if(this.print_stats) {
-                                __cfaabi_bits_print_safe( STDERR_FILENO,
-                                        "----- I/O uRing Stats -----\n"
-                                        "- total submit calls  : %llu\n"
-                                        "- avg submit          : %lf\n"
-                                        "- total wait calls    : %llu\n"
-                                        "- avg completion/wait : %lf\n",
-                                        this.io.submit_q.stats.submit_avg.cnt,
-                                        ((double)this.io.submit_q.stats.submit_avg.val) / this.io.submit_q.stats.submit_avg.cnt,
-                                        this.io.completion_q.stats.completed_avg.cnt,
-                                        ((double)this.io.completion_q.stats.completed_avg.val) / this.io.completion_q.stats.completed_avg.cnt
-                                );
+                                with(this.io->submit_q.stats, this.io->completion_q.stats) {
+                                        __cfaabi_bits_print_safe( STDERR_FILENO,
+                                                "----- I/O uRing Stats -----\n"
+                                                "- total submit calls  : %'15llu\n"
+                                                "- avg submit          : %'18.2lf\n"
+                                                "- pre-submit block %%  : %'18.2lf\n"
+                                                "- total wait calls    : %'15llu   (%'llu slow, %'llu fast)\n"
+                                                "- avg completion/wait : %'18.2lf\n",
+                                                submit_avg.cnt,
+                                                ((double)submit_avg.val) / submit_avg.cnt,
+                                                (100.0 * submit_avg.block) / submit_avg.cnt,
+                                                completed_avg.slow_cnt + completed_avg.fast_cnt,
+                                                completed_avg.slow_cnt,  completed_avg.fast_cnt,
+                                                ((double)completed_avg.val) / (completed_avg.slow_cnt + completed_avg.fast_cnt)
+                                        );
+                                }
                         }
                 #endif
 
                 // Shutdown the io rings
-                struct io_uring_sq & sq = this.io.submit_q;
-                struct io_uring_cq & cq = this.io.completion_q;
+                struct __submition_data  & sq = this.io->submit_q;
+                struct __completion_data & cq = this.io->completion_q;
 
                 // unmap the submit queue entries
@@ -289 +439 @@
 
                 // close the file descriptor
-                close(this.io.fd);
+                close(this.io->fd);
+
+                free( this.io );
         }
 
@@ -302 +454 @@
         // Process a single completion message from the io_uring
         // This is NOT thread-safe
-        static int __drain_io( struct io_ring & ring, sigset_t * mask, int waitcnt, bool in_kernel ) {
+        static int __drain_io( struct __io_data & ring, sigset_t * mask, int waitcnt, bool in_kernel ) {
                 int ret = syscall( __NR_io_uring_enter, ring.fd, 0, waitcnt, IORING_ENTER_GETEVENTS, mask, _NSIG / 8);
                 if( ret < 0 ) {
@@ -320 +472 @@
                 // Nothing was new return 0
                 if (head == tail) {
-                        #if !defined(__CFA_NO_STATISTICS__)
-                                ring.completion_q.stats.completed_avg.cnt += 1;
-                        #endif
                         return 0;
                 }
@@ -348 +497 @@
                 __atomic_fetch_add( ring.completion_q.head, count, __ATOMIC_RELAXED );
 
-                // Update statistics
-                #if !defined(__CFA_NO_STATISTICS__)
-                        ring.completion_q.stats.completed_avg.val += count;
-                        ring.completion_q.stats.completed_avg.cnt += 1;
-                #endif
-
                 return count;
         }
@@ -359 +502 @@
         static void * __io_poller_slow( void * arg ) {
                 cluster * cltr = (cluster *)arg;
-                struct io_ring & ring = cltr->io;
+                struct __io_data & ring = *cltr->io;
 
                 sigset_t mask;
@@ -372 +515 @@
                 verify( (*ring.completion_q.head) == (*ring.completion_q.tail) );
 
-                while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) {
-                        #if defined(__CFA_IO_POLLING_USER__)
-
+                __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller for ring %p ready\n", &ring);
+
+                if( ring.cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD ) {
+                        while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) {
                                 // In the user-thread approach drain and if anything was drained,
                                 // batton pass to the user-thread
                                 int count = __drain_io( ring, &mask, 1, true );
+
+                                // Update statistics
+                                #if !defined(__CFA_NO_STATISTICS__)
+                                        ring.completion_q.stats.completed_avg.val += count;
+                                        ring.completion_q.stats.completed_avg.slow_cnt += 1;
+                                #endif
+
                                 if(count > 0) {
                                         __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to fast poller\n", &ring);
@@ -383 +534 @@
                                         wait( ring.poller.sem );
                                 }
-
-                        #else
-
+                        }
+                }
+                else {
+                        while(!__atomic_load_n(&ring.done, __ATOMIC_SEQ_CST)) {
                                 //In the naive approach, just poll the io completion queue directly
-                                __drain_io( ring, &mask, 1, true );
-
+                                int count = __drain_io( ring, &mask, 1, true );
+
+                                // Update statistics
+                                #if !defined(__CFA_NO_STATISTICS__)
+                                        ring.completion_q.stats.completed_avg.val += count;
+                                        ring.completion_q.stats.completed_avg.slow_cnt += 1;
+                                #endif
+                        }
+                }
+
+                __cfadbg_print_safe(io_core, "Kernel I/O : Slow poller for ring %p stopping\n", &ring);
+
+                return 0p;
+        }
+
+        void main( __io_poller_fast & this ) {
+                verify( this.ring->cltr_flags & CFA_CLUSTER_IO_POLLER_USER_THREAD );
+
+                // Start parked
+                park( __cfaabi_dbg_ctx );
+
+                __cfadbg_print_safe(io_core, "Kernel I/O : Fast poller for ring %p ready\n", &this.ring);
+
+                int reset = 0;
+
+                // Then loop until we need to start
+                while(!__atomic_load_n(&this.ring->done, __ATOMIC_SEQ_CST)) {
+                        // Drain the io
+                        this.waiting = false;
+                        int count = __drain_io( *this.ring, 0p, 0, false );
+                        reset += count > 0 ? 1 : 0;
+
+                        // Update statistics
+                        #if !defined(__CFA_NO_STATISTICS__)
+                                this.ring->completion_q.stats.completed_avg.val += count;
+                                this.ring->completion_q.stats.completed_avg.fast_cnt += 1;
                         #endif
-                }
-
-                return 0p;
-        }
-
-        #if defined(__CFA_IO_POLLING_USER__)
-                void main( __io_poller_fast & this ) {
-                        // Start parked
-                        park( __cfaabi_dbg_ctx );
-
-                        // Then loop until we need to start
-                        while(!__atomic_load_n(&this.ring->done, __ATOMIC_SEQ_CST)) {
-                                // Drain the io
-                                if(0 > __drain_io( *this.ring, 0p, 0, false )) {
-                                        // If we got something, just yield and check again
-                                        yield();
-                                }
-                                else {
-                                        // We didn't get anything baton pass to the slow poller
-                                        __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to slow poller\n", &this.ring);
-                                        post( this.ring->poller.sem );
-                                        park( __cfaabi_dbg_ctx );
-                                }
+
+                        this.waiting = true;
+                        if(reset < 5) {
+                                // If we got something, just yield and check again
+                                yield();
                         }
-                }
-        #endif
+                        else {
+                                // We didn't get anything baton pass to the slow poller
+                                __cfadbg_print_safe(io_core, "Kernel I/O : Moving to ring %p to slow poller\n", &this.ring);
+                                post( this.ring->poller.sem );
+                                park( __cfaabi_dbg_ctx );
+                                reset = 0;
+                        }
+                }
+
+                __cfadbg_print_safe(io_core, "Kernel I/O : Fast poller for ring %p stopping\n", &this.ring);
+        }
 
 //=============================================================================================
@@ -445 +622 @@
 //
 
-        static inline [* struct io_uring_sqe, uint32_t] __submit_alloc( struct io_ring & ring ) {
+        static inline [* struct io_uring_sqe, uint32_t] __submit_alloc( struct __io_data & ring ) {
                 // Wait for a spot to be available
-                P(ring.submit);
+                __attribute__((unused)) bool blocked = P(ring.submit);
+                #if !defined(__CFA_NO_STATISTICS__)
+                        __atomic_fetch_add( &ring.submit_q.stats.submit_avg.block, blocked ? 1ul64 : 0ul64, __ATOMIC_RELAXED );
+                #endif
 
                 // Allocate the sqe
@@ -463 +643 @@
         }
 
-        static inline void __submit( struct io_ring & ring, uint32_t idx ) {
+        static inline void __submit( struct __io_data & ring, uint32_t idx ) {
                 // get mutual exclusion
                 lock(ring.submit_q.lock __cfaabi_dbg_ctx2);
@@ -524 +704 @@
 
         #define __submit_prelude \
-                struct io_ring & ring = active_cluster()->io; \
+                struct __io_data & ring = *active_cluster()->io; \
                 struct io_uring_sqe * sqe; \
                 uint32_t idx; \

libcfa/src/concurrency/kernel.cfa

@@ -256 +256 @@
 }
 
-void ?{}(cluster & this, const char name[], Duration preemption_rate) with( this ) {
+void ?{}(cluster & this, const char name[], Duration preemption_rate, int io_flags) with( this ) {
         this.name = name;
         this.preemption_rate = preemption_rate;
@@ -270 +270 @@
         threads{ __get };
 
-        __kernel_io_startup( this, &this == mainCluster );
+        __kernel_io_startup( this, io_flags, &this == mainCluster );
 
         doregister(this);
@@ -993 +993 @@
 void ^?{}(semaphore & this) {}
 
-void P(semaphore & this) with( this ){
+bool P(semaphore & this) with( this ){
         lock( lock __cfaabi_dbg_ctx2 );
         count -= 1;
@@ -1003 +1003 @@
                 unlock( lock );
                 park( __cfaabi_dbg_ctx );
+                return true;
         }
         else {
             unlock( lock );
+            return false;
         }
 }

libcfa/src/concurrency/kernel.hfa

@@ -38 +38 @@
 void  ?{}(semaphore & this, int count = 1);
 void ^?{}(semaphore & this);
-void   P (semaphore & this);
+bool   P (semaphore & this);
 bool   V (semaphore & this);
 bool   V (semaphore & this, unsigned count);
@@ -114 +114 @@
 //-----------------------------------------------------------------------------
 // I/O
-#if defined(HAVE_LINUX_IO_URING_H)
-struct io_uring_sq {
-        // Head and tail of the ring (associated with array)
-        volatile uint32_t * head;
-        volatile uint32_t * tail;
+struct __io_data;
 
-        // The actual kernel ring which uses head/tail
-        // indexes into the sqes arrays
-        uint32_t * array;
-
-        // number of entries and mask to go with it
-        const uint32_t * num;
-        const uint32_t * mask;
-
-        // Submission flags (Not sure what for)
-        uint32_t * flags;
-
-        // number of sqes not submitted (whatever that means)
-        uint32_t * dropped;
-
-        // Like head/tail but not seen by the kernel
-        volatile uint32_t alloc;
-        volatile uint32_t ready;
-
-        __spinlock_t lock;
-
-        // A buffer of sqes (not the actual ring)
-        struct io_uring_sqe * sqes;
-
-        // The location and size of the mmaped area
-        void * ring_ptr;
-        size_t ring_sz;
-
-        // Statistics
-        #if !defined(__CFA_NO_STATISTICS__)
-                struct {
-                        struct {
-                                unsigned long long int val;
-                                unsigned long long int cnt;
-                        } submit_avg;
-                } stats;
-        #endif
-};
-
-struct io_uring_cq {
-        // Head and tail of the ring
-        volatile uint32_t * head;
-        volatile uint32_t * tail;
-
-        // number of entries and mask to go with it
-        const uint32_t * mask;
-        const uint32_t * num;
-
-        // number of cqes not submitted (whatever that means)
-        uint32_t * overflow;
-
-        // the kernel ring
-        struct io_uring_cqe * cqes;
-
-        // The location and size of the mmaped area
-        void * ring_ptr;
-        size_t ring_sz;
-
-        // Statistics
-        #if !defined(__CFA_NO_STATISTICS__)
-                struct {
-                        struct {
-                                unsigned long long int val;
-                                unsigned long long int cnt;
-                        } completed_avg;
-                } stats;
-        #endif
-};
-
-#if defined(__CFA_IO_POLLING_USER__)
-        struct __io_poller_fast {
-                struct io_ring * ring;
-                $thread thrd;
-        };
-#endif
-
-struct io_ring {
-        struct io_uring_sq submit_q;
-        struct io_uring_cq completion_q;
-        uint32_t flags;
-        int fd;
-        semaphore submit;
-        volatile bool done;
-        struct {
-                struct {
-                        void * stack;
-                        pthread_t kthrd;
-                } slow;
-                #if defined(__CFA_IO_POLLING_USER__)
-                        __io_poller_fast fast;
-                        __bin_sem_t sem;
-                #endif
-        } poller;
-};
-#endif
+#define CFA_CLUSTER_IO_POLLER_USER_THREAD 1 << 0
+// #define CFA_CLUSTER_IO_POLLER_KERNEL_SIDE 1 << 1
 
 //-----------------------------------------------------------------------------
@@ -247 +151 @@
         } node;
 
-        #if defined(HAVE_LINUX_IO_URING_H)
-                struct io_ring io;
-        #endif
+        struct __io_data * io;
 
         #if !defined(__CFA_NO_STATISTICS__)
@@ -257 +159 @@
 extern Duration default_preemption();
 
-void ?{} (cluster & this, const char name[], Duration preemption_rate);
+void ?{} (cluster & this, const char name[], Duration preemption_rate, int flags);
 void ^?{}(cluster & this);
 
-static inline void ?{} (cluster & this)                           { this{"Anonymous Cluster", default_preemption()}; }
-static inline void ?{} (cluster & this, Duration preemption_rate) { this{"Anonymous Cluster", preemption_rate}; }
-static inline void ?{} (cluster & this, const char name[])        { this{name, default_preemption()}; }
+static inline void ?{} (cluster & this)                                      { this{"Anonymous Cluster", default_preemption(), 0}; }
+static inline void ?{} (cluster & this, Duration preemption_rate)            { this{"Anonymous Cluster", preemption_rate, 0}; }
+static inline void ?{} (cluster & this, const char name[])                   { this{name, default_preemption(), 0}; }
+static inline void ?{} (cluster & this, int flags)                           { this{"Anonymous Cluster", default_preemption(), flags}; }
+static inline void ?{} (cluster & this, Duration preemption_rate, int flags) { this{"Anonymous Cluster", preemption_rate, flags}; }
+static inline void ?{} (cluster & this, const char name[], int flags)        { this{name, default_preemption(), flags}; }
 
 static inline [cluster *&, cluster *& ] __get( cluster & this ) __attribute__((const)) { return this.node.[next, prev]; }

libcfa/src/concurrency/kernel_private.hfa

@@ -59 +59 @@
 extern volatile thread_local __cfa_kernel_preemption_state_t preemption_state __attribute__ ((tls_model ( "initial-exec" )));
 
+extern cluster * mainCluster;
+
 //-----------------------------------------------------------------------------
 // Threads
@@ -75 +77 @@
 //-----------------------------------------------------------------------------
 // I/O
-void __kernel_io_startup     ( cluster &, bool );
+void __kernel_io_startup     ( cluster &, int, bool );
 void __kernel_io_finish_start( cluster & );
 void __kernel_io_prepare_stop( cluster & );

libcfa/src/heap.cfa

@@ -10 +10 @@
 // Created On       : Tue Dec 19 21:58:35 2017
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Sat Apr 18 17:43:15 2020
-// Update Count     : 718
+// Last Modified On : Wed May  6 17:29:26 2020
+// Update Count     : 727
 //
 
@@ -19 +19 @@
 #include <errno.h>                                                                              // errno
 #include <string.h>                                                                             // memset, memcpy
+#include <limits.h>                                                                             // ULONG_MAX
 extern "C" {
 #include <sys/mman.h>                                                                   // mmap, munmap
 } // extern "C"
 
-// #comment TD : Many of these should be merged into math I believe
 #include "bits/align.hfa"                                                               // libPow2
 #include "bits/defs.hfa"                                                                // likely, unlikely
@@ -30 +30 @@
 //#include "stdlib.hfa"                                                                 // bsearchl
 #include "malloc.h"
+#include "bitmanip.hfa"                                                                 // ceiling
 
 #define MIN(x, y) (y > x ? x : y)
@@ -81 +82 @@
 };
 
+size_t default_heap_expansion() __attribute__(( weak )) {
+        return __CFA_DEFAULT_HEAP_EXPANSION__;
+} // default_heap_expansion
+
 size_t default_mmap_start() __attribute__(( weak )) {
         return __CFA_DEFAULT_MMAP_START__;
 } // default_mmap_start
-
-size_t default_heap_expansion() __attribute__(( weak )) {
-        return __CFA_DEFAULT_HEAP_EXPANSION__;
-} // default_heap_expansion
 
 
@@ -360 +361 @@
 
 
-static inline bool setHeapExpand( size_t value ) {
-  if ( heapExpand < pageSize ) return true;
-        heapExpand = value;
-        return false;
-} // setHeapExpand
-
-
 // thunk problem
 size_t Bsearchl( unsigned int key, const unsigned int * vals, size_t dim ) {
@@ -383 +377 @@
 
 static inline bool setMmapStart( size_t value ) {               // true => mmapped, false => sbrk
-  if ( value < pageSize || bucketSizes[NoBucketSizes - 1] < value ) return true;
+  if ( value < pageSize || bucketSizes[NoBucketSizes - 1] < value ) return false;
         mmapStart = value;                                                                      // set global
 
@@ -390 +384 @@
         assert( maxBucketsUsed < NoBucketSizes );                       // subscript failure ?
         assert( mmapStart <= bucketSizes[maxBucketsUsed] ); // search failure ?
-        return false;
+        return true;
 } // setMmapStart
 
@@ -449 +443 @@
 
         #ifdef __CFA_DEBUG__
-        checkHeader( addr < heapBegin || header < (HeapManager.Storage.Header *)heapBegin, name, addr ); // bad low address ?
+        checkHeader( addr < heapBegin, name, addr );            // bad low address ?
         #endif // __CFA_DEBUG__
 
@@ -508 +502 @@
         // along with the block and is a multiple of the alignment size.
 
-  if ( unlikely( size > ~0ul - sizeof(HeapManager.Storage) ) ) return 0p;
+  if ( unlikely( size > ULONG_MAX - sizeof(HeapManager.Storage) ) ) return 0p;
         size_t tsize = size + sizeof(HeapManager.Storage);
         if ( likely( tsize < mmapStart ) ) {                            // small size => sbrk
@@ -560 +554 @@
                 block->header.kind.real.home = freeElem;                // pointer back to free list of apropriate size
         } else {                                                                                        // large size => mmap
-  if ( unlikely( size > ~0ul - pageSize ) ) return 0p;
+  if ( unlikely( size > ULONG_MAX - pageSize ) ) return 0p;
                 tsize = libCeiling( tsize, pageSize );                  // must be multiple of page size
                 #ifdef __STATISTICS__
@@ -702 +696 @@
         #endif // FASTLOOKUP
 
-        if ( setMmapStart( default_mmap_start() ) ) {
+        if ( ! setMmapStart( default_mmap_start() ) ) {
                 abort( "HeapManager : internal error, mmap start initialization failure." );
         } // if
@@ -708 +702 @@
 
         char * end = (char *)sbrk( 0 );
-        sbrk( (char *)libCeiling( (long unsigned int)end, libAlign() ) - end ); // move start of heap to multiple of alignment
-        heapBegin = heapEnd = sbrk( 0 );                                        // get new start point
+        heapBegin = heapEnd = sbrk( (char *)libCeiling( (long unsigned int)end, libAlign() ) - end ); // move start of heap to multiple of alignment
 } // HeapManager
 
@@ -735 +728 @@
         //assert( heapManager.heapBegin != 0 );
         //heapManager{};
-        if ( heapManager.heapBegin == 0p ) heapManager{};
+        if ( heapManager.heapBegin == 0p ) heapManager{};       // sanity check
 } // memory_startup
 
@@ -863 +856 @@
                 #endif // __STATISTICS__
 
-                size_t size = dim * elemSize;
-                char * addr = (char *)mallocNoStats( size );
-          if ( unlikely( addr == 0p ) ) return 0p;
-
-                HeapManager.Storage.Header * header;
-                HeapManager.FreeHeader * freeElem;
-                size_t bsize, alignment;
-                headers( "aalloc", addr, header, freeElem, bsize, alignment );
-
-                header->kind.real.blockSize |= 2;                               // mark as zero filled
-                return addr;
+                return mallocNoStats( dim * elemSize );
         } // aalloc
 
@@ -914 +897 @@
         
                 // change size, DO NOT preserve STICKY PROPERTIES.
+                free( oaddr );
                 void * naddr = mallocNoStats( size );                   // create new area
-                free( oaddr );
                 return naddr;
         } // resize
@@ -988 +971 @@
                 #endif // __STATISTICS__
 
-                size_t size = dim * elemSize;
-                char * addr = (char *)memalignNoStats( alignment, size );
-          if ( unlikely( addr == 0p ) ) return 0p;
-                HeapManager.Storage.Header * header;
-                HeapManager.FreeHeader * freeElem;
-                size_t bsize;
-                headers( "amemalign", addr, header, freeElem, bsize, alignment );
-
-                header->kind.real.blockSize |= 2;                               // mark as zero filled
-                return addr;
+                return memalignNoStats( alignment, dim * elemSize );
         } // amemalign
 
@@ -1043 +1017 @@
 
         // Frees the memory space pointed to by ptr, which must have been returned by a previous call to malloc(), calloc()
-        // or realloc().  Otherwise, or if free(ptr) has already been called before, undefined behavior occurs. If ptr is
+        // or realloc().  Otherwise, or if free(ptr) has already been called before, undefined behaviour occurs. If ptr is
         // 0p, no operation is performed.
         void free( void * addr ) {
@@ -1170 +1144 @@
 
 
-        // Adjusts parameters that control the behavior of the memory-allocation functions (see malloc). The param argument
+        // Adjusts parameters that control the behaviour of the memory-allocation functions (see malloc). The param argument
         // specifies the parameter to be modified, and value specifies the new value for that parameter.
         int mallopt( int option, int value ) {
                 choose( option ) {
                   case M_TOP_PAD:
-                        if ( setHeapExpand( value ) ) return 1;
+                        heapExpand = ceiling( value, pageSize ); return 1;
                   case M_MMAP_THRESHOLD:
                         if ( setMmapStart( value ) ) return 1;
+                        break;
                 } // switch
                 return 0;                                                                               // error, unsupported

libcfa/src/iostream.cfa

@@ -10 +10 @@
 // Created On       : Wed May 27 17:56:53 2015
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Thu Apr 30 10:50:31 2020
-// Update Count     : 1001
+// Last Modified On : Sat May  2 18:30:25 2020
+// Update Count     : 1017
 //
 
@@ -583 +583 @@
                                         if ( fmt.flags.pc && fmt.pc > 64 ) fmt.pc -= 64; else { fmt.flags.pc = false; fmt.pc = 0; } \
                                         if ( fmt.flags.left ) { \
-                                                fmt2.wd = fmt.wd; \
-                                                if ( fmt2.wd <= 64 ) { \
-                                                        fmt2.wd = 64; \
-                                                } else { \
-                                                        if ( fmt.pc > 0 ) { \
-                                                                fmt2.wd -= fmt.pc; \
-                                                        } else { \
-                                                                fmt2.wd = fmt.wd - high1( msig ); \
-                                                        } /* if */ \
-                                                        if ( ! fmt.flags.nobsdp ) fmt2.wd -= 2; \
-                                                        if ( fmt2.wd < 0 ) fmt2.wd = 0; \
-                                                        fmt2.flags.left = true; \
-                                                        fmt.wd = 0; \
-                                                } /* if */ \
-                                                printf( "left %d %d %x\n", f.wd, f.pc, f.all ); \
-                                                printf( "left %d %d %x\n", fmt.wd, fmt.pc, fmt.all ); \
+                                                fmt.flags.left = false; \
+                                                fmt.wd = 0; \
+                                                /* printf( "L %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
+                                                fmt2.flags.left = true; \
+                                                int msigd = high1( msig ); \
+                                                fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd); \
+                                                if ( ! fmt.flags.nobsdp ) fmt2.wd -= 2; /* compensate for 0b base specifier */ \
+                                                if ( (int)fmt2.wd < 64 ) fmt2.wd = 64; /* cast deals with negative value */ \
+                                                fmt2.flags.pc = true; fmt2.pc = 64; \
                                         } else { \
                                                 if ( fmt.wd > 64 ) fmt.wd -= 64; \
-                                                /* if ( ! fmt.flags.nobsdp && fmt.pc < fmt.wd ) fmt.wd -= 1; */ \
+                                                else fmt.wd = 1; \
+                                                /* printf( "R %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
                                                 fmt2.wd = 64; \
-                                                /* printf( "R %llo %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt2.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
                                         } /* if */ \
-                                        /* printf( "\nC %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
+                                        /* printf( "C %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
                                         (ostype &)(os | fmt | "" | fmt2); \
                                 } else if ( f.base == 'o' ) { \
@@ -614 +607 @@
                                                 fmt.flags.left = false; \
                                                 fmt.wd = 0; \
-                                                /* if ( fmt2.val > 1 && fmt.flags.pc && fmt.pc > 0 ) fmt.pc -= 1; */ \
                                                 /* printf( "L %llo %llo %llo %d %d '%c' %x %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all, fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
                                                 (ostype &)(os | fmt | "" | fmt2); \
                                                 sepOff( os ); \
                                                 fmt2.flags.left = true; \
-                                                int msigd = ceiling( high1( fmt.val ), 3 ) + 1; \
-                                                fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd) - 1; \
+                                                int msigd = ceiling( high1( fmt.val ), 3 ); \
+                                                fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd); \
+                                                if ( ! fmt.flags.nobsdp ) fmt2.wd -= 1; /* compensate for 0 base specifier */ \
                                                 if ( (int)fmt2.wd < 21 ) fmt2.wd = 21; /* cast deals with negative value */ \
                                                 fmt2.flags.pc = true; fmt2.pc = 21; \
                                         } else { \
                                                 if ( fmt.wd > 22 ) fmt.wd -= 22; \
-                                                /* compensate for leading 0 */ \
-                                                /*if ( ! fmt.flags.nobsdp && fmt.pc < fmt.wd ) fmt.wd -= 1;*/ \
-                                                fmt2.wd = 1; \
+                                                else fmt.wd = 1; \
                                                 /* printf( "R %llo %llo %llo %d %d '%c' %x %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all, fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
                                                 (ostype &)(os | fmt | "" | fmt2); \
@@ -636 +627 @@
                                         /* printf( "\nC %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
                                         (ostype &)(os | fmt2); \
-                                } else { \
-                                        if ( fmt.flags.pc && fmt.pc > 16 ) fmt.pc -= 16; \
+                                } else { /* f.base == 'x'  | f.base == 'X' */ \
+                                        if ( fmt.flags.pc && fmt.pc > 16 ) fmt.pc -= 16; else { fmt.flags.pc = false; fmt.pc = 0; } \
                                         if ( fmt.flags.left ) { \
-                                                fmt2.wd = fmt.wd; \
-                                                if ( fmt2.wd <= 16 ) { \
-                                                        fmt2.wd = 16; \
-                                                } else { \
-                                                        if ( fmt.pc > 0 ) { \
-                                                                fmt2.wd -= fmt.pc; \
-                                                        } else { \
-                                                                fmt2.wd = fmt.wd - ceiling2( high1( msig ), 4 ) / 4; \
-                                                        } /* if */ \
-                                                        if ( ! fmt.flags.nobsdp ) fmt2.wd -= 2; \
-                                                        if ( fmt2.wd < 0 ) fmt2.wd = 0; \
-                                                        fmt2.flags.left = true; \
-                                                        fmt.wd = 0; \
-                                                } /* if */ \
+                                                fmt.flags.left = false; \
+                                                fmt.wd = 0; \
+                                                /* printf( "L %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
+                                                fmt2.flags.left = true; \
+                                                int msigd = high1( msig ); \
+                                                fmt2.wd = f.wd - (fmt.pc > msigd ? fmt.pc : msigd); \
+                                                if ( ! fmt.flags.nobsdp ) fmt2.wd -= 2; /* compensate for 0x base specifier */ \
+                                                if ( (int)fmt2.wd < 16 ) fmt2.wd = 16; /* cast deals with negative value */ \
+                                                fmt2.flags.pc = true; fmt2.pc = 16; \
                                         } else { \
+                                                if ( fmt.wd > 16 ) fmt.wd -= 16; \
+                                                else fmt.wd = 1; \
+                                                /* printf( "R %llo %llo %llo %d %d '%c' %x\n", msig, lsig, fmt.val, fmt.wd, fmt.pc, fmt.base, fmt.all ); */ \
                                                 fmt2.wd = 16; \
-                                                if ( fmt.wd > 16 ) fmt.wd -= 16; \
                                         } /* if */ \
-                                        fmt2.pc = 16; fmt2.flags.pc = true; \
+                                        /* printf( "C %llo %d %d '%c' %x\n", fmt2.val, fmt2.wd, fmt2.pc, fmt2.base, fmt2.all ); */ \
                                         (ostype &)(os | fmt | "" | fmt2); \
                                 } /* if */ \

libcfa/src/startup.cfa

@@ -14 +14 @@
 //
 
-#include <time.h>                                                                               // tzset
+#include <time.h>                // tzset
+#include <locale.h>        // setlocale
 #include "startup.hfa"
 
@@ -21 +22 @@
     void __cfaabi_appready_startup( void ) {
                 tzset();                                                                                // initialize time global variables
+                setlocale(LC_NUMERIC, "");
                 #ifdef __CFA_DEBUG__
                 extern void heapAppStart();

src/Parser/parser.yy

@@ -10 +10 @@
 // Created On       : Sat Sep  1 20:22:55 2001
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Fri Mar  6 17:26:45 2020
-// Update Count     : 4474
+// Last Modified On : Mon Apr 27 12:25:42 2020
+// Update Count     : 4483
 //
 
@@ -966 +966 @@
 
 tuple_expression_list:
-        assignment_expression_opt
-        | tuple_expression_list ',' assignment_expression_opt
+        assignment_expression
+        | '@'                                                                                           // CFA
+                { SemanticError( yylloc, "Eliding tuple element with '@' is currently unimplemented." ); $$ = nullptr; }
+        | tuple_expression_list ',' assignment_expression
                 { $$ = (ExpressionNode *)($1->set_last( $3 )); }
+        | tuple_expression_list ',' '@'
+                { SemanticError( yylloc, "Eliding tuple element with '@' is currently unimplemented." ); $$ = nullptr; }
         ;
 

tests/exceptions/conditional.cfa

@@ -4 +4 @@
 // up the non-trivial exception is reasonable to do.
 
-#include "except-mac.hfa"
+#include <exception.hfa>
 #include <stdio.h>
 
-DECLARE_EXCEPT(num_error, BASE_EXCEPT,
+VTABLE_DECLARATION(num_error)(
         int (*code)(num_error *this);
 );
@@ -36 +36 @@
     this.num = other.num;
 }
-void copy(num_error * this, num_error * other) {
-        *this = *other;
-}
 void ^?{}(num_error & this) {
     if( this.msg ) free( this.msg );
@@ -46 +43 @@
 }
 
-VTABLE_INSTANCE(num_error, BASE_EXCEPT, copy, ^?{},
-        num_error_msg, num_error_code
+VTABLE_INSTANCE(num_error)(
+        num_error_msg,
+        num_error_code,
 );
 
@@ -58 +56 @@
 
         try {
-                THROW(&exc);
+                throw &exc;
         } catch (num_error * error ; 3 == error->virtual_table->code( error )) {
                 caught_num_error(3, error);
@@ -66 +64 @@
 
         try {
-                THROW_RESUME(&exc);
+                throwResume &exc;
         } catchResume (num_error * error ; 3 == error->virtual_table->code( error )) {
                 caught_num_error(3, error);

tests/exceptions/finally.cfa

@@ -1 +1 @@
 // Finally Clause Tests
 
-#include "except-mac.hfa"
+#include <exception.hfa>
 #include "except-io.hfa"
 
@@ -12 +12 @@
                 try {
                         printf("termination throw\n");
-                        THROW(&exc);
+                        throw &exc;
                 } finally {
                         loud_exit a = "termination inner finally";
@@ -28 +28 @@
                 try {
                         printf("resumption throw\n");
-                        THROW_RESUME(&exc);
+                        throwResume &exc;
                 } finally {
                         loud_exit a = "resumption inner finally";

tests/exceptions/interact.cfa

@@ -1 +1 @@
 // Testing Interactions Between Termination and Resumption
 
-#include "except-mac.hfa"
+#include <exception.hfa>
 #include "except-io.hfa"
 
@@ -10 +10 @@
         // Resume falls back to terminate.
         try {
-                THROW_RESUME(&(star){});
+                throwResume &(star){};
         } catch (star *) {
                 printf("caught as termination\n");
@@ -17 +17 @@
         try {
                 loud_region a = "try block with resume throw";
-                THROW_RESUME(&(star){});
+                throwResume &(star){};
         } catch (star *) {
                 printf("caught as termination\n");
@@ -29 +29 @@
         try {
                 try {
-                        THROW(&(star){});
+                        throw &(star){};
                 } catchResume (star *) {
                         printf("resume catch on terminate\n");
@@ -43 +43 @@
         try {
                 try {
-                        THROW_RESUME(&(star){});
+                        throwResume &(star){};
                 } catch (star *) {
                         printf("terminate catch on resume\n");
@@ -58 +58 @@
                 try {
                         try {
-                                THROW(&(star){});
+                                throw &(star){};
                         } catchResume (star *) {
                                 printf("inner resume catch (error)\n");
@@ -64 +64 @@
                 } catch (star * error) {
                         printf("termination catch, will resume\n");
-                        THROW_RESUME(error);
+                        throwResume error;
                 }
         } catchResume (star *) {
@@ -75 +75 @@
                 try {
                         try {
-                                THROW_RESUME(&(star){});
+                                throwResume &(star){};
                         } catch (star *) {
                                 printf("inner termination catch\n");
@@ -81 +81 @@
                 } catchResume (star * error) {
                         printf("resumption catch, will terminate\n");
-                        THROW(error);
+                        throw error;
                 }
         } catch (star *) {
@@ -94 +94 @@
                                 try {
                                         printf("throwing resume moon\n");
-                                        THROW_RESUME(&(moon){});
+                                        throwResume &(moon){};
                                 } catch (star *) {
                                         printf("termination catch\n");
                                 }
                                 printf("throwing resume star\n");
-                                THROW_RESUME(&(star){});
+                                throwResume &(star){};
                         } catchResume (star *) {
                                 printf("resumption star catch\n");
@@ -105 +105 @@
                 } catchResume (moon *) {
                         printf("resumption moon catch, will terminate\n");
-                        THROW(&(star){});
+                        throw &(star){};
                 }
         } catchResume (star *) {

tests/exceptions/resume.cfa

@@ -1 +1 @@
 // Resumption Exception Tests
 
-#include "except-mac.hfa"
+#include <exception.hfa>
 #include "except-io.hfa"
 
@@ -14 +14 @@
                 loud_exit a = "simple try clause";
                 printf("simple throw\n");
-                THROW_RESUME(&(zen){});
+                throwResume &(zen){};
                 printf("end of try clause\n");
         } catchResume (zen * error) {
@@ -25 +25 @@
         try {
                 printf("throwing child exception\n");
-                THROW_RESUME(&(moment_of){});
+                throwResume &(moment_of){};
         } catchResume (zen *) {
                 printf("inner parent match\n");
@@ -36 +36 @@
         try {
                 try {
-                        THROW_RESUME(&(yin){});
+                        throwResume &(yin){};
                 } catchResume (zen *) {
                         printf("caught yin as zen\n");
@@ -52 +52 @@
                         loud_exit a = "rethrow inner try";
                         printf("rethrow inner try\n");
-                        THROW_RESUME(&(zen){});
+                        throwResume &(zen){};
                 } catchResume (zen *) {
                         loud_exit a = "rethrowing catch clause";
@@ -67 +67 @@
         try {
                 try {
-                        THROW_RESUME(&(yin){});
+                        throwResume &(yin){};
                 } catchResume (yin *) {
                         printf("caught yin, will throw yang\n");
-                        THROW_RESUME(&(yang){});
+                        throwResume &(yang){};
                 } catchResume (yang *) {
                         printf("caught exception from same try\n");
@@ -83 +83 @@
                 try {
                         printf("throwing first exception\n");
-                        THROW_RESUME(&(yin){});
+                        throwResume &(yin){};
                 } catchResume (yin *) {
                         printf("caught first exception\n");
                         try {
                                 printf("throwing second exception\n");
-                                THROW_RESUME(&(yang){});
+                                throwResume &(yang){};
                         } catchResume (yang *) {
                                 printf("caught second exception\n");
@@ -104 +104 @@
         try {
                 try {
-                        THROW_RESUME(&(zen){});
-                        THROW_RESUME(&(zen){});
+                        throwResume &(zen){};
+                        throwResume &(zen){};
                 } catchResume (zen *) {
                         printf("inner catch\n");
                 }
-                THROW_RESUME(&(zen){});
+                throwResume &(zen){};
         } catchResume (zen *) {
                 printf("outer catch\n");

tests/exceptions/terminate.cfa

@@ -1 +1 @@
 // Termination Exception Tests
 
-#include "except-mac.hfa"
+#include <exception.hfa>
 #include "except-io.hfa"
 
@@ -14 +14 @@
                 loud_exit a = "simple try clause";
                 printf("simple throw\n");
-                THROW(&(zen){});
+                throw &(zen){};
                 printf("end of try clause\n");
         } catch (zen * error) {
@@ -25 +25 @@
         try {
                 printf("throwing child exception\n");
-                THROW(&(moment_of){});
+                throw &(moment_of){};
         } catch (zen *) {
                 printf("inner parent match\n");
@@ -36 +36 @@
         try {
                 try {
-                        THROW(&(yin){});
+                        throw &(yin){};
                 } catch (zen *) {
                         printf("caught yin as zen\n");
@@ -52 +52 @@
                         loud_exit a = "rethrow inner try";
                         printf("rethrow inner try\n");
-                        THROW(&(zen){});
+                        throw &(zen){};
                 } catch (zen *) {
                         loud_exit a = "rethrowing catch clause";
@@ -67 +67 @@
         try {
                 try {
-                        THROW(&(yin){});
+                        throw &(yin){};
                 } catch (yin *) {
                         printf("caught yin, will throw yang\n");
-                        THROW(&(yang){});
+                        throw &(yang){};
                 } catch (yang *) {
                         printf("caught exception from same try\n");
@@ -83 +83 @@
                 try {
                         printf("throwing first exception\n");
-                        THROW(&(yin){});
+                        throw &(yin){};
                 } catch (yin *) {
                         printf("caught first exception\n");
                         try {
                                 printf("throwing second exception\n");
-                                THROW(&(yang){});
+                                throw &(yang){};
                         } catch (yang *) {
                                 printf("caught second exception\n");
@@ -104 +104 @@
         try {
                 try {
-                        THROW(&(zen){});
-                        THROW(&(zen){});
+                        throw &(zen){};
+                        throw &(zen){};
                 } catch (zen *) {
                         printf("inner catch\n");
                 }
-                THROW(&(zen){});
+                throw &(zen){};
         } catch (zen *) {
                 printf("outer catch\n");

tests/manipulatorsOutput1.cfa

@@ -7 +7 @@
 // Created On       : Sat Jun  8 18:04:11 2019
 // Last Modified By : Peter A. Buhr
-// Last Modified On : Mon Jun 10 12:37:28 2019
-// Update Count     : 8
+// Last Modified On : Fri May  1 11:51:44 2020
+// Update Count     : 9
 // 
 
@@ -17 +17 @@
         signed char sc = -12;
         printf( "%hhd %2hhd %5.2hhd %-5.2hhd %hho %#hho %hhx %#hhx %#8hhx %#8.10hhx %#8.3hhX %+-8.3hhd %08hhd\n", sc, sc, sc, sc, sc, sc, sc, sc, sc, sc, sc, sc, sc );
-        sout | sc | wd(2,sc) | wd(5,2,sc) | left(wd(5,2,sc)) | nobase(oct(sc)) | oct(sc) | nobase(hex(sc)) | hex(sc) | wd(8,hex(sc)) | wd(8,10,hex(sc)) | upcase(wd(8,3,hex(sc))) | left(sign(upcase(wd(8,3,sc)))) | pad0(wd(8,sc));
+        sout | sc | wd(2,sc) | wd(5,2,sc) | left(wd(5,2,sc)) | nobase(oct(sc)) | oct(sc) | nonl;
+        sout | nobase(hex(sc)) | hex(sc) | wd(8,hex(sc)) | wd(8,10,hex(sc)) | upcase(wd(8,3,hex(sc))) | nonl;
+        sout | left(sign(upcase(wd(8,3,sc)))) | pad0(wd(8,sc));
 
         sout | "unsigned char";
         unsigned char usc = 12;
         printf( "%hhu %2hhu %5.2hhu %-5.2hhu %hho %#hho %hhx %#hhx %#8hhx %#8.10hhx %#8.3hhX %-8.3hhu %08hhu\n", usc, usc, usc, usc, usc, usc, usc, usc, usc, usc, usc, usc, usc );
-        sout | usc | wd(2,usc) | wd(5,2,usc) | left(wd(5,2,usc)) | nobase(oct(usc)) | oct(usc) | nobase(hex(usc)) | hex(usc) | wd(8,hex(usc)) | wd(8,10,hex(usc)) | upcase(wd(8,3,hex(usc))) | left(upcase(wd(8,3,usc))) | pad0(wd(8,usc));
+        sout | usc | wd(2,usc) | wd(5,2,usc) | left(wd(5,2,usc)) | nobase(oct(usc)) | oct(usc) | nonl;
+        sout | nobase(hex(usc)) | hex(usc) | wd(8,hex(usc)) | wd(8,10,hex(usc)) | upcase(wd(8,3,hex(usc))) | nonl;
+        sout | left(upcase(wd(8,3,usc))) | pad0(wd(8,usc));
 
         sout | "signed short int";
         signed short int si = -12;
         printf( "%hd %2hd %5.2hd %-5.2hd %ho %#ho %hx %#hx %#8hx %#8.10hx %#8.3hX %+-8.3hd %08hd\n", si, si, si, si, si, si, si, si, si, si, si, si, si );
-        sout | si | wd(2,si) | wd(5,2,si) | left(wd(5,2,si)) | nobase(oct(si)) | oct(si) | nobase(hex(si)) | hex(si) | wd(8,hex(si)) | wd(8,10,hex(si)) | upcase(wd(8,3,hex(si))) | left(sign(upcase(wd(8,3,si)))) | pad0(wd(8,si));
+        sout | si | wd(2,si) | wd(5,2,si) | left(wd(5,2,si)) | nobase(oct(si)) | oct(si) | nonl;
+        sout | nobase(hex(si)) | hex(si) | wd(8,hex(si)) | wd(8,10,hex(si)) | upcase(wd(8,3,hex(si))) | nonl;
+        sout | left(sign(upcase(wd(8,3,si)))) | pad0(wd(8,si));
 
         sout | "unsigned short int";
         unsigned short int usi = 12;
         printf( "%hu %2hu %5.2hu %-5.2hu %ho %#ho %hx %#hx %#8hx %#8.10hx %#8.3hX %-8.3hu %08hu\n", usi, usi, usi, usi, usi, usi, usi, usi, usi, usi, usi, usi, usi );
-        sout | usi | wd(2,usi) | wd(5,2,usi) | left(wd(5,2,usi)) | nobase(oct(usi)) | oct(usi) | nobase(hex(usi)) | hex(usi) | wd(8,hex(usi)) | wd(8,10,hex(usi)) | upcase(wd(8,3,hex(usi))) | left(upcase(wd(8,3,usi))) | pad0(wd(8,usi));
+        sout | usi | wd(2,usi) | wd(5,2,usi) | left(wd(5,2,usi)) | nobase(oct(usi)) | oct(usi) | nonl;
+        sout | nobase(hex(usi)) | hex(usi) | wd(8,hex(usi)) | wd(8,10,hex(usi)) | upcase(wd(8,3,hex(usi))) | nonl;
+        sout | left(upcase(wd(8,3,usi))) | pad0(wd(8,usi));
 
         sout | "signed int";
         signed int i = -12;
         printf( "%d %2d %5.2d %-5.2d %o %#o %x %#x %#8x %#8.10x %#8.3X %+-8.3d %08d\n", i, i, i, i, i, i, i, i, i, i, i, i, i );
-        sout | i | wd(2,i) | wd(5,2,i) | left(wd(5,2,i)) | nobase(oct(i)) | oct(i) | nobase(hex(i)) | hex(i) | wd(8,hex(i)) | wd(8,10,hex(i)) | upcase(wd(8,3,hex(i))) | left(sign(upcase(wd(8,3,i)))) | pad0(wd(8,i));
+        sout | i | wd(2,i) | wd(5,2,i) | left(wd(5,2,i)) | nobase(oct(i)) | oct(i) | nonl;
+        sout | nobase(hex(i)) | hex(i) | wd(8,hex(i)) | wd(8,10,hex(i)) | upcase(wd(8,3,hex(i))) | nonl;
+        sout | left(sign(upcase(wd(8,3,i)))) | pad0(wd(8,i));
 
         sout | "unsigned int";
         unsigned int ui = 12;
         printf( "%u %2u %5.2u %-5.2u %o %#o %x %#x %#8x %#8.10x %#8.3X %-8.3u %08u\n", ui, ui, ui, ui, ui, ui, ui, ui, ui, ui, ui, ui, ui );
-        sout | ui | wd(2,ui) | wd(5,2,ui) | left(wd(5,2,ui)) | nobase(oct(ui)) | oct(ui) | nobase(hex(ui)) | hex(ui) | wd(8,hex(ui)) | wd(8,10,hex(ui)) | upcase(wd(8,3,hex(ui))) | left(upcase(wd(8,3,ui))) | pad0(wd(8,ui));
+        sout | ui | wd(2,ui) | wd(5,2,ui) | left(wd(5,2,ui)) | nobase(oct(ui)) | oct(ui) | nonl;
+        sout | nobase(hex(ui)) | hex(ui) | wd(8,hex(ui)) | wd(8,10,hex(ui)) | upcase(wd(8,3,hex(ui))) | nonl;
+        sout | left(upcase(wd(8,3,ui))) | pad0(wd(8,ui));
 
         sout | "signed long long int";
         signed long long int lli = -12;
         printf( "%lld %2lld %5.2lld %-5.2lld %llo %#llo %llx %#llx %#8llx %#8.10llx %#8.3llX %+-8.3lld %08lld\n", lli, lli, lli, lli, lli, lli, lli, lli, lli, lli, lli, lli, lli );
-        sout | lli | wd(2,lli) | wd(5,2,lli) | left(wd(5,2,lli)) | nobase(oct(lli)) | oct(lli) | nobase(hex(lli)) | hex(lli) | wd(8,hex(lli)) | wd(8,10,hex(lli)) | upcase(wd(8,3,hex(lli))) | left(sign(upcase(wd(8,3,lli)))) | pad0(wd(8,lli));
+        sout | lli | wd(2,lli) | wd(5,2,lli) | left(wd(5,2,lli)) | nobase(oct(lli)) | oct(lli) | nonl;
+        sout | nobase(hex(lli)) | hex(lli) | wd(8,hex(lli)) | wd(8,10,hex(lli)) | upcase(wd(8,3,hex(lli))) | nonl;
+        sout | left(sign(upcase(wd(8,3,lli)))) | pad0(wd(8,lli));
 
         sout | "unsigned long long int";
         unsigned long long int ulli = 12;
         printf( "%llu %2llu %5.2llu %-5.2llu %llo %#llo %llx %#llx %#8llx %#8.10llx %#8.3llX %-8.3llu %08llu\n", ulli, ulli, ulli, ulli, ulli, ulli, ulli, ulli, ulli, ulli, ulli, ulli, ulli );
-        sout | ulli | wd(2,ulli) | wd(5,2,ulli) | left(wd(5,2,ulli)) | nobase(oct(ulli)) | oct(ulli) | nobase(hex(ulli)) | hex(ulli) | wd(8,hex(ulli)) | wd(8,10,hex(ulli)) | upcase(wd(8,3,hex(ulli))) | left(upcase(wd(8,3,ulli))) | pad0(wd(8,ulli));
+        sout | ulli | wd(2,ulli) | wd(5,2,ulli) | left(wd(5,2,ulli)) | nobase(oct(ulli)) | oct(ulli) | nonl;
+        sout | nobase(hex(ulli)) | hex(ulli) | wd(8,hex(ulli)) | wd(8,10,hex(ulli)) | upcase(wd(8,3,hex(ulli))) | nonl;
+        sout | left(upcase(wd(8,3,ulli))) | pad0(wd(8,ulli));
 
         sout | nl | "binary integral";
-        sout | bin(0) | bin(13) | upcase(bin(13)) | nobase(bin(13)) | left(wd(8,bin(13))) | wd(8,bin(13)) | pad0(left(wd(8,bin(13)))) | pad0(wd(8,bin(13))) | pad0(wd(8,10,bin(13))) | pad0(wd(8,6,bin(13)));
+        sout | bin(0) | bin(13) | upcase(bin(13)) | nobase(bin(13)) | left(wd(8,bin(13))) | wd(8,bin(13)) | nonl;
+        sout | pad0(left(wd(8,bin(13)))) | pad0(wd(8,bin(13))) | pad0(wd(8,10,bin(13))) | pad0(wd(8,6,bin(13)));
 
 
@@ -62 +79 @@
         printf( "%g  %8g %#8g %g %8g %8.0g %#8.0g %8.2g %#8.2g %-8.2g %-8.2g %-#8.2g %-+8.2g %-+#8.2g %08.2g %8.2E %8.2a %#8.2A %#8.2e\n",
                     0.0,3.0F,3.0F, f,  f,    f,     f,    f,     f,  3.0F,      f,      f,      f,       f,     f,    f,    f,     f,     f );
-        sout | 0.0 | wd(8, 3.0F) | nodp(wd(8, 3.0F)) | f | wd(8, f) | ws(8,0, f) | nodp(ws(8,0, f)) | ws(8,2, f) | nodp(ws(8,2, f)) | left(ws(8,2, 3.0F)) | left(ws(8,2, f)) | left(nodp(ws(8,2, f))) | left(sign(ws(8,2, f))) | left(sign(nodp(ws(8,2, f)))) | pad0(ws(8,2, f)) | upcase(wd(8,2, sci(f))) | wd(8,2, hex(f)) | upcase(wd(8,2, hex(f))) | nodp(wd(8,2, sci(f)));
+        sout | 0.0 | wd(8, 3.0F) | nodp(wd(8, 3.0F)) | f | wd(8, f) | ws(8,0, f) | nodp(ws(8,0, f)) | ws(8,2, f) | nodp(ws(8,2, f)) | nonl;
+        sout | left(ws(8,2, 3.0F)) | left(ws(8,2, f)) | left(nodp(ws(8,2, f))) | left(sign(ws(8,2, f))) | left(sign(nodp(ws(8,2, f)))) | nonl;
+        sout | pad0(ws(8,2, f)) | upcase(wd(8,2, sci(f))) | wd(8,2, hex(f)) | upcase(wd(8,2, hex(f))) | nodp(wd(8,2, sci(f)));
 
         sout | "double";
@@ -68 +87 @@
         printf( "%g  %#8f %g %8f %#8.0f %8.0f %8.2f %-8.2f %-+#8.2f %08.2F %8.2E %8.2a %8.2A %8.2e\n",
                         0.0,  3.0, d,  d,     d,    d,    d,     d,       d,     d,    d,    d,    d,    d );
-        sout | 0.0 | wd(8, 3.0) | d | wd(8, d) | nodp(wd(8,0, d)) | wd(8,0, d) | wd(8,2, d) | left(wd(8,2, d)) | left(sign(wd(8,2, d))) | pad0(upcase(wd(8,2, d))) | upcase(wd(8,2, sci(d))) | wd(8,2, hex(d)) | upcase(wd(8,2, hex(d))) | wd(8,2, sci(d));
+        sout | 0.0 | wd(8, 3.0) | d | wd(8, d) | nodp(wd(8,0, d)) | wd(8,0, d) | wd(8,2, d) | nonl;
+        sout | left(wd(8,2, d)) | left(sign(wd(8,2, d))) | pad0(upcase(wd(8,2, d))) | upcase(wd(8,2, sci(d))) | wd(8,2, hex(d)) | upcase(wd(8,2, hex(d))) | wd(8,2, sci(d));
 
         sout | "long double";
@@ -74 +94 @@
         printf( "%Lg  %#8Lf %Lg %8Lf %#8.0Lf %8.0Lf %8.2Lf %-8.2Lf %-+#8.2Lf %08.2LF %8.2LE %8.2La %8.2LA %8.2Le\n",
                         0.0L,  3.0L, ld,  ld,     ld,    ld,    ld,     ld,       ld,     ld,    ld,    ld,    ld,    ld );
-        sout | 0.0L | wd(8, 3.0L) | ld | wd(8, ld) | nodp(wd(8,0, ld)) | wd(8,0, ld) | wd(8,2, ld) | left(wd(8,2, ld)) | left(sign(wd(8,2, ld))) | pad0(upcase(wd(8,2, ld))) | upcase(wd(8,2, sci(ld))) | wd(8,2, hex(ld)) | upcase(wd(8,2, hex(ld))) | wd(8,2, sci(ld));
+        sout | 0.0L | wd(8, 3.0L) | ld | wd(8, ld) | nodp(wd(8,0, ld)) | wd(8,0, ld) | wd(8,2, ld) | nonl;
+        sout | left(wd(8,2, ld)) | left(sign(wd(8,2, ld))) | pad0(upcase(wd(8,2, ld))) | upcase(wd(8,2, sci(ld))) | wd(8,2, hex(ld)) | upcase(wd(8,2, hex(ld))) | wd(8,2, sci(ld));
 
 
@@ -80 +101 @@
         char c = 'a';
         printf( "%c %2c %5c %-5c %hho %#hho %hhx %#hhx %#8hhx %#8hhX %-8c %8c\n", c, c, c, c, c, c, c, c, c, c, c, c );
-        sout | c | ' ' | wd(2,c) | wd(5,c) | left(wd(5,c)) | nobase(oct(c)) | oct(c) | nobase(hex(c)) | hex(c) | wd(8,hex(c)) | upcase(wd(8,hex(c))) | left(wd(8,c)) | wd(8,c);
+        sout | c | ' ' | wd(2,c) | wd(5,c) | left(wd(5,c)) | nobase(oct(c)) | oct(c) | nonl;
+        sout | nobase(hex(c)) | hex(c) | wd(8,hex(c)) | upcase(wd(8,hex(c))) | left(wd(8,c)) | wd(8,c);
 
         sout | nl | "string";

tests/manipulatorsOutput3.cfa

@@ -52 +52 @@
     sout | left(wd( 10,40, oct(y) )) | 'X';
 
-    y = 123456789;
+    y = 01234567;
     sout | left(wd( 45, 49, oct(y) )) | 'X';
+    y = -y;
+    sout | wd(0, oct(y)) | 'Y';
+    sout | left(wd(0, oct(y))) | 'Y';
     sout | nl;
 
@@ -66 +69 @@
     sout | left(wd( 45,49, upcase(hex(x)) )) | 'X';
 
-    sout | nl | nl;
+    sout | nl;
 
     int128 divisor = 0x4b3b4ca85a86c47a;
     divisor <<= 16;
     divisor += 0x98a224000000000;
+    
+    // base 2
+    sout | "base 2";
+    sout | bin(divisor);
+    sout | upcase(bin(divisor));
+    sout | wd(38, upcase(bin(divisor)));
+    sout | wd(40, upcase(bin(divisor)));
+    sout | wd(40, 38, upcase(bin(divisor)));
+    sout | wd(40, 30, upcase(bin(divisor)));
+    sout | pad0(sign(wd(40, 38, upcase(bin(divisor)))));
+    sout | nl;
+    
+    // oct
+    sout | "base 8";
+    sout | upcase(oct(divisor));
+    sout | wd(38, upcase(oct(divisor)));
+    sout | wd(40, upcase(oct(divisor)));
+    sout | wd(40, 38, upcase(oct(divisor)));
+    sout | wd(40, 30, upcase(oct(divisor)));
+    sout | pad0(sign(wd(40, 38, upcase(oct(divisor)))));
+    sout | nl;
 
-    // base 10
+    // decimal
+    sout | "base 10";
     sout | divisor;
     sout | wd(2, divisor);
@@ -87 +112 @@
     sout | pad0(wd(40, divisor));
     sout | pad0(sign(wd(40,divisor)));
-    
-    // base 2
-    sout | upcase(bin(divisor));
-    sout | wd(38, upcase(bin(divisor)));
-    sout | wd(40, upcase(bin(divisor)));
-    sout | wd(40, 38, upcase(bin(divisor)));
-    sout | wd(40, 30, upcase(bin(divisor)));
-    sout | pad0(sign(wd(40, 38, upcase(bin(divisor)))));
-    
-    // oct
-    sout | upcase(oct(divisor));
-    sout | wd(38, upcase(oct(divisor)));
-    sout | wd(40, upcase(oct(divisor)));
-    sout | wd(40, 38, upcase(oct(divisor)));
-    sout | wd(40, 30, upcase(oct(divisor)));
-    sout | pad0(sign(wd(40, 38, upcase(oct(divisor)))));
+    sout | nl;
     
     // hex
+    sout | "base 16";
     sout | upcase(hex(divisor));
     sout | wd(38, upcase(hex(divisor)));
@@ -111 +122 @@
     sout | wd(40, 30, upcase(hex(divisor)));
     sout | pad0(sign(wd(40, 38, upcase(hex(divisor)))));
+    sout | nl;