source: libcfa/src/concurrency/coroutine.cfa

//
// Cforall Version 1.0.0 Copyright (C) 2016 University of Waterloo
//
// The contents of this file are covered under the licence agreement in the
// file "LICENCE" distributed with Cforall.
//
// coroutine.c --
//
// Author           : Thierry Delisle
// Created On       : Mon Nov 28 12:27:26 2016
// Last Modified By : Peter A. Buhr
// Last Modified On : Mon Sep 18 21:47:12 2023
// Update Count     : 25
//

#define __cforall_thread__

#include "coroutine.hfa"

#include <stddef.h>
#include <malloc.h>
#include <errno.h>
#include <string.h>
#include <unistd.h>
#include <sys/mman.h>                                                                   // mprotect
#include <unwind.h>

#include "kernel/private.hfa"
#include "exception.hfa"
#include "exception.h"
#include "math.hfa"

#define CFA_COROUTINE_USE_MMAP 0

#define __CFA_INVOKE_PRIVATE__
#include "invoke.h"

extern "C" {
        void _CtxCoroutine_Unwind(struct _Unwind_Exception * storage, struct coroutine$ *) __attribute__ ((__noreturn__));
        static void _CtxCoroutine_UnwindCleanup(_Unwind_Reason_Code, struct _Unwind_Exception *) __attribute__ ((__noreturn__));
        static void _CtxCoroutine_UnwindCleanup(_Unwind_Reason_Code, struct _Unwind_Exception *) {
                abort();
        }

        extern void CtxRet( struct __stack_context_t * to ) asm ("CtxRet") __attribute__ ((__noreturn__));
}

//-----------------------------------------------------------------------------
forall(T &)
void copy(CoroutineCancelled(T) * dst, CoroutineCancelled(T) * src) libcfa_public {
        dst->virtual_table = src->virtual_table;
        dst->the_coroutine = src->the_coroutine;
        dst->the_exception = src->the_exception;
}

forall(T &)
const char * msg(CoroutineCancelled(T) *) libcfa_public {
        return "CoroutineCancelled(...)";
}

// This code should not be inlined. It is the error path on resume.
forall(T & | is_coroutine(T))
void __cfaehm_cancelled_coroutine(
                T & cor, coroutine$ * desc, EHM_DEFAULT_VTABLE(CoroutineCancelled(T)) ) libcfa_public {
        verify( desc->cancellation );
        desc->state = Cancelled;
        exception_t * except = __cfaehm_cancellation_exception( desc->cancellation );

        // TODO: Remove explitate vtable set once trac#186 is fixed.
        CoroutineCancelled(T) except;
        except.virtual_table = &_default_vtable;
        except.the_coroutine = &cor;
        except.the_exception = except;
        // Why does this need a cast?
        throwResume (CoroutineCancelled(T) &)except;

        except->virtual_table->free( except );
        free( desc->cancellation );
        desc->cancellation = 0p;
}

// helper for popping from coroutine's ehm buffer
inline nonlocal_exception * pop_ehm_head( coroutine$ * this ) {
    lock( this->ehm_state.buffer_lock __cfaabi_dbg_ctx2 );
    nonlocal_exception * nl_ex = pop_head( this->ehm_state.ehm_buffer );
    unlock( this->ehm_state.buffer_lock );
    return nl_ex;
}

//-----------------------------------------------------------------------------
// Global state variables

// minimum feasible stack size in bytes
static const size_t MinStackSize = 1000;
extern size_t __page_size;                              // architecture pagesize HACK, should go in proper runtime singleton
extern int __map_prot;

void __stack_prepare( __stack_info_t * this, size_t create_size );
static void __stack_clean  ( __stack_info_t * this );

//-----------------------------------------------------------------------------
// Coroutine ctors and dtors
void ?{}( __stack_info_t & this, void * storage, size_t storageSize ) {
        this.storage   = (__stack_t *)storage;

        // Did we get a piece of storage ?
        if (this.storage || storageSize != 0) {
                // We either got a piece of storage or the user asked for a specific size
                // Immediately create the stack
                // (This is slightly unintuitive that non-default sized coroutines create are eagerly created
                // but it avoids that all coroutines carry an unnecessary size)
                verify( storageSize != 0 );
                __stack_prepare( &this, storageSize );
        }
}

void ^?{}(__stack_info_t & this) {
        bool userStack = ((intptr_t)this.storage & 0x1) != 0;
        if ( ! userStack && this.storage ) {
                __stack_clean( &this );
        }
}

void ?{}( coroutine$ & this, const char name[], void * storage, size_t storageSize ) libcfa_public with( this ) {
        (this.context){0p, 0p};
        (this.stack){storage, storageSize};
        this.name = name;
        state = Start;
        starter = 0p;
        last = 0p;
        cancellation = 0p;
    ehm_state.ehm_buffer{};
    ehm_state.buffer_lock{};
    ehm_state.ehm_enabled = false;
}

void ^?{}(coroutine$& this) libcfa_public {
    // handle any leftover pending non-local exceptions
    nonlocal_exception * nl_ex = pop_ehm_head( &this );
    unsigned unhandled_ex = 0;
    
    // if any leftover exceptions handle 
    while ( nl_ex != 0p ){
        unhandled_ex++;
        free( nl_ex->the_exception );
        free( nl_ex );
        nl_ex = pop_ehm_head( &this );
    }

    #ifdef __CFA_DEBUG__
    if ( unhandled_ex > 0 )
        printf( "Warning: Coroutine %p exited with %u pending nonlocal exceptions.\n", &this, unhandled_ex );
    #endif

        if(this.state != Halted && this.state != Start && this.state != Primed) {
                coroutine$ * src = active_coroutine();
                coroutine$ * dst = &this;

                struct _Unwind_Exception storage;
                storage.exception_class = -1;
                storage.exception_cleanup = _CtxCoroutine_UnwindCleanup;
                this.cancellation = &storage;
                this.last = src;

                // not resuming self ?
                if ( src == dst ) {
                        abort( "Attempt by coroutine %.256s (%p) to terminate itself.\n", src->name, src );
                }

                $ctx_switch( src, dst );
        }
}

// Part of the Public API
// Not inline since only ever called once per coroutine
forall(T & | is_coroutine(T) | { EHM_DEFAULT_VTABLE(CoroutineCancelled(T)); })
void prime(T& cor) libcfa_public {
        coroutine$* this = get_coroutine(cor);
        assert(this->state == Start);

        this->state = Primed;
        resume(cor);
}

static [void *, size_t] __stack_alloc( size_t storageSize ) {
        const size_t stack_data_size = libCeiling( sizeof(__stack_t), 16 ); // minimum alignment
        assert(__page_size != 0l);
        size_t size = libCeiling( storageSize, 16 ) + stack_data_size;
        size = ceiling(size, __page_size);

        // If we are running debug, we also need to allocate a guardpage to catch stack overflows.
        void * storage;
        #if CFA_COROUTINE_USE_MMAP
                storage = mmap(0p, size + __page_size, PROT_EXEC | PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
                if(storage == ((void*)-1)) {
                        abort( "coroutine stack creation : internal error, mmap failure, error(%d) %s.", errno, strerror( errno ) );
                }
                if ( mprotect( storage, __page_size, PROT_NONE ) == -1 ) {
                        abort( "coroutine stack creation : internal error, mprotect failure, error(%d) %s.", errno, strerror( errno ) );
                } // if
                storage = (void *)(((intptr_t)storage) + __page_size);
        #else
                __cfaabi_dbg_debug_do(
                        storage = memalign( __page_size, size + __page_size );
                );
                __cfaabi_dbg_no_debug_do(
                        storage = (void*)malloc(size);
                );

                __cfaabi_dbg_debug_do(
                        if ( mprotect( storage, __page_size, PROT_NONE ) == -1 ) {
                                abort( "__stack_alloc : internal error, mprotect failure, error(%d) %s.", (int)errno, strerror( (int)errno ) );
                        }
                        storage = (void *)(((intptr_t)storage) + __page_size);
                );
        #endif
        __cfaabi_dbg_print_safe("Kernel : Created stack %p of size %zu\n", storage, size);

        verify( ((intptr_t)storage & (libAlign() - 1)) == 0ul );
        return [storage, size];
}

static void __stack_clean  ( __stack_info_t * this ) {
        void * storage = this->storage->limit;

        #if CFA_COROUTINE_USE_MMAP
                size_t size = ((intptr_t)this->storage->base) - ((intptr_t)this->storage->limit) + sizeof(__stack_t);
                storage = (void *)(((intptr_t)storage) - __page_size);
                if(munmap(storage, size + __page_size) == -1) {
                        abort( "coroutine stack destruction : internal error, munmap failure, error(%d) %s.", errno, strerror( errno ) );
                }
        #else
                __cfaabi_dbg_debug_do(
                        storage = (char*)(storage) - __page_size;
                        if ( mprotect( storage, __page_size, __map_prot ) == -1 ) {
                                abort( "(coStack_t *)%p.^?{}() : internal error, mprotect failure, error(%d) %s.", &this, errno, strerror( errno ) );
                        }
                );

                free( storage );
        #endif
        __cfaabi_dbg_print_safe("Kernel : Deleting stack %p\n", storage);
}

void __stack_prepare( __stack_info_t * this, size_t create_size ) libcfa_public {
        const size_t stack_data_size = libCeiling( sizeof(__stack_t), 16 ); // minimum alignment
        bool userStack;
        void * storage;
        size_t size;
        if ( !this->storage ) {
                userStack = false;
                [storage, size] = __stack_alloc( create_size );
        } else {
                userStack = true;
                __cfaabi_dbg_print_safe("Kernel : stack obj %p using user stack %p(%zd bytes)\n", this, this->storage, (intptr_t)this->storage->limit - (intptr_t)this->storage->base);

                // The stack must be aligned, advance the pointer to the next align data
                storage = (void*)libCeiling( (intptr_t)this->storage, libAlign());

                // The size needs to be shrinked to fit all the extra data structure and be aligned
                ptrdiff_t diff = (intptr_t)storage - (intptr_t)this->storage;
                size = libFloor(create_size - stack_data_size - diff, libAlign());
        } // if
        assertf( size >= MinStackSize, "Stack size %zd provides less than minimum of %zd bytes for a stack.", size, MinStackSize );

        this->storage = (__stack_t *)((intptr_t)storage + size - sizeof(__stack_t));
        this->storage->limit = storage;
        this->storage->base  = (void*)((intptr_t)storage + size - sizeof(__stack_t));
        this->storage->exception_context.top_resume = 0p;
        this->storage->exception_context.current_exception = 0p;
        __attribute__((may_alias)) intptr_t * istorage = (intptr_t*)&this->storage;
        *istorage |= userStack ? 0x1 : 0x0;
}

// We need to call suspend from invoke.c, so we expose this wrapper that
// is not inline (We can't inline Cforall in C)
extern "C" {
        void __cfactx_cor_leave( struct coroutine$ * src ) {
                coroutine$ * starter = src->cancellation != 0 ? src->last : src->starter;

                src->state = Halted;

                assertf( starter != 0,
                        "Attempt to suspend/leave coroutine \"%.256s\" (%p) that has never been resumed.\n"
                        "Possible cause is a suspend executed in a member called by a coroutine user rather than by the coroutine main.",
                        src->name, src );
                assertf( starter->state != Halted,
                        "Attempt by coroutine \"%.256s\" (%p) to suspend/leave back to terminated coroutine \"%.256s\" (%p).\n"
                        "Possible cause is terminated coroutine's main routine has already returned.",
                        src->name, src, starter->name, starter );

                $ctx_switch( src, starter );
        }

        struct coroutine$ * __cfactx_cor_finish(void) {
                struct coroutine$ * cor = active_coroutine();

                // get the active thread once
                thread$ * athrd = active_thread();

                /* paranoid */ verify( athrd->corctx_flag );
                athrd->corctx_flag = false;

                if(cor->state == Primed) {
                        __cfactx_suspend();
                }

                cor->state = Active;

                return cor;
        }
}


////////////////////////////////////////////////////////////////////////////////////////////////////
// non local ehm routines

void defaultResumeAtHandler( exception_t * except ) {
    __cfaehm_allocate_exception( except );
    __cfaehm_begin_unwind( (void(*)(exception_t *))defaultTerminationHandler );
}

struct ehm_cleanup {
        exception_t * ex;
};

void ^?{}( ehm_cleanup & this ) { free( this.ex ); }

bool poll( coroutine$ * cor ) libcfa_public {
    nonlocal_exception * nl_ex = pop_ehm_head( cor );

    // if no exceptions return false
    if ( nl_ex == 0p ) return false;
    
    // otherwise loop and throwResume all pending exceptions
    while ( nl_ex != 0p ){
                ehm_cleanup ex_holder{ nl_ex->the_exception };
        free( nl_ex );
        __cfaehm_throw_resume( ex_holder.ex , defaultResumeAtHandler );
        
        nl_ex = pop_ehm_head( cor );
    }
    
    return true;
}

bool poll() libcfa_public { return poll( active_coroutine() ); }
void enable_ehm() libcfa_public { active_coroutine()->ehm_state.ehm_enabled = true; }
void disable_ehm() libcfa_public { active_coroutine()->ehm_state.ehm_enabled = false; }
bool checked_poll() libcfa_public { return active_coroutine()->ehm_state.ehm_enabled ? poll( active_coroutine() ) : false; }
coroutine$ * resumer() libcfa_public { return active_coroutine()->last; }
coroutine$ * first_resumer() libcfa_public { return active_coroutine()->starter; }

// user facing ehm operations
forall(T & | is_coroutine(T)) {
    // enable/disable non-local exceptions
    void enable_ehm( T & cor ) libcfa_public { get_coroutine( cor )->ehm_state.ehm_enabled = true; }
    void disable_ehm( T & cor ) libcfa_public { get_coroutine( cor )->ehm_state.ehm_enabled = false; }

    // poll for non-local exceptions
    bool poll( T & cor ) libcfa_public { return poll( get_coroutine( cor ) ); }

    // poll iff nonlocal ehm is enabled
    bool checked_poll( T & cor ) libcfa_public { return get_coroutine( cor )->ehm_state.ehm_enabled ? poll( cor ) : false; }

    coroutine$ * resumer( T & cor ) libcfa_public { return get_coroutine( cor )->last; }
    coroutine$ * first_resumer( T & cor ) libcfa_public { return get_coroutine( cor )->starter; }
}

// resume non local exception at receiver (i.e. enqueue in ehm buffer)
forall(exceptT *, T & | ehm_resume_at( exceptT, T ))
void resumeAt( T & receiver, exceptT & ex ) libcfa_public {
    coroutine$ * cor = get_coroutine( receiver );
    nonlocal_exception * nl_ex = alloc();
    exceptT * ex_copy = alloc();
    memcpy( ex_copy, &ex, sizeof(exceptT) );
    (*nl_ex){ (exception_t *)ex_copy };
    lock( cor->ehm_state.buffer_lock __cfaabi_dbg_ctx2 );
    append( cor->ehm_state.ehm_buffer, nl_ex ); 
    unlock( cor->ehm_state.buffer_lock );
}

forall(exceptT * | { void $throwResume(exceptT &); })
void resumeAt( coroutine$ * receiver, exceptT & ex ) libcfa_public {
    nonlocal_exception * nl_ex = alloc();
    exceptT * ex_copy = alloc();
    memcpy( ex_copy, &ex, sizeof(exceptT) );
    (*nl_ex){ (exception_t *)ex_copy };
    lock( receiver->ehm_state.buffer_lock __cfaabi_dbg_ctx2 );
    append( receiver->ehm_state.ehm_buffer, nl_ex ); 
    unlock( receiver->ehm_state.buffer_lock );
}

// Local Variables: //
// mode: c //
// tab-width: 4 //
// End: //