Changeset 5b544a6

Timestamp:

Mar 9, 2020, 11:09:52 AM (4 years ago)

Author:

Thierry Delisle <tdelisle@…>

Branches:

ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, jenkins-sandbox, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum

Children:

6565321

Parents:

87f572e (diff), e6cfa8ff (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'generator'

Files:

• benchmark/ctxswitch/cfa_cor.cfa (modified) (1 diff)
• libcfa/prelude/builtins.c (modified) (1 diff)
• libcfa/src/concurrency/coroutine.cfa (modified) (1 diff)
• libcfa/src/concurrency/coroutine.hfa (modified) (2 diffs)
• src/AST/Convert.cpp (modified) (2 diffs)
• src/AST/Decl.hpp (modified) (1 diff)
• src/AST/Fwd.hpp (modified) (1 diff)
• src/AST/Pass.hpp (modified) (1 diff)
• src/AST/Pass.impl.hpp (modified) (1 diff)
• src/AST/Print.cpp (modified) (1 diff)
• src/AST/Stmt.hpp (modified) (1 diff)
• src/AST/Visitor.hpp (modified) (1 diff)
• src/Common/PassVisitor.h (modified) (2 diffs)
• src/Common/PassVisitor.impl.h (modified) (1 diff)
• src/Concurrency/Keywords.cc (modified) (5 diffs)
• src/Parser/ParseNode.h (modified) (1 diff)
• src/Parser/StatementNode.cc (modified) (1 diff)
• src/Parser/TypeData.cc (modified) (1 diff)
• src/Parser/lex.ll (modified) (2 diffs)
• src/Parser/parser.yy (modified) (3 diffs)
• src/SynTree/Declaration.h (modified) (1 diff)
• src/SynTree/Mutator.h (modified) (1 diff)
• src/SynTree/Statement.cc (modified) (1 diff)
• src/SynTree/Statement.h (modified) (1 diff)
• src/SynTree/SynTree.h (modified) (1 diff)
• src/SynTree/Visitor.h (modified) (1 diff)
• tests/concurrent/.expect/suspend_then.txt (added)
• tests/concurrent/coroutineYield.cfa (modified) (1 diff)
• tests/concurrent/suspend_then.cfa (moved) (moved from tests/concurrent/coroutineThen.cfa) (6 diffs)
• tests/coroutine/.expect/fmtLines.txt (modified) (1 diff)
• tests/coroutine/.in/fmtLines.txt (modified) (1 diff)
• tests/coroutine/cntparens.cfa (modified) (3 diffs)
• tests/coroutine/devicedriver.cfa (modified) (5 diffs)
• tests/coroutine/fibonacci.cfa (modified) (1 diff)
• tests/coroutine/fibonacci_1.cfa (modified) (2 diffs)
• tests/coroutine/fmtLines.cfa (modified) (1 diff)
• tests/coroutine/raii.cfa (modified) (1 diff)
• tests/coroutine/runningTotal.cfa (modified) (1 diff)
• tests/coroutine/suspend_then.cfa (modified) (1 diff)
• tests/errors/.expect/completeType.txt (modified) (3 diffs)
• tests/errors/suspend.cfa (added)
• tests/generator/.expect/fibonacci.txt (added)
• tests/generator/.expect/fmtLines.txt (added)
• tests/generator/.expect/suspend_then.txt (added)
• tests/generator/.in/fmtLines.txt (added)
• tests/generator/fibonacci.cfa (added)
• tests/generator/fmtLines.cfa (added)
• tests/generator/suspend_then.cfa (added)

benchmark/ctxswitch/cfa_cor.cfa

@@ -7 +7 @@
 void main( __attribute__((unused)) C & ) {
         while () {
-                suspend();
+                suspend;
         }
 }

libcfa/prelude/builtins.c

@@ -48 +48 @@
 void exit( int status, const char fmt[], ... ) __attribute__ (( format(printf, 2, 3), __nothrow__, __leaf__, __noreturn__ ));
 void abort( const char fmt[], ... ) __attribute__ (( format(printf, 1, 2), __nothrow__, __leaf__, __noreturn__ ));
+
+forall(dtype T)
+static inline T & identity(T & i) {
+        return i;
+}
+
+// generator support
+struct $generator {
+        inline int;
+};
+
+static inline void  ?{}($generator & this) { ((int&)this) = 0; }
+static inline void ^?{}($generator &) {}
+
+trait is_generator(dtype T) {
+      void main(T & this);
+      $generator * get_generator(T & this);
+};
+
+forall(dtype T | is_generator(T))
+static inline T & resume(T & gen) {
+        main(gen);
+        return gen;
+}
 
 // implicit increment, decrement if += defined, and implicit not if != defined

libcfa/src/concurrency/coroutine.cfa

@@ -208 +208 @@
 
                 if(cor->state == Primed) {
-                        suspend();
+                        __cfactx_suspend();
                 }
 

libcfa/src/concurrency/coroutine.hfa

@@ -46 +46 @@
 //-----------------------------------------------------------------------------
 // Public coroutine API
-static inline void suspend(void);
-
-forall(dtype T | is_coroutine(T))
-static inline T & resume(T & cor);
-
 forall(dtype T | is_coroutine(T))
 void prime(T & cor);
@@ -96 +91 @@
 
 // Suspend implementation inlined for performance
-static inline void suspend(void) {
-        // optimization : read TLS once and reuse it
-        // Safety note: this is preemption safe since if
-        // preemption occurs after this line, the pointer
-        // will also migrate which means this value will
-        // stay in syn with the TLS
-        $coroutine * src = TL_GET( this_thread )->curr_cor;
+extern "C" {
+        static inline void __cfactx_suspend(void) {
+                // optimization : read TLS once and reuse it
+                // Safety note: this is preemption safe since if
+                // preemption occurs after this line, the pointer
+                // will also migrate which means this value will
+                // stay in syn with the TLS
+                $coroutine * src = TL_GET( this_thread )->curr_cor;
 
-        assertf( src->last != 0,
-                "Attempt to suspend coroutine \"%.256s\" (%p) that has never been resumed.\n"
-                "Possible cause is a suspend executed in a member called by a coroutine user rather than by the coroutine main.",
-                src->name, src );
-        assertf( src->last->state != Halted,
-                "Attempt by coroutine \"%.256s\" (%p) to suspend back to terminated coroutine \"%.256s\" (%p).\n"
-                "Possible cause is terminated coroutine's main routine has already returned.",
-                src->name, src, src->last->name, src->last );
+                assertf( src->last != 0,
+                        "Attempt to suspend coroutine \"%.256s\" (%p) that has never been resumed.\n"
+                        "Possible cause is a suspend executed in a member called by a coroutine user rather than by the coroutine main.",
+                        src->name, src );
+                assertf( src->last->state != Halted,
+                        "Attempt by coroutine \"%.256s\" (%p) to suspend back to terminated coroutine \"%.256s\" (%p).\n"
+                        "Possible cause is terminated coroutine's main routine has already returned.",
+                        src->name, src, src->last->name, src->last );
 
-        $ctx_switch( src, src->last );
+                $ctx_switch( src, src->last );
+        }
 }
 

src/AST/Convert.cpp

@@ -493 +493 @@
         }
 
+        const ast::Stmt * visit(const ast::SuspendStmt * node ) override final {
+                if ( inCache( node ) ) return nullptr;
+                auto stmt = new SuspendStmt();
+                stmt->then   = get<CompoundStmt>().accept1( node->then   );
+                switch(node->type) {
+                        case ast::SuspendStmt::None     : stmt->type = SuspendStmt::None     ; break;
+                        case ast::SuspendStmt::Coroutine: stmt->type = SuspendStmt::Coroutine; break;
+                        case ast::SuspendStmt::Generator: stmt->type = SuspendStmt::Generator; break;
+                }
+                return stmtPostamble( stmt, node );
+        }
+
         const ast::Stmt * visit( const ast::WaitForStmt * node ) override final {
                 if ( inCache( node ) ) return nullptr;
@@ -1859 +1871 @@
         }
 
+        virtual void visit( const SuspendStmt * old ) override final {
+                if ( inCache( old ) ) return;
+                ast::SuspendStmt::Type type;
+                switch (old->type) {
+                        case SuspendStmt::Coroutine: type = ast::SuspendStmt::Coroutine; break;
+                        case SuspendStmt::Generator: type = ast::SuspendStmt::Generator; break;
+                        case SuspendStmt::None     : type = ast::SuspendStmt::None     ; break;
+                        default: abort();
+                }
+                this->node = new ast::SuspendStmt(
+                        old->location,
+                        GET_ACCEPT_1(then  , CompoundStmt),
+                        type,
+                        GET_LABELS_V(old->labels)
+                );
+                cache.emplace( old, this->node );
+        }
+
         virtual void visit( const WaitForStmt * old ) override final {
                 if ( inCache( old ) ) return;

src/AST/Decl.hpp

@@ -259 +259 @@
 
         bool is_coroutine() { return kind == Coroutine; }
-        bool is_monitor() { return kind == Monitor; }
-        bool is_thread() { return kind == Thread; }
+        bool is_generator() { return kind == Generator; }
+        bool is_monitor  () { return kind == Monitor  ; }
+        bool is_thread   () { return kind == Thread   ; }
 
         const Decl * accept( Visitor & v ) const override { return v.visit( this ); }

src/AST/Fwd.hpp

@@ -53 +53 @@
 class CatchStmt;
 class FinallyStmt;
+class SuspendStmt;
 class WaitForStmt;
 class WithStmt;

src/AST/Pass.hpp

@@ -111 +111 @@
         const ast::Stmt *             visit( const ast::CatchStmt            * ) override final;
         const ast::Stmt *             visit( const ast::FinallyStmt          * ) override final;
+        const ast::Stmt *             visit( const ast::SuspendStmt          * ) override final;
         const ast::Stmt *             visit( const ast::WaitForStmt          * ) override final;
         const ast::Decl *             visit( const ast::WithStmt             * ) override final;

src/AST/Pass.impl.hpp

@@ -823 +823 @@
 
 //--------------------------------------------------------------------------
+// FinallyStmt
+template< typename pass_t >
+const ast::Stmt * ast::Pass< pass_t >::visit( const ast::SuspendStmt * node ) {
+        VISIT_START( node );
+
+        VISIT(
+                maybe_accept( node, &SuspendStmt::then   );
+        )
+
+        VISIT_END( Stmt, node );
+}
+
+//--------------------------------------------------------------------------
 // WaitForStmt
 template< typename pass_t >

src/AST/Print.cpp

@@ -674 +674 @@
                 safe_print( node->body );
                 --indent;
+
+                return node;
+        }
+
+        virtual const ast::Stmt * visit( const ast::SuspendStmt * node ) override final {
+                os << "Suspend Statement";
+                switch (node->type) {
+                        case ast::SuspendStmt::None     : os << " with implicit target"; break;
+                        case ast::SuspendStmt::Generator: os << " for generator"; break;
+                        case ast::SuspendStmt::Coroutine: os << " for coroutine"; break;
+                }
+                os << endl;
+
+                ++indent;
+                if(node->then) {
+                        os << indent << " with post statement :" << endl;
+                        safe_print( node->then );
+                }
+                ++indent;
 
                 return node;

src/AST/Stmt.hpp

@@ -342 +342 @@
 };
 
+/// Suspend statement
+class SuspendStmt final : public Stmt {
+public:
+        ptr<CompoundStmt> then;
+        enum Type { None, Coroutine, Generator } type = None;
+
+        SuspendStmt( const CodeLocation & loc, const CompoundStmt * then, Type type, std::vector<Label> && labels = {} )
+        : Stmt(loc, std::move(labels)), then(then), type(type) {}
+
+        const Stmt * accept( Visitor & v ) const override { return v.visit( this ); }
+private:
+        SuspendStmt * clone() const override { return new SuspendStmt{ *this }; }
+        MUTATE_FRIEND
+};
+
 /// Wait for concurrency statement `when (...) waitfor (... , ...) ... timeout(...) ... else ...`
 class WaitForStmt final : public Stmt {

src/AST/Visitor.hpp

@@ -47 +47 @@
     virtual const ast::Stmt *             visit( const ast::CatchStmt            * ) = 0;
     virtual const ast::Stmt *             visit( const ast::FinallyStmt          * ) = 0;
+    virtual const ast::Stmt *             visit( const ast::SuspendStmt          * ) = 0;
     virtual const ast::Stmt *             visit( const ast::WaitForStmt          * ) = 0;
     virtual const ast::Decl *             visit( const ast::WithStmt             * ) = 0;

src/Common/PassVisitor.h

@@ -110 +110 @@
         virtual void visit( FinallyStmt * finallyStmt ) override final;
         virtual void visit( const FinallyStmt * finallyStmt ) override final;
+        virtual void visit( SuspendStmt * suspendStmt ) override final;
+        virtual void visit( const SuspendStmt * suspendStmt ) override final;
         virtual void visit( WaitForStmt * waitforStmt ) override final;
         virtual void visit( const WaitForStmt * waitforStmt ) override final;
@@ -276 +278 @@
         virtual Statement * mutate( CatchStmt * catchStmt ) override final;
         virtual Statement * mutate( FinallyStmt * finallyStmt ) override final;
+        virtual Statement * mutate( SuspendStmt * suspendStmt ) override final;
         virtual Statement * mutate( WaitForStmt * waitforStmt ) override final;
         virtual Declaration * mutate( WithStmt * withStmt ) override final;

src/Common/PassVisitor.impl.h

@@ -1522 +1522 @@
 
 //--------------------------------------------------------------------------
+// SuspendStmt
+template< typename pass_type >
+void PassVisitor< pass_type >::visit( SuspendStmt * node ) {
+        VISIT_START( node );
+
+        maybeAccept_impl( node->then  , *this );
+
+        VISIT_END( node );
+}
+
+template< typename pass_type >
+void PassVisitor< pass_type >::visit( const SuspendStmt * node ) {
+        VISIT_START( node );
+
+        maybeAccept_impl( node->then  , *this );
+
+        VISIT_END( node );
+}
+
+template< typename pass_type >
+Statement * PassVisitor< pass_type >::mutate( SuspendStmt * node ) {
+        MUTATE_START( node );
+
+        maybeMutate_impl( node->then  , *this );
+
+        MUTATE_END( Statement, node );
+}
+
+//--------------------------------------------------------------------------
 // WaitForStmt
 template< typename pass_type >

src/Concurrency/Keywords.cc

@@ -16 +16 @@
 #include "Concurrency/Keywords.h"
 
-#include <cassert>                 // for assert
-#include <string>                  // for string, operator==
-
-#include "Common/PassVisitor.h"    // for PassVisitor
-#include "Common/SemanticError.h"  // for SemanticError
-#include "Common/utility.h"        // for deleteAll, map_range
-#include "CodeGen/OperatorTable.h" // for isConstructor
-#include "InitTweak/InitTweak.h"   // for getPointerBase
-#include "SynTree/LinkageSpec.h"   // for Cforall
-#include "SynTree/Constant.h"      // for Constant
-#include "SynTree/Declaration.h"   // for StructDecl, FunctionDecl, ObjectDecl
-#include "SynTree/Expression.h"    // for VariableExpr, ConstantExpr, Untype...
-#include "SynTree/Initializer.h"   // for SingleInit, ListInit, Initializer ...
-#include "SynTree/Label.h"         // for Label
-#include "SynTree/Statement.h"     // for CompoundStmt, DeclStmt, ExprStmt
-#include "SynTree/Type.h"          // for StructInstType, Type, PointerType
-#include "SynTree/Visitor.h"       // for Visitor, acceptAll
+#include <cassert>                        // for assert
+#include <string>                         // for string, operator==
+
+#include "Common/PassVisitor.h"           // for PassVisitor
+#include "Common/SemanticError.h"         // for SemanticError
+#include "Common/utility.h"               // for deleteAll, map_range
+#include "CodeGen/OperatorTable.h"        // for isConstructor
+#include "ControlStruct/LabelGenerator.h" // for LebelGenerator
+#include "InitTweak/InitTweak.h"          // for getPointerBase
+#include "SynTree/LinkageSpec.h"          // for Cforall
+#include "SynTree/Constant.h"             // for Constant
+#include "SynTree/Declaration.h"          // for StructDecl, FunctionDecl, ObjectDecl
+#include "SynTree/Expression.h"           // for VariableExpr, ConstantExpr, Untype...
+#include "SynTree/Initializer.h"          // for SingleInit, ListInit, Initializer ...
+#include "SynTree/Label.h"                // for Label
+#include "SynTree/Statement.h"            // for CompoundStmt, DeclStmt, ExprStmt
+#include "SynTree/Type.h"                 // for StructInstType, Type, PointerType
+#include "SynTree/Visitor.h"              // for Visitor, acceptAll
 
 class Attribute;
@@ -147 +148 @@
         };
 
+
+
         //-----------------------------------------------------------------------------
         //Handles monitor type declarations :
@@ -180 +183 @@
 
         //-----------------------------------------------------------------------------
+        //Handles generator type declarations :
+        // generator MyGenerator {                   struct MyGenerator {
+        //      int data;                                  int data;
+        //      a_struct_t more_data;                      a_struct_t more_data;
+        //                                =>             int __gen_next;
+        // };                                        };
+        //
+        class GeneratorKeyword final : public ConcurrentSueKeyword {
+          public:
+
+                GeneratorKeyword() : ConcurrentSueKeyword(
+                        "$generator",
+                        "__generator_state",
+                        "get_generator",
+                        "Unable to find builtin type $generator\n",
+                        true,
+                        AggregateDecl::Generator
+                )
+                {}
+
+                virtual ~GeneratorKeyword() {}
+
+                virtual bool is_target( StructDecl * decl ) override final { return decl->is_generator(); }
+
+                static void implement( std::list< Declaration * > & translationUnit ) {
+                        PassVisitor< GeneratorKeyword > impl;
+                        mutateAll( translationUnit, impl );
+                }
+        };
+
+
+        //-----------------------------------------------------------------------------
+        class SuspendKeyword final : public WithStmtsToAdd, public WithGuards {
+        public:
+                SuspendKeyword() = default;
+                virtual ~SuspendKeyword() = default;
+
+                void  premutate( FunctionDecl * );
+                DeclarationWithType * postmutate( FunctionDecl * );
+
+                Statement * postmutate( SuspendStmt * );
+
+                static void implement( std::list< Declaration * > & translationUnit ) {
+                        PassVisitor< SuspendKeyword > impl;
+                        mutateAll( translationUnit, impl );
+                }
+
+        private:
+                DeclarationWithType * is_main( FunctionDecl * );
+                bool is_real_suspend( FunctionDecl * );
+
+                Statement * make_generator_suspend( SuspendStmt * );
+                Statement * make_coroutine_suspend( SuspendStmt * );
+
+                struct LabelPair {
+                        Label obj;
+                        int   idx;
+                };
+
+                LabelPair make_label() {
+                        labels.push_back( gen.newLabel("generator") );
+                        return { labels.back(), int(labels.size()) };
+                }
+
+                DeclarationWithType * in_generator = nullptr;
+                FunctionDecl * decl_suspend = nullptr;
+                std::vector<Label> labels;
+                ControlStruct::LabelGenerator & gen = *ControlStruct::LabelGenerator::getGenerator();
+        };
+
+        //-----------------------------------------------------------------------------
         //Handles mutex routines definitions :
         // void foo( A * mutex a, B * mutex b,  int i ) {                  void foo( A * a, B * b,  int i ) {
@@ -251 +325 @@
                 CoroutineKeyword        ::implement( translationUnit );
                 MonitorKeyword  ::implement( translationUnit );
+                GeneratorKeyword  ::implement( translationUnit );
+                SuspendKeyword    ::implement( translationUnit );
         }
 
@@ -446 +522 @@
 
                 declsToAddAfter.push_back( get_decl );
-
-                // get_decl->fixUniqueId();
-        }
+        }
+
+        //=============================================================================================
+        // Suspend keyword implementation
+        //=============================================================================================
+        DeclarationWithType * SuspendKeyword::is_main( FunctionDecl * func) {
+                if(func->name != "main") return nullptr;
+                if(func->type->parameters.size() != 1) return nullptr;
+
+                auto param = func->type->parameters.front();
+
+                auto type  = dynamic_cast<ReferenceType * >(param->get_type());
+                if(!type) return nullptr;
+
+                auto obj   = dynamic_cast<StructInstType *>(type->base);
+                if(!obj) return nullptr;
+
+                if(!obj->baseStruct->is_generator()) return nullptr;
+
+                return param;
+        }
+
+        bool SuspendKeyword::is_real_suspend( FunctionDecl * func ) {
+                if(isMangled(func->linkage)) return false; // the real suspend isn't mangled
+                if(func->name != "__cfactx_suspend") return false; // the real suspend has a specific name
+                if(func->type->parameters.size() != 0) return false; // Too many parameters
+                if(func->type->returnVals.size() != 0) return false; // Too many return values
+
+                return true;
+        }
+
+        void SuspendKeyword::premutate( FunctionDecl * func ) {
+                GuardValue(in_generator);
+                in_generator = nullptr;
+
+                // Is this the real suspend?
+                if(is_real_suspend(func)) {
+                        decl_suspend = decl_suspend ? decl_suspend : func;
+                        return;
+                }
+
+                // Is this the main of a generator?
+                auto param = is_main( func );
+                if(!param) return;
+
+                if(func->type->returnVals.size() != 0) SemanticError(func->location, "Generator main must return void");
+
+                in_generator = param;
+                GuardValue(labels);
+                labels.clear();
+        }
+
+        DeclarationWithType * SuspendKeyword::postmutate( FunctionDecl * func ) {
+                if( !func->statements ) return func; // Not the actual definition, don't do anything
+                if( !in_generator     ) return func; // Not in a generator, don't do anything
+                if( labels.empty()    ) return func; // Generator has no states, nothing to do, could throw a warning
+
+                // This is a generator main, we need to add the following code to the top
+                // static void * __generator_labels[] = {&&s0, &&s1, ...};
+                // goto * __generator_labels[gen.__generator_state];
+                const auto & loc = func->location;
+
+                const auto first_label = gen.newLabel("generator");
+
+                // for each label add to declaration
+                std::list<Initializer*> inits = { new SingleInit( new LabelAddressExpr( first_label ) ) };
+                for(const auto & label : labels) {
+                        inits.push_back(
+                                new SingleInit(
+                                        new LabelAddressExpr( label )
+                                )
+                        );
+                }
+                auto init = new ListInit(std::move(inits), noDesignators, true);
+                labels.clear();
+
+                // create decl
+                auto decl = new ObjectDecl(
+                        "__generator_labels",
+                        Type::StorageClasses( Type::Static ),
+                        LinkageSpec::AutoGen,
+                        nullptr,
+                        new ArrayType(
+                                Type::Qualifiers(),
+                                new PointerType(
+                                        Type::Qualifiers(),
+                                        new VoidType( Type::Qualifiers() )
+                                ),
+                                nullptr,
+                                false, false
+                        ),
+                        init
+                );
+
+                // create the goto
+                assert(in_generator);
+
+                auto go_decl = new ObjectDecl(
+                        "__generator_label",
+                        noStorageClasses,
+                        LinkageSpec::AutoGen,
+                        nullptr,
+                        new PointerType(
+                                Type::Qualifiers(),
+                                new VoidType( Type::Qualifiers() )
+                        ),
+                        new SingleInit(
+                                new UntypedExpr(
+                                        new NameExpr("?[?]"),
+                                        {
+                                                new NameExpr("__generator_labels"),
+                                                new UntypedMemberExpr(
+                                                        new NameExpr("__generator_state"),
+                                                        new VariableExpr( in_generator )
+                                                )
+                                        }
+                                )
+                        )
+                );
+                go_decl->location = loc;
+
+                auto go = new BranchStmt(
+                        new VariableExpr( go_decl ),
+                        BranchStmt::Goto
+                );
+                go->location = loc;
+                go->computedTarget->location = loc;
+
+                auto noop = new NullStmt({ first_label });
+                noop->location = loc;
+
+                // wrap everything in a nice compound
+                auto body = new CompoundStmt({
+                        new DeclStmt( decl ),
+                        new DeclStmt( go_decl ),
+                        go,
+                        noop,
+                        func->statements
+                });
+                body->location   = loc;
+                func->statements = body;
+
+                return func;
+        }
+
+        Statement * SuspendKeyword::postmutate( SuspendStmt * stmt ) {
+                SuspendStmt::Type type = stmt->type;
+                if(type == SuspendStmt::None) {
+                        // This suspend has a implicit target, find it
+                        type = in_generator ? SuspendStmt::Generator : SuspendStmt::Coroutine;
+                }
+
+                // Check that the target makes sense
+                if(!in_generator && type == SuspendStmt::Generator) SemanticError( stmt->location, "'suspend generator' must be used inside main of generator type.");
+
+                // Act appropriately
+                switch(type) {
+                        case SuspendStmt::Generator: return make_generator_suspend(stmt);
+                        case SuspendStmt::Coroutine: return make_coroutine_suspend(stmt);
+                        default: abort();
+                }
+        }
+
+        Statement * SuspendKeyword::make_generator_suspend( SuspendStmt * stmt ) {
+                assert(in_generator);
+                // Target code is :
+                //   gen.__generator_state = X;
+                //   { THEN }
+                //   return;
+                //   __gen_X:;
+
+                // Save the location and delete the old statement, we only need the location from this point on
+                auto loc = stmt->location;
+
+                // Build the label and get its index
+                auto label = make_label();
+
+                // Create the context saving statement
+                auto save = new ExprStmt( new UntypedExpr(
+                        new NameExpr( "?=?" ),
+                        {
+                                new UntypedMemberExpr(
+                                        new NameExpr("__generator_state"),
+                                        new VariableExpr( in_generator )
+                                ),
+                                new ConstantExpr(
+                                        Constant::from_int( label.idx )
+                                )
+                        }
+                ));
+                assert(save->expr);
+                save->location = loc;
+                stmtsToAddBefore.push_back( save );
+
+                // if we have a then add it here
+                auto then = stmt->then;
+                stmt->then = nullptr;
+                delete stmt;
+                if(then) stmtsToAddBefore.push_back( then );
+
+                // Create the return statement
+                auto ret = new ReturnStmt( nullptr );
+                ret->location = loc;
+                stmtsToAddBefore.push_back( ret );
+
+                // Create the null statement with the created label
+                auto noop = new NullStmt({ label.obj });
+                noop->location = loc;
+
+                // Return the null statement to take the place of the previous statement
+                return noop;
+        }
+
+        Statement * SuspendKeyword::make_coroutine_suspend( SuspendStmt * stmt ) {
+                if(stmt->then) SemanticError( stmt->location, "Compound statement following coroutines is not implemented.");
+
+                // Save the location and delete the old statement, we only need the location from this point on
+                auto loc = stmt->location;
+                delete stmt;
+
+                // Create the call expression
+                if(!decl_suspend) SemanticError( loc, "suspend keyword applied to coroutines requires coroutines to be in scope, add #include <coroutine.hfa>\n");
+                auto expr = new UntypedExpr( VariableExpr::functionPointer( decl_suspend ) );
+                expr->location = loc;
+
+                // Change this statement into a regular expr
+                assert(expr);
+                auto nstmt = new ExprStmt( expr );
+                nstmt->location = loc;
+                return nstmt;
+        }
+
 
         //=============================================================================================

src/Parser/ParseNode.h

@@ -428 +428 @@
 Statement * build_asm( bool voltile, Expression * instruction, ExpressionNode * output = nullptr, ExpressionNode * input = nullptr, ExpressionNode * clobber = nullptr, LabelNode * gotolabels = nullptr );
 Statement * build_directive( std::string * directive );
+SuspendStmt * build_suspend( StatementNode *, SuspendStmt::Type = SuspendStmt::None);
 WaitForStmt * build_waitfor( ExpressionNode * target, StatementNode * stmt, ExpressionNode * when );
 WaitForStmt * build_waitfor( ExpressionNode * target, StatementNode * stmt, ExpressionNode * when, WaitForStmt * existing );

src/Parser/StatementNode.cc

@@ -249 +249 @@
 } // build_finally
 
+SuspendStmt * build_suspend( StatementNode * then, SuspendStmt::Type type ) {
+        auto node = new SuspendStmt();
+
+        node->type = type;
+
+        std::list< Statement * > stmts;
+        buildMoveList< Statement, StatementNode >( then, stmts );
+        if(!stmts.empty()) {
+                assert( stmts.size() == 1 );
+                node->then = dynamic_cast< CompoundStmt * >( stmts.front() );
+        }
+
+        return node;
+}
+
 WaitForStmt * build_waitfor( ExpressionNode * targetExpr, StatementNode * stmt, ExpressionNode * when ) {
         auto node = new WaitForStmt();

src/Parser/TypeData.cc

@@ -769 +769 @@
           case AggregateDecl::Struct:
           case AggregateDecl::Coroutine:
+          case AggregateDecl::Generator:
           case AggregateDecl::Monitor:
           case AggregateDecl::Thread:

src/Parser/lex.ll

@@ -65 +65 @@
 #define FLOATXX(v) KEYWORD_RETURN(v);
 #else
-#define FLOATXX(v) IDENTIFIER_RETURN(); 
+#define FLOATXX(v) IDENTIFIER_RETURN();
 #endif // HAVE_KEYWORDS_FLOATXX
 
@@ -301 +301 @@
 _Static_assert  { KEYWORD_RETURN(STATICASSERT); }               // C11
 struct                  { KEYWORD_RETURN(STRUCT); }
-        /* suspend                      { KEYWORD_RETURN(SUSPEND); }                    // CFA */
+suspend                 { KEYWORD_RETURN(SUSPEND); }                    // CFA
 switch                  { KEYWORD_RETURN(SWITCH); }
 thread                  { KEYWORD_RETURN(THREAD); }                             // C11

src/Parser/parser.yy

@@ -278 +278 @@
 %token OTYPE FTYPE DTYPE TTYPE TRAIT                                    // CFA
 %token SIZEOF OFFSETOF
-// %token SUSPEND RESUME                                                                        // CFA
+// %token RESUME                                                                        // CFA
+%token SUSPEND                                                                  // CFA
 %token ATTRIBUTE EXTENSION                                                              // GCC
 %token IF ELSE SWITCH CASE DEFAULT DO WHILE FOR BREAK CONTINUE GOTO RETURN
@@ -1259 +1260 @@
         | RETURN '{' initializer_list_opt comma_opt '}' ';'
                 { SemanticError( yylloc, "Initializer return is currently unimplemented." ); $$ = nullptr; }
-        // | SUSPEND ';'
-        //      { SemanticError( yylloc, "Suspend expression is currently unimplemented." ); $$ = nullptr; }
-        // | SUSPEND compound_statement ';'
-        //      { SemanticError( yylloc, "Suspend expression is currently unimplemented." ); $$ = nullptr; }
+        | SUSPEND ';'
+                { $$ = new StatementNode( build_suspend( nullptr ) ); }
+        | SUSPEND compound_statement
+                { $$ = new StatementNode( build_suspend( $2 ) ); }
+        | SUSPEND COROUTINE ';'
+                { $$ = new StatementNode( build_suspend( nullptr, SuspendStmt::Coroutine ) ); }
+        | SUSPEND COROUTINE compound_statement
+                { $$ = new StatementNode( build_suspend( $3, SuspendStmt::Coroutine ) ); }
+        | SUSPEND GENERATOR ';'
+                { $$ = new StatementNode( build_suspend( nullptr, SuspendStmt::Generator ) ); }
+        | SUSPEND GENERATOR compound_statement
+                { $$ = new StatementNode( build_suspend( $3, SuspendStmt::Generator ) ); }
         | THROW assignment_expression_opt ';'                           // handles rethrow
                 { $$ = new StatementNode( build_throw( $2 ) ); }
@@ -2077 +2086 @@
 aggregate_control:                                                                              // CFA
         GENERATOR
-                { SemanticError( yylloc, "generator is currently unimplemented." ); $$ = AggregateDecl::NoAggregate; }
+                { yyy = true; $$ = AggregateDecl::Generator; }
         | MONITOR GENERATOR
                 { SemanticError( yylloc, "monitor generator is currently unimplemented." ); $$ = AggregateDecl::NoAggregate; }

src/SynTree/Declaration.h

@@ -302 +302 @@
 
         bool is_coroutine() { return kind == Coroutine; }
-        bool is_monitor() { return kind == Monitor; }
-        bool is_thread() { return kind == Thread; }
+        bool is_generator() { return kind == Generator; }
+        bool is_monitor  () { return kind == Monitor  ; }
+        bool is_thread   () { return kind == Thread   ; }
 
         virtual StructDecl * clone() const override { return new StructDecl( *this ); }

src/SynTree/Mutator.h

@@ -51 +51 @@
         virtual Statement * mutate( CatchStmt * catchStmt ) = 0;
         virtual Statement * mutate( FinallyStmt * catchStmt ) = 0;
+        virtual Statement * mutate( SuspendStmt * suspendStmt ) = 0;
         virtual Statement * mutate( WaitForStmt * waitforStmt ) = 0;
         virtual Declaration * mutate( WithStmt * withStmt ) = 0;

src/SynTree/Statement.cc

@@ -420 +420 @@
 }
 
+SuspendStmt::SuspendStmt( const SuspendStmt & other )
+        : Statement( other )
+        , then( maybeClone(other.then) )
+{}
+
+SuspendStmt::~SuspendStmt() {
+        delete then;
+}
+
+void SuspendStmt::print( std::ostream & os, Indenter indent ) const {
+        os << "Suspend Statement";
+        switch (type) {
+                case None     : os << " with implicit target"; break;
+                case Generator: os << " for generator"       ; break;
+                case Coroutine: os << " for coroutine"       ; break;
+        }
+        os << endl;
+        indent += 1;
+
+        if(then) {
+                os << indent << " with post statement :" << endl;
+                then->print( os, indent + 1);
+        }
+}
+
 WaitForStmt::WaitForStmt() : Statement() {
         timeout.time      = nullptr;

src/SynTree/Statement.h

@@ -422 +422 @@
 };
 
+class SuspendStmt : public Statement {
+  public:
+        CompoundStmt * then = nullptr;
+        enum Type { None, Coroutine, Generator } type = None;
+
+        SuspendStmt() = default;
+        SuspendStmt( const SuspendStmt & );
+        virtual ~SuspendStmt();
+
+        virtual SuspendStmt * clone() const override { return new SuspendStmt( *this ); }
+        virtual void accept( Visitor & v ) override { v.visit( this ); }
+        virtual void accept( Visitor & v ) const override { v.visit( this ); }
+        virtual Statement * acceptMutator( Mutator & m )  override { return m.mutate( this ); }
+        virtual void print( std::ostream & os, Indenter indent = {} ) const override;
+};
+
 class WaitForStmt : public Statement {
   public:

src/SynTree/SynTree.h

@@ -54 +54 @@
 class CatchStmt;
 class FinallyStmt;
+class SuspendStmt;
 class WaitForStmt;
 class WithStmt;

src/SynTree/Visitor.h

@@ -78 +78 @@
         virtual void visit( FinallyStmt * node ) { visit( const_cast<const FinallyStmt *>(node) ); }
         virtual void visit( const FinallyStmt * finallyStmt ) = 0;
+        virtual void visit( SuspendStmt * node ) { visit( const_cast<const SuspendStmt *>(node) ); }
+        virtual void visit( const SuspendStmt * suspendStmt ) = 0;
         virtual void visit( WaitForStmt * node ) { visit( const_cast<const WaitForStmt *>(node) ); }
         virtual void visit( const WaitForStmt * waitforStmt ) = 0;

tests/concurrent/coroutineYield.cfa

@@ -33 +33 @@
                         sout | "Coroutine 2";
                 #endif
-                suspend();
+                suspend;
         }
 }

tests/concurrent/suspend_then.cfa

@@ -1 +1 @@
 #include <fstream.hfa>
 #include <kernel.hfa>
-#include <monitor.hfa>
 #include <thread.hfa>
 #include <time.hfa>
@@ -10 +9 @@
 #include "long_tests.hfa"
 
-#ifndef PREEMPTION_RATE
-#define PREEMPTION_RATE 10`ms
-#endif
-
 Duration default_preemption() {
-        return PREEMPTION_RATE;
+        return 0;
 }
 
@@ -26 +21 @@
 #if !defined(TEST_FOREVER)
         static inline void print(const char * const text ) {
-                write( STDERR_FILENO, text, strlen(text) );
+                write( STDOUT_FILENO, text, strlen(text) );
         }
 #else
@@ -32 +27 @@
 #endif
 
-coroutine Coroutine {};
+generator Coroutine { int i; };
 
 volatile bool done = false;
@@ -49 +44 @@
 
 void main(Coroutine& this) {
-        suspend();
-        for(int i = 0; TEST(i < N); i++) {
+        this.i = 0;
+        suspend;
+        for(;TEST(this.i < N); this.i++) {
 
-                print("C - Suspending");
-                void publish() {
-                        print("C - Publishing");
+                print("C - Suspending\n");
+                suspend{
+                        print("C - Publishing\n");
                         assert(!the_cor);
                         store( this );
                 }
-                suspend_then(publish);
                 assert(!the_cor);
-                print("Coroutine 2");
+                print("C - Back\n");
                 KICK_WATCHDOG;
                 yield();
         }
         done = true;
-        suspend();
+        suspend;
 }
 
@@ -77 +72 @@
                 if(!mine) continue;
 
-                print("T - took");
+                print("T - took\n");
                 resume(*mine);
-                print("T - back");
         }
 }

tests/coroutine/.expect/fmtLines.txt

@@ -48 +48 @@
 {                                                         // f  or n  ewli  
 ne c  hara  cter  s                                     su  
-spen  d();                                      if   ( fm  
-t.ch   !=   '\n'   ) b  reak  
-;               /  / ig  nore   new  line  
-                                  } //   for                              sout  
- | f  mt.c  h;                                                  //   
-prin  t ch  arac  ter                   }   
-// f  or                        sou  t |   "  "  
-;                                                               //   prin  t bl  
-ock   sepa  rato  r             }   //   
-for             sou  t |   nl;                                    
-                                  // p  rint   gro  up s  
-epar  ator      } /  / fo  r} /  
-/ ma  invo  id p  rt(   Form  
-at &   fmt  , ch  ar c  h )   
-{      fmt  .ch   = ch  ;     
- res  ume(   fmt   );}   //   
-prti  nt m  ain(  ) {     Form  
-at f  mt;         char   ch;    for  
- ( ;  ; )   {           s  in |   ch;  
-                                                                                //   rea  d on  
-e ch  arac  ter     if   ( e  
-of(   sin   ) )   brea  k;                
-                                        //   eof   ?            p  rt(   
-fmt,   ch   );  }   //   for}  
- //   main  // L  ocal   Var  
-iabl  es:   ////   tab  -wid  
-th:   4 //  // c  ompi  le-c  
-omma  nd:   "cfa   fmt  Line  
-s.cf  a" /  ///   End:   //
+spen  d;                                        i  f (   fmt.  
+ch !  = '\  n' )   bre  ak;       
+        //   igno  re n  ewli  ne                 
+                }   // f  or                            so  ut |  
+ fmt  .ch;                                                      /  / pr  
+int   char  acte  r                       } //  
+ for                    s  out   | "    ";        
+                                                        /  / pr  int   bloc  
+k se  para  tor         } /  / fo  
+r               s  out   | nl  ;                                                          
+                //   pri  nt g  roup   sep  
+arat  or        }   //   for}   //   
+main  void   prt  ( Fo  rmat  
+ & f  mt,   char   ch   ) {   
+   f  mt.c  h =   ch;      r  
+esum  e( f  mt )  ;} /  / pr  
+tint   mai  n()   {     Fo  rmat  
+ fmt  ; ch  ar c  h;    f  or (  
+ ;;   ) {               sin   | c  h;             
+                                                                  // r  ead   one   
+char  acte  r       if (   eof  
+( si  n )   ) br  eak;                                    
+                        /  / eo  f ?            prt  ( fm  
+t, c  h );      } /  / fo  r} /  
+/ ma  in//   Loc  al V  aria  
+bles  : //  // t  ab-w  idth  
+: 4   ////   com  pile  -com  
+mand  : "c  fa f  mtLi  nes.  
+cfa"   ///  / En  d: /  /

tests/coroutine/.in/fmtLines.txt

@@ -35 +35 @@
                         for ( fmt.b = 0; fmt.b < 4; fmt.b += 1 ) {      // blocks of 4 characters
                                 for ( ;; ) {                                                    // for newline characters
-                                        suspend();
+                                        suspend;
                                         if ( fmt.ch != '\n' ) break;            // ignore newline
                                 } // for

tests/coroutine/cntparens.cfa

@@ -1 @@
-// 
+//
 // Cforall Version 1.0.0 Copyright (C) 2017 University of Waterloo
 //
 // The contents of this file are covered under the licence agreement in the
 // file "LICENCE" distributed with Cforall.
-// 
+//
 // cntparens.cfa -- match left/right parenthesis
-// 
+//
 // Author           : Peter A. Buhr
 // Created On       : Sat Apr 20 11:04:45 2019
@@ -12 +12 @@
 // Last Modified On : Sat Apr 20 11:06:21 2019
 // Update Count     : 1
-// 
+//
 
 #include <fstream.hfa>
@@ -26 +26 @@
 void main( CntParens & cpns ) with( cpns ) {
         for ( ; ch == '('; cnt += 1 ) {                                         // left parenthesis
-                suspend();
+                suspend;
         }
         for ( ; ch == ')' && cnt > 1; cnt -= 1 ) {                      // right parenthesis
-                suspend();
+                suspend;
         }
         status = ch == ')' ? Match : Error;
 } // main
-        
+
 void ?{}( CntParens & cpns ) with( cpns ) { status = Cont; cnt = 0; }
 

tests/coroutine/devicedriver.cfa

@@ -1 @@
-// 
+//
 // Cforall Version 1.0.0 Copyright (C) 2017 University of Waterloo
 //
 // The contents of this file are covered under the licence agreement in the
 // file "LICENCE" distributed with Cforall.
-// 
-// devicedriver.cfa -- 
-// 
+//
+// devicedriver.cfa --
+//
 // Author           : Peter A. Buhr
 // Created On       : Sat Mar 16 15:30:34 2019
@@ -12 +12 @@
 // Last Modified On : Sat Apr 20 09:07:19 2019
 // Update Count     : 90
-// 
+//
 
 #include <fstream.hfa>
@@ -29 +29 @@
 
 void checkCRC( Driver & d, unsigned int sum ) with( d ) {
-        suspend();
+        suspend;
         unsigned short int crc = byte << 8;                                     // sign extension over written
-        suspend();
+        suspend;
         // prevent sign extension for signed char
         status = (crc | (unsigned char)byte) == sum ? MSG : ECRC;
@@ -41 +41 @@
                 status = CONT;
                 unsigned int lnth = 0, sum = 0;
-                while ( byte != STX ) suspend();
+                while ( byte != STX ) suspend;
           emsg: for () {
-                        suspend();
+                        suspend;
                         choose ( byte ) {                                                       // process byte
                           case STX:
-                                status = ESTX; suspend(); continue msg;
+                                status = ESTX; suspend; continue msg;
                           case ETX:
                                 break emsg;
                           case ESC:
-                                suspend();
+                                suspend;
                         } // choose
                         if ( lnth >= MaxMsg ) {                                         // buffer full ?
-                                status = ELNTH; suspend(); continue msg;
+                                status = ELNTH; suspend; continue msg;
                         } // if
                         msg[lnth++] = byte;
@@ -60 +60 @@
                 msg[lnth] = '\0';                                                               // terminate string
                 checkCRC( d, sum );                                                             // refactor CRC check
-                suspend();
+                suspend;
         } // for
 } // main

tests/coroutine/fibonacci.cfa

@@ -22 +22 @@
         int fn1, fn2;                                                                           // retained between resumes
         fn = 0;  fn1 = fn;                                                                      // 1st case
-        suspend();                                                                                      // restart last resume
+        suspend;                                                                                        // restart last resume
         fn = 1;  fn2 = fn1;  fn1 = fn;                                          // 2nd case
-        suspend();                                                                                      // restart last resume
+        suspend;                                                                                        // restart last resume
         for () {
                 fn = fn1 + fn2;  fn2 = fn1;  fn1 = fn;                  // general case
-                suspend();                                                                              // restart last resume
+                suspend;                                                                                // restart last resume
         } // for
 }

tests/coroutine/fibonacci_1.cfa

@@ -12 +12 @@
 // Last Modified On : Thu Mar 21 08:10:45 2019
 // Update Count     : 25
-// 
+//
 
 #include <fstream.hfa>
@@ -23 +23 @@
         [fn1, fn] = [0, 1];                                                                     // precompute first two states
         for () {
-                suspend();                                                                              // restart last resume
+                suspend;                                                                                // restart last resume
                 [fn1, fn] = [fn, fn1 + fn];                                             // general case
         } // for

tests/coroutine/fmtLines.cfa

@@ -27 +27 @@
                         for ( b = 0; b < 4; b += 1 ) {                          // blocks of 4 characters
                                 for () {                                                                // for newline characters
-                                        suspend();
+                                        suspend;
                                   if ( ch != '\n' ) break;                              // ignore newline
                                 } // for

tests/coroutine/raii.cfa

@@ -39 +39 @@
         Raii raii = { "Coroutine" };
         sout | "Before Suspend";
-        suspend();
+        suspend;
         sout | "After Suspend";
 }

tests/coroutine/runningTotal.cfa

@@ -25 +25 @@
 void update( RunTotal & rntl, int input ) with( rntl ) { // helper
         total += input;                                                                         // remember between activations
-        suspend();                                                                                      // inactivate on stack
+        suspend;                                                                                        // inactivate on stack
 }
 

tests/coroutine/suspend_then.cfa

@@ -15 +15 @@
 
 #include <fstream.hfa>
-#include <coroutine.hfa>
 
-void then() {
-        sout | "Then!";
-}
-
-coroutine Fibonacci { int fn; };                                                // used for communication
+generator Fibonacci {
+        int fn;                                                                         // used for communication
+        int fn1, fn2;                                                           // retained between resumes
+};
 
 void main( Fibonacci & fib ) with( fib ) {                              // called on first resume
-        int fn1, fn2;                                                           // retained between resumes
         fn = 0;  fn1 = fn;                                                      // 1st case
-        suspend_then(then);                                                     // restart last resume
+        suspend { sout | "Then!"; }                                             // restart last resume
         fn = 1;  fn2 = fn1;  fn1 = fn;                                  // 2nd case
-        suspend_then(then);                                                     // restart last resume
+        suspend { sout | "Then!"; }                                             // restart last resume
         for () {
                 fn = fn1 + fn2;  fn2 = fn1;  fn1 = fn;                  // general case
-                suspend_then(then);                                             // restart last resume
+                suspend { sout | "Then!"; }                                     // restart last resume
         } // for
 }

tests/errors/.expect/completeType.txt

@@ -27 +27 @@
     void 
   )
-  Environment:( _83_4_DT ) -> instance of struct A with body 0 (no widening)
+  Environment:( _85_4_DT ) -> instance of struct A with body 0 (no widening)
 
 
@@ -50 +50 @@
     void 
   )
-  Environment:( _83_4_DT ) -> instance of struct B with body 1 (no widening)
+  Environment:( _85_4_DT ) -> instance of struct B with body 1 (no widening)
 
 
@@ -127 +127 @@
           void 
         )
-        Environment:( _102_0_T ) -> instance of type T (not function type) (no widening)
+        Environment:( _104_0_T ) -> instance of type T (not function type) (no widening)
 
       Could not satisfy assertion:
 ?=?: pointer to function
         ... with parameters
-          reference to instance of type _102_0_T (not function type)
-          instance of type _102_0_T (not function type)
+          reference to instance of type _104_0_T (not function type)
+          instance of type _104_0_T (not function type)
         ... returning
-          _retval__operator_assign: instance of type _102_0_T (not function type)
+          _retval__operator_assign: instance of type _104_0_T (not function type)
           ... with attributes:
             Attribute with name: unused