Changeset 20632a2

Timestamp:

Nov 15, 2017, 2:45:45 PM (6 years ago)

Author:

Rob Schluntz <rschlunt@…>

Branches:

ADT, aaron-thesis, arm-eh, ast-experimental, cleanup-dtors, deferred_resn, demangler, enum, forall-pointer-decay, jacob/cs343-translation, jenkins-sandbox, master, new-ast, new-ast-unique-expr, new-env, no_list, persistent-indexer, pthread-emulation, qualifiedEnum, resolv-new, with_gc

Children:

0873968c

Parents:

490db327 (diff), b3ffb61 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge branch 'master' of plg.uwaterloo.ca:/u/cforall/software/cfa/cfa-cc

Location:

doc/proposals/concurrency

Files:

• Makefile (modified) (3 diffs)
• annex/local.bib (modified) (1 diff)
• figures/int_monitor.fig (modified) (3 diffs)
• figures/int_monitor.fig.bak (added)
• figures/monitor_structs.fig (added)
• text/basics.tex (modified) (2 diffs)
• text/concurrency.tex (modified) (6 diffs)
• text/future.tex (modified) (2 diffs)
• text/internals.tex (modified) (14 diffs)
• text/parallelism.tex (modified) (2 diffs)
• text/results.tex (modified) (6 diffs)
• text/together.tex (modified) (2 diffs)
• version (modified) (1 diff)

doc/proposals/concurrency/Makefile

@@ -32 +32 @@
 PICTURES = ${addprefix build/, ${addsuffix .pstex, \
         system \
+        monitor_structs \
 }}
 
@@ -83 +84 @@
         dvips $< -o $@
 
-build/${basename ${DOCUMENT}}.dvi : Makefile ${GRAPHS} ${PROGRAMS} ${PICTURES} ${FIGURES} ${SOURCES} ${basename ${DOCUMENT}}.tex ../../LaTeXmacros/common.tex ../../LaTeXmacros/indexstyle
+build/${basename ${DOCUMENT}}.dvi : Makefile ${GRAPHS} ${PROGRAMS} ${PICTURES} ${FIGURES} ${SOURCES} ${basename ${DOCUMENT}}.tex ../../LaTeXmacros/common.tex ../../LaTeXmacros/indexstyle annex/local.bib
 
         @ if [ ! -r ${basename $@}.ind ] ; then touch ${basename $@}.ind ; fi                           # Conditionally create an empty *.ind (index) file for inclusion until makeindex is run.
@@ -94 +95 @@
         @ -${BibTeX} ${basename $@}
         @ echo "Glossary"
-        makeglossaries -q -s ${basename $@}.ist ${basename $@}                                          # Make index from *.aux entries and input index at end of document
+        @ makeglossaries -q -s ${basename $@}.ist ${basename $@}                                                # Make index from *.aux entries and input index at end of document
         @ echo ".dvi generation"
         @ -build/bump_ver.sh

doc/proposals/concurrency/annex/local.bib

@@ -52 +52 @@
         year            = 2017
 }
+
+@manual{Cpp-Transactions,
+        keywords        = {C++, Transactional Memory},
+        title           = {Technical Specification for C++ Extensions for Transactional Memory},
+        organization= {International Standard ISO/IEC TS 19841:2015 },
+        publisher   = {American National Standards Institute},
+        address = {http://www.iso.org},
+        year            = 2015,
+}
+
+@article{BankTransfer,
+        keywords        = {Bank Transfer},
+        title   = {Bank Account Transfer Problem},
+        publisher       = {Wiki Wiki Web},
+        address = {http://wiki.c2.com},
+        year            = 2010
+}
+
+@misc{2FTwoHardThings,
+        keywords        = {Hard Problem},
+        title   = {TwoHardThings},
+        author  = {Martin Fowler},
+        address = {https://martinfowler.com/bliki/TwoHardThings.html},
+        year            = 2009
+}
+
+@article{IntrusiveData,
+        title           = {Intrusive Data Structures},
+        author  = {Jiri Soukup},
+        journal = {CppReport},
+        year            = 1998,
+        month           = May,
+        volume  = {10/No5.},
+        page            = 22
+}
+
+@misc{affinityLinux,
+        title           = "{Linux man page - sched\_setaffinity(2)}"
+}
+
+@misc{affinityWindows,
+        title           = "{Windows (vs.85) - SetThreadAffinityMask function}"
+}
+
+@misc{affinityFreebsd,
+        title           = "{FreeBSD General Commands Manual - CPUSET(1)}"
+}
+
+@misc{affinityNetbsd,
+        title           = "{NetBSD Library Functions Manual - AFFINITY(3)}"
+}
+
+@misc{affinityMacosx,
+        title           = "{Affinity API Release Notes for OS X v10.5}"
+}

doc/proposals/concurrency/figures/int_monitor.fig

@@ -8 +8 @@
 -2
 1200 2
-5 1 0 1 0 7 50 -1 -1 0.000 0 1 0 0 600.000 2625.000 600 2325 300 2625 600 2925
-6 3225 4500 7425 4800
-1 3 0 1 -1 -1 0 0 20 0.000 1 0.0000 3375 4650 80 80 3375 4650 3455 4730
-1 3 0 1 -1 -1 0 0 -1 0.000 1 0.0000 4725 4650 105 105 4725 4650 4830 4755
-1 3 0 1 -1 -1 0 0 4 0.000 1 0.0000 6225 4650 105 105 6225 4650 6330 4755
-4 0 -1 0 0 0 12 0.0000 2 135 1035 4950 4725 blocked task\001
-4 0 -1 0 0 0 12 0.0000 2 135 870 3525 4725 active task\001
-4 0 -1 0 0 0 12 0.0000 2 180 930 6450 4725 routine ptrs\001
+5 1 0 1 0 7 50 -1 -1 0.000 0 1 0 0 675.000 2700.000 675 2400 375 2700 675 3000
+6 4533 2866 4655 3129
+5 1 0 1 0 7 50 -1 -1 0.000 0 1 0 0 4657.017 2997.000 4655 2873 4533 2997 4655 3121
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
+         4655 2866 4655 3129
 -6
-6 8445 1695 8655 1905
-1 3 0 1 -1 -1 0 0 -1 0.000 1 0.0000 8550 1800 105 105 8550 1800 8655 1905
-4 1 -1 0 0 0 10 0.0000 2 75 75 8550 1860 a\001
+6 4725 2866 4847 3129
+5 1 0 1 0 7 50 -1 -1 0.000 0 1 0 0 4849.017 2997.000 4847 2873 4725 2997 4847 3121
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
+         4847 2866 4847 3129
 -6
-6 8445 1395 8655 1605
-1 3 0 1 -1 -1 0 0 -1 0.000 1 0.0000 8550 1500 105 105 8550 1500 8655 1605
-4 1 -1 0 0 0 10 0.0000 2 105 90 8550 1560 b\001
+6 4911 2866 5033 3129
+5 1 0 1 0 7 50 -1 -1 0.000 0 1 0 0 5035.017 2997.000 5033 2873 4911 2997 5033 3121
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
+         5033 2866 5033 3129
 -6
-6 3945 1695 4155 1905
-1 3 0 1 -1 -1 0 0 -1 0.000 1 0.0000 4050 1800 105 105 4050 1800 4155 1905
-4 1 -1 0 0 0 10 0.0000 2 75 75 4050 1860 a\001
+6 9027 2866 9149 3129
+5 1 0 1 0 7 50 -1 -1 0.000 0 0 0 0 9024.983 2997.000 9027 2873 9149 2997 9027 3121
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
+         9027 2866 9027 3129
 -6
-6 3945 1395 4155 1605
-1 3 0 1 -1 -1 0 0 -1 0.000 1 0.0000 4050 1500 105 105 4050 1500 4155 1605
-4 1 -1 0 0 0 10 0.0000 2 105 90 4050 1560 b\001
+6 9253 2866 9375 3129
+5 1 0 1 0 7 50 -1 -1 0.000 0 0 0 0 9250.983 2997.000 9253 2873 9375 2997 9253 3121
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
+         9253 2866 9253 3129
+-6
+6 9478 2866 9600 3129
+5 1 0 1 0 7 50 -1 -1 0.000 0 0 0 0 9475.983 2997.000 9478 2873 9600 2997 9478 3121
+2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
+         9478 2866 9478 3129
 -6
 1 3 0 1 -1 -1 0 0 20 0.000 1 0.0000 7650 3675 80 80 7650 3675 7730 3755
 1 3 0 1 -1 -1 0 0 20 0.000 1 0.0000 3150 3675 80 80 3150 3675 3230 3755
+1 3 0 1 0 7 50 -1 -1 0.000 1 0.0000 4047 1793 125 125 4047 1793 3929 1752
+1 3 0 1 0 7 50 -1 -1 0.000 1 0.0000 4050 1500 125 125 4050 1500 3932 1459
+1 3 0 1 0 7 50 -1 -1 0.000 1 0.0000 8550 1500 125 125 8550 1500 8432 1459
+1 3 0 1 0 7 50 -1 -1 0.000 1 0.0000 8550 1800 125 125 8550 1800 8432 1759
+1 3 0 1 0 7 50 -1 -1 0.000 1 0.0000 1200 2850 125 125 1200 2850 1082 2809
+1 3 0 1 0 7 50 -1 -1 0.000 1 0.0000 900 2850 125 125 900 2850 782 2809
+1 3 0 1 -1 -1 0 0 4 0.000 1 0.0000 6225 4650 105 105 6225 4650 6330 4755
+1 3 0 1 -1 -1 0 0 20 0.000 1 0.0000 3150 4650 80 80 3150 4650 3230 4730
+1 3 0 1 -1 -1 0 0 -1 0.000 1 0.0000 4575 4650 105 105 4575 4650 4680 4755
 2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
          3900 1950 4200 2100
@@ -62 +77 @@
          3000 4050 3300 4200
 2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
-         600 2925 1350 2925
+         675 3000 1425 3000
 2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
-         600 2325 1350 2325
+         675 2400 1425 2400
 2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
-         1350 2625 1425 2850
+         1425 2700 1500 2925
 2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
-         1350 2325 1275 2550
+         1425 2400 1350 2625
 2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
-         600 2625 1350 2625
-2 3 0 1 0 7 50 -1 -1 0.000 0 0 0 0 0 7
-         1350 2775 1275 2645 1125 2645 1050 2775 1125 2905 1275 2905
-         1350 2775
-2 3 0 1 0 7 50 -1 -1 0.000 0 0 0 0 0 7
-         975 2775 900 2645 750 2645 675 2775 750 2905 900 2905
-         975 2775
-2 3 0 1 0 7 50 -1 -1 0.000 0 0 0 0 0 7
-         4800 3000 4725 2870 4575 2870 4500 3000 4575 3130 4725 3130
-         4800 3000
-2 3 0 1 0 7 50 -1 -1 0.000 0 0 0 0 0 7
-         5100 3000 5025 2870 4875 2870 4800 3000 4875 3130 5025 3130
-         5100 3000
-2 3 0 1 0 7 50 -1 -1 0.000 0 0 0 0 0 7
-         9300 3000 9225 2870 9075 2870 9000 3000 9075 3130 9225 3130
-         9300 3000
-2 3 0 1 0 7 50 -1 -1 0.000 0 0 0 0 0 7
-         9600 3000 9525 2870 9375 2870 9300 3000 9375 3130 9525 3130
-         9600 3000
-2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
-         675 2775 975 2775
-2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
-         1050 2775 1350 2775
-2 3 0 1 0 7 50 -1 -1 0.000 0 0 0 0 0 7
-         4875 4950 4800 4820 4650 4820 4575 4950 4650 5080 4800 5080
-         4875 4950
-2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 2
-         4575 4950 4875 4950
-2 3 0 1 0 7 50 -1 -1 0.000 0 0 0 0 0 7
-         3525 4970 3450 4840 3300 4840 3225 4970 3300 5100 3450 5100
-         3525 4970
+         675 2700 1425 2700
 4 1 -1 0 0 0 12 0.0000 2 135 315 2850 4275 exit\001
 4 1 -1 0 0 0 12 0.0000 2 135 315 7350 4275 exit\001
@@ -121 +106 @@
 4 1 -1 0 0 0 12 0.0000 2 135 495 4050 1275 queue\001
 4 1 -1 0 0 0 12 0.0000 2 165 420 4050 1050 entry\001
-4 0 0 50 -1 0 11 0.0000 2 120 705 450 2250 Condition\001
-4 0 0 50 -1 0 11 0.0000 2 165 630 3600 5025 signalled\001
-4 0 0 50 -1 0 11 0.0000 2 165 525 4950 5025 waiting\001
+4 0 0 50 -1 0 11 0.0000 2 120 705 600 2325 Condition\001
+4 0 -1 0 0 0 12 0.0000 2 180 930 6450 4725 routine ptrs\001
+4 0 -1 0 0 0 12 0.0000 2 135 1050 3300 4725 active thread\001
+4 0 -1 0 0 0 12 0.0000 2 135 1215 4725 4725 blocked thread\001

doc/proposals/concurrency/text/basics.tex

@@ -11 +11 @@
 Execution with a single thread and multiple stacks where the thread is self-scheduling deterministically across the stacks is called coroutining. Execution with a single and multiple stacks but where the thread is scheduled by an oracle (non-deterministic from the thread perspective) across the stacks is called concurrency.
 
-Therefore, a minimal concurrency system can be achieved by creating coroutines, which instead of context switching among each other, always ask an oracle where to context switch next. While coroutines can execute on the caller's stack-frame, stackfull coroutines allow full generality and are sufficient as the basis for concurrency. The aforementioned oracle is a scheduler and the whole system now follows a cooperative threading-model \cit. The oracle/scheduler can either be a stackless or stackfull entity and correspondingly require one or two context switches to run a different coroutine. In any case, a subset of concurrency related challenges start to appear. For the complete set of concurrency challenges to occur, the only feature missing is preemption.
-
-A scheduler introduces order of execution uncertainty, while preemption introduces uncertainty about where context-switches occur. Mutual-exclusion and synchronisation are ways of limiting non-determinism in a concurrent system. Now it is important to understand that uncertainty is desireable; uncertainty can be used by runtime systems to significantly increase performance and is often the basis of giving a user the illusion that tasks are running in parallel. Optimal performance in concurrent applications is often obtained by having as much non-determinism as correctness allows\cit.
+Therefore, a minimal concurrency system can be achieved by creating coroutines, which instead of context switching among each other, always ask an oracle where to context switch next. While coroutines can execute on the caller's stack-frame, stackfull coroutines allow full generality and are sufficient as the basis for concurrency. The aforementioned oracle is a scheduler and the whole system now follows a cooperative threading-model (a.k.a non-preemptive scheduling). The oracle/scheduler can either be a stackless or stackfull entity and correspondingly require one or two context switches to run a different coroutine. In any case, a subset of concurrency related challenges start to appear. For the complete set of concurrency challenges to occur, the only feature missing is preemption.
+
+A scheduler introduces order of execution uncertainty, while preemption introduces uncertainty about where context-switches occur. Mutual-exclusion and synchronisation are ways of limiting non-determinism in a concurrent system. Now it is important to understand that uncertainty is desireable; uncertainty can be used by runtime systems to significantly increase performance and is often the basis of giving a user the illusion that tasks are running in parallel. Optimal performance in concurrent applications is often obtained by having as much non-determinism as correctness allows.
 
 \section{\protect\CFA 's Thread Building Blocks}
@@ -307 +307 @@
 \subsection{Alternative: Lamda Objects}
 
-For coroutines as for threads, many implementations are based on routine pointers or function objects\cit. For example, Boost implements coroutines in terms of four functor object types:
+For coroutines as for threads, many implementations are based on routine pointers or function objects\cite{Butenhof97, ANSI14:C++, MS:VisualC++, BoostCoroutines15}. For example, Boost implements coroutines in terms of four functor object types:
 \begin{cfacode}
 asymmetric_coroutine<>::pull_type

doc/proposals/concurrency/text/concurrency.tex

@@ -8 +8 @@
 Approaches based on shared memory are more closely related to non-concurrent paradigms since they often rely on basic constructs like routine calls and shared objects. At the lowest level, concurrent paradigms are implemented as atomic operations and locks. Many such mechanisms have been proposed, including semaphores~\cite{Dijkstra68b} and path expressions~\cite{Campbell74}. However, for productivity reasons it is desireable to have a higher-level construct be the core concurrency paradigm~\cite{HPP:Study}.
 
-An approach that is worth mentioning because it is gaining in popularity is transactionnal memory~\cite{Dice10}[Check citation]. While this approach is even pursued by system languages like \CC\cit, the performance and feature set is currently too restrictive to be the main concurrency paradigm for systems language, which is why it was rejected as the core paradigm for concurrency in \CFA.
+An approach that is worth mentioning because it is gaining in popularity is transactionnal memory~\cite{Dice10}[Check citation]. While this approach is even pursued by system languages like \CC\cite{Cpp-Transactions}, the performance and feature set is currently too restrictive to be the main concurrency paradigm for systems language, which is why it was rejected as the core paradigm for concurrency in \CFA.
 
 One of the most natural, elegant, and efficient mechanisms for synchronization and communication, especially for shared-memory systems, is the \emph{monitor}. Monitors were first proposed by Brinch Hansen~\cite{Hansen73} and later described and extended by C.A.R.~Hoare~\cite{Hoare74}. Many programming languages---e.g., Concurrent Pascal~\cite{ConcurrentPascal}, Mesa~\cite{Mesa}, Modula~\cite{Modula-2}, Turing~\cite{Turing:old}, Modula-3~\cite{Modula-3}, NeWS~\cite{NeWS}, Emerald~\cite{Emerald}, \uC~\cite{Buhr92a} and Java~\cite{Java}---provide monitors as explicit language constructs. In addition, operating-system kernels and device drivers have a monitor-like structure, although they often use lower-level primitives such as semaphores or locks to simulate monitors. For these reasons, this project proposes monitors as the core concurrency-construct.
@@ -139 +139 @@
 The \gls{multi-acq} monitor lock allows a monitor lock to be acquired by both \code{bar} or \code{baz} and acquired again in \code{foo}. In the calls to \code{bar} and \code{baz} the monitors are acquired in opposite order.
 
-However, such use leads to the lock acquiring order problem. In the example above, the user uses implicit ordering in the case of function \code{foo} but explicit ordering in the case of \code{bar} and \code{baz}. This subtle mistake means that calling these routines concurrently may lead to deadlock and is therefore undefined behavior. As shown\cit, solving this problem requires:
+However, such use leads to the lock acquiring order problem. In the example above, the user uses implicit ordering in the case of function \code{foo} but explicit ordering in the case of \code{bar} and \code{baz}. This subtle mistake means that calling these routines concurrently may lead to deadlock and is therefore undefined behavior. As shown\cite{Lister77}, solving this problem requires:
 \begin{enumerate}
         \item Dynamically tracking of the monitor-call order.
         \item Implement rollback semantics.
 \end{enumerate}
-While the first requirement is already a significant constraint on the system, implementing a general rollback semantics in a C-like language is prohibitively complex \cit. In \CFA, users simply need to be carefull when acquiring multiple monitors at the same time or only use \gls{bulk-acq} of all the monitors. While \CFA provides only a partial solution, many system provide no solution and the \CFA partial solution handles many useful cases.
+While the first requirement is already a significant constraint on the system, implementing a general rollback semantics in a C-like language is still prohibitively complex \cite{Dice10}. In \CFA, users simply need to be carefull when acquiring multiple monitors at the same time or only use \gls{bulk-acq} of all the monitors. While \CFA provides only a partial solution, many system provide no solution and the \CFA partial solution handles many useful cases.
 
 For example, \gls{multi-acq} and \gls{bulk-acq} can be used together in interesting ways:
@@ -157 +157 @@
 }
 \end{cfacode}
-This example shows a trivial solution to the bank-account transfer-problem\cit. Without \gls{multi-acq} and \gls{bulk-acq}, the solution to this problem is much more involved and requires carefull engineering.
+This example shows a trivial solution to the bank-account transfer-problem\cite{BankTransfer}. Without \gls{multi-acq} and \gls{bulk-acq}, the solution to this problem is much more involved and requires carefull engineering.
 
 \subsection{\code{mutex} statement} \label{mutex-stmt}
 
-The call semantics discussed aboved have one software engineering issue, only a named routine can acquire the mutual-exclusion of a set of monitor. \CFA offers the \code{mutex} statement to workaround the need for unnecessary names, avoiding a major software engineering problem\cit. Listing \ref{lst:mutex-stmt} shows an example of the \code{mutex} statement, which introduces a new scope in which the mutual-exclusion of a set of monitor is acquired. Beyond naming, the \code{mutex} statement has no semantic difference from a routine call with \code{mutex} parameters.
+The call semantics discussed aboved have one software engineering issue, only a named routine can acquire the mutual-exclusion of a set of monitor. \CFA offers the \code{mutex} statement to workaround the need for unnecessary names, avoiding a major software engineering problem\cite{2FTwoHardThings}. Listing \ref{lst:mutex-stmt} shows an example of the \code{mutex} statement, which introduces a new scope in which the mutual-exclusion of a set of monitor is acquired. Beyond naming, the \code{mutex} statement has no semantic difference from a routine call with \code{mutex} parameters.
 
 \begin{figure}
@@ -232 +232 @@
 % ======================================================================
 % ======================================================================
-In addition to mutual exclusion, the monitors at the core of \CFA's concurrency can also be used to achieve synchronisation. With monitors, this capability is generally achieved with internal or external scheduling as in\cit. Since internal scheduling within a single monitor is mostly a solved problem, this thesis concentrates on extending internal scheduling to multiple monitors. Indeed, like the \gls{bulk-acq} semantics, internal scheduling extends to multiple monitors in a way that is natural to the user but requires additional complexity on the implementation side.
+In addition to mutual exclusion, the monitors at the core of \CFA's concurrency can also be used to achieve synchronisation. With monitors, this capability is generally achieved with internal or external scheduling as in \cite{Hoare74}. Since internal scheduling within a single monitor is mostly a solved problem, this thesis concentrates on extending internal scheduling to multiple monitors. Indeed, like the \gls{bulk-acq} semantics, internal scheduling extends to multiple monitors in a way that is natural to the user but requires additional complexity on the implementation side.
 
 First, here is a simple example of such a technique:
@@ -305 +305 @@
 This version uses \gls{bulk-acq} (denoted using the {\sf\&} symbol), but the presence of multiple monitors does not add a particularly new meaning. Synchronization happens between the two threads in exactly the same way and order. The only difference is that mutual exclusion covers more monitors. On the implementation side, handling multiple monitors does add a degree of complexity as the next few examples demonstrate.
 
-While deadlock issues can occur when nesting monitors, these issues are only a symptom of the fact that locks, and by extension monitors, are not perfectly composable. For monitors, a well known deadlock problem is the Nested Monitor Problem\cit, which occurs when a \code{wait} is made by a thread that holds more than one monitor. For example, the following pseudo-code runs into the nested-monitor problem :
+While deadlock issues can occur when nesting monitors, these issues are only a symptom of the fact that locks, and by extension monitors, are not perfectly composable. For monitors, a well known deadlock problem is the Nested Monitor Problem \cite{Lister77}, which occurs when a \code{wait} is made by a thread that holds more than one monitor. For example, the following pseudo-code runs into the nested-monitor problem :
 \begin{multicols}{2}
 \begin{pseudo}
@@ -771 +771 @@
 For the first two conditions, it is easy to implement a check that can evaluate the condition in a few instruction. However, a fast check for \pscode{monitor accepts me} is much harder to implement depending on the constraints put on the monitors. Indeed, monitors are often expressed as an entry queue and some acceptor queue as in the following figure:
 
+\begin{figure}[H]
 \begin{center}
 {\resizebox{0.4\textwidth}{!}{\input{monitor}}}
 \end{center}
+\label{fig:monitor}
+\end{figure}
 
 There are other alternatives to these pictures, but in the case of this picture, implementing a fast accept check is relatively easy. Restricted to a fixed number of mutex members, N, the accept check reduces to updating a bitmask when the acceptor queue changes, a check that executes in a single instruction even with a fairly large number (e.g., 128) of mutex members. This technique cannot be used in \CFA because it relies on the fact that the monitor type enumerates (declares) all the acceptable routines. For OO languages this does not compromise much since monitors already have an exhaustive list of member routines. However, for \CFA this is not the case; routines can be added to a type anywhere after its declaration. It is important to note that the bitmask approach does not actually require an exhaustive list of routines, but it requires a dense unique ordering of routines with an upper-bound and that ordering must be consistent across translation units.

doc/proposals/concurrency/text/future.tex

@@ -6 +6 @@
 
 \section{Flexible Scheduling} \label{futur:sched}
-
+An important part of concurrency is scheduling. Different scheduling algorithm can affact peformance (both in terms of average and variation). However, no single scheduler is optimal for all workloads and therefore there is value in being able to change the scheduler for given programs. One solution is to offer various tweaking options to users, allowing the scheduler to be adjusted the to requirements of the workload. However, in order to be truly flexible, it would be interesting to allow users to add arbitrary data and arbirary scheduling algorithms to the scheduler. For example, a web server could attach Type-of-Service information to threads and have a ``ToS aware'' scheduling algorithm tailored to this specific web server. This path of flexible schedulers will be explored for \CFA.
 
 \section{Non-Blocking IO} \label{futur:nbio}
 While most of the parallelism tools
-However, many modern workloads are not bound on computation but on IO operations, an common case being webservers and XaaS (anything as a service). These type of workloads often require significant engineering around amortising costs of blocking IO operations. While improving throughtput of these operations is outside what \CFA can do as a language, it can help users to make better use of the CPU time otherwise spent waiting on IO operations. The current trend is to use asynchronous programming using tools like callbacks and/or futurs and promises\cit. However, while these are valid solutions, they lead to code that is harder to read and maintain because it is much less linear
-
-
+However, many modern workloads are not bound on computation but on IO operations, an common case being webservers and XaaS (anything as a service). These type of workloads often require significant engineering around amortising costs of blocking IO operations. While improving throughtput of these operations is outside what \CFA can do as a language, it can help users to make better use of the CPU time otherwise spent waiting on IO operations. The current trend is to use asynchronous programming using tools like callbacks and/or futurs and promises\cite. However, while these are valid solutions, they lead to code that is harder to read and maintain because it is much less linear
 
 \section{Other concurrency tools} \label{futur:tools}
-
+While monitors offer a flexible and powerful concurent core for \CFA, other concurrency tools are also necessary for a complete multi-paradigm concurrency package. Example of such tools can include simple locks and condition variables, futures and promises\cite{promises}, and executors. These additional features are useful when monitors offer a level of abstraction which is indaquate for certain tasks.
 
 \section{Implicit threading} \label{futur:implcit}
-Simpler applications can benefit greatly from having implicit parallelism. That is, parallelism that does not rely on the user to write concurrency. This type of parallelism can be achieved both at the language level and at the library level. The cannonical example of implcit parallelism is parallel for loops, which are the simplest example of a divide and conquer algorithm\cit. Listing \ref{lst:parfor} shows three different code examples that accomplish pointwise sums of large arrays. Note that none of these example explicitly declare any concurrency or parallelism objects.
+Simpler applications can benefit greatly from having implicit parallelism. That is, parallelism that does not rely on the user to write concurrency. This type of parallelism can be achieved both at the language level and at the library level. The cannonical example of implcit parallelism is parallel for loops, which are the simplest example of a divide and conquer algorithm\cite{uC++book}. Listing \ref{lst:parfor} shows three different code examples that accomplish pointwise sums of large arrays. Note that none of these example explicitly declare any concurrency or parallelism objects.
 
 \begin{figure}
@@ -103 +101 @@
 \end{figure}
 
-Implicit parallelism is a general solution and therefore is
-
-\section{Multiple Paradigms} \label{futur:paradigms}
+Implicit parallelism is a general solution and therefore has its limitations. However, it is a quick and simple approach to parallelism which may very well be sufficient for smaller applications and reduces the amount of boiler-plate that is needed to start benefiting from parallelism in modern CPUs.
 
 
-\section{Transactions} \label{futur:transaction}
-Concurrency and parallelism is still a very active field that strongly benefits from hardware advances. As such certain features that aren't necessarily mature enough in their current state could become relevant in the lifetime of \CFA.

doc/proposals/concurrency/text/internals.tex

@@ -1 +1 @@
 
 \chapter{Behind the scene}
-There are several challenges specific to \CFA when implementing concurrency. These challenges are direct results of \gls{bulk-acq} and loose object definitions. These two constraints are to root cause of most design decisions in the implementation. Furthermore, to avoid the head-aches of dynamically allocating memory in a concurrent environment, the internal-scheduling design is (almost) entirely free of mallocs and other dynamic memory allocation scheme. This is to avoid the chicken and egg problem \cite{Chicken} of having a memory allocator that relies on the threading system and a threading system that relies on the runtime. This extra goal, means that memory management is a constant concern in the design of the system.
-
-The main memory concern for concurrency is queues. All blocking operations are made by parking threads onto queues. These queues need to be intrinsic\cit to avoid the need memory allocation. This entails that all the fields needed to keep track of all needed information. Since many conconcurrency operations can use an unbound amount of memory (depending on \gls{bulk-acq}) statically defining information in the intrusive fields of threads is insufficient. The only variable sized container that does not require memory allocation is the callstack, which is heavily used in the implementation of internal scheduling. Particularly the GCC extension variable length arrays which is used extensively.
+There are several challenges specific to \CFA when implementing concurrency. These challenges are a direct result of \gls{bulk-acq} and loose object-definitions. These two constraints are the root cause of most design decisions in the implementation. Furthermore, to avoid contention from dynamically allocating memory in a concurrent environment, the internal-scheduling design is (almost) entirely free of mallocs. This is to avoid the chicken and egg problem \cite{Chicken} of having a memory allocator that relies on the threading system and a threading system that relies on the runtime. This extra goal, means that memory management is a constant concern in the design of the system.
+
+The main memory concern for concurrency is queues. All blocking operations are made by parking threads onto queues. The queue design needs to be intrusive\cite{IntrusiveData} to avoid the need for memory allocation, which entails that all the nodes need specific fields to keep track of all needed information. Since many concurrency operations can use an unbound amount of memory (depending on \gls{bulk-acq}), statically defining information in the intrusive fields of threads is insufficient. The only variable sized container that does not require memory allocation is the callstack, which is heavily used in the implementation of internal scheduling. Particularly variable length arrays, which are used extensively.
 
 Since stack allocation is based around scope, the first step of the implementation is to identify the scopes that are available to store the information, and which of these can have a variable length. The threads and the condition both allow a fixed amount of memory to be stored, while mutex-routines and the actual blocking call allow for an unbound amount (though the later is preferable in terms of performance).
 
-Note that since the major contributions of this thesis are extending monitor semantics to \gls{bulk-acq} and loose object definitions, any challenges that are not resulting of these characteristiques of \CFA are consired as problems which have already been solved and therefore will not be discussed further.
+Note that since the major contributions of this thesis are extending monitor semantics to \gls{bulk-acq} and loose object definitions, any challenges that are not resulting of these characteristiques of \CFA are considered as solved problems and therefore not discussed further.
 
 % ======================================================================
@@ -15 +15 @@
 % ======================================================================
 
-The first step towards the monitor implementation is simple mutex-routines using monitors. In the single monitor case, this is done using the entry/exit procedure highlighted in listing \ref{lst:entry1}. This entry/exit procedure doesn't actually have to be extended to support multiple monitors, indeed it is sufficient to enter/leave monitors one-by-one as long as the order is correct to prevent deadlocks\cit. In \CFA, ordering of monitor relies on memory ordering, this is sufficient because all objects are guaranteed to have distinct non-overlaping memory layouts and mutual-exclusion for a monitor is only defined for its lifetime, meaning that destroying a monitor while it is acquired is undefined behavior. When a mutex call is made, the concerned monitors are agregated into an variable-length pointer array and sorted based on pointer values. This array is concerved during the entire duration of the mutual-exclusion and it's ordering reused extensively.
+The first step towards the monitor implementation is simple mutex-routines using monitors. In the single monitor case, this is done using the entry/exit procedure highlighted in listing \ref{lst:entry1}. This entry/exit procedure does not actually have to be extended to support multiple monitors, indeed it is sufficient to enter/leave monitors one-by-one as long as the order is correct to prevent deadlocks\cite{Havender68}. In \CFA, ordering of monitor relies on memory ordering, this is sufficient because all objects are guaranteed to have distinct non-overlaping memory layouts and mutual-exclusion for a monitor is only defined for its lifetime, meaning that destroying a monitor while it is acquired is undefined behavior. When a mutex call is made, the concerned monitors are agregated into a variable-length pointer array and sorted based on pointer values. This array presists for the entire duration of the mutual-exclusion and its ordering reused extensively.
 \begin{figure}
 \begin{multicols}{2}
@@ -96 +96 @@
 \end{tabular}
 \end{center}
-\caption{Callsite vs entry-point locking for mutex calls}
+\caption{Call-site vs entry-point locking for mutex calls}
 \label{fig:locking-site}
 \end{figure}
 
-Note the \code{mutex} keyword relies on the type system, which means that in cases where a generic monitor routine is actually desired, writing a mutex routine is possible with the proper trait, for example:
+Note the \code{mutex} keyword relies on the type system, which means that in cases where a generic monitor routine is desired, writing the mutex routine is possible with the proper trait, for example:
 \begin{cfacode}
-//Incorrect: T is not a monitor
+//Incorrect: T may not be monitor
 forall(dtype T)
 void foo(T * mutex t);
@@ -111 +111 @@
 \end{cfacode}
 
-Both entry-point and callsite locking are valid implementations. The current \CFA implementations uses entry-point locking because it seems to require less work if done using \gls{raii}, effectively transferring the burden of implementation to object construction/destruction. The same could be said of callsite locking, the difference being that the later does not necessarily have an existing scope that matches exactly the scope of the mutual exclusion, i.e.: the function body.
+Both entry-point and callsite locking are feasible implementations. The current \CFA implementations uses entry-point locking because it requires less work when using \gls{raii}, effectively transferring the burden of implementation to object construction/destruction. The same could be said of callsite locking, the difference being that the later does not necessarily have an existing scope that matches exactly the scope of the mutual exclusion, i.e.: the function body. Furthermore, entry-point locking requires less code generation since any useful routine is called at least as often as it is define, there can be only one entry-point but many callsites.
 
 % ======================================================================
@@ -119 +119 @@
 % ======================================================================
 
-Figure \ref{fig:system1} shows a high-level picture if the \CFA runtime system in regards to concurrency.
+Figure \ref{fig:system1} shows a high-level picture if the \CFA runtime system in regards to concurrency. Each component of the picture is explained in details in the fllowing sections.
 
 \begin{figure}
@@ -130 +130 @@
 
 \subsection{Context Switching}
-As mentionned in section \ref{coroutine}, coroutines are a stepping stone for implementing threading. This is because they share the same mechanism for context-switching between different stacks. To improve performance and simplicity, context-switching is implemented using the following assumption: all context-switches happen inside a specific function call. This assumptions means that the basic recipe for context-switch is only to copy all callee-saved registers unto the stack and then switch the stack registers with the ones of the target coroutine/thread. Note that instruction pointer can be left untouched since the context-switch always inside the same function. In the case of coroutines, that is the entire story. Threads however do not simply context-switch between each other directly. The context-switch to processors which is where the scheduling happens. This method is called a 2-step context-switch and has the advantage of having a clear distinction between user code and the "kernel" where scheduling and other system operation happen. Obiously, this has the cost of doubling the context-switch cost from because threads must context-switch to an intermediate stack. However, the performance of the 2-step context-switch is still superior to a \code{pthread_yield}(see section \ref{results}). additionally, for users in need for optimal performance, it is important to note that having a 2-step context-switch as the default does not prevent \CFA from offering a 1-step context-switch to use manually (or as part of monitors). This option is not currently present in \CFA but the changes required to add it are strictly additive.
+As mentionned in section \ref{coroutine}, coroutines are a stepping stone for implementing threading. This is because they share the same mechanism for context-switching between different stacks. To improve performance and simplicity, context-switching is implemented using the following assumption: all context-switches happen inside a specific function call. This assumption means that the context-switch only has to copy the callee-saved registers onto the stack and then switch the stack registers with the ones of the target coroutine/thread. Note that the instruction pointer can be left untouched since the context-switch is always inside the same function. Threads however do not context-switch between each other directly. They context-switch to the scheduler. This method is called a 2-step context-switch and has the advantage of having a clear distinction between user code and the kernel where scheduling and other system operation happen. Obiously, this has the cost of doubling the context-switch cost because threads must context-switch to an intermediate stack. However, the performance of the 2-step context-switch is still superior to a \code{pthread_yield}(see section \ref{results}). additionally, for users in need for optimal performance, it is important to note that having a 2-step context-switch as the default does not prevent \CFA from offering a 1-step context-switch to use manually (or as part of monitors). This option is not currently present in \CFA but the changes required to add it are strictly additive.
 
 \subsection{Processors}
-Parallelism in \CFA are built around using processors to specify how much parallelism is desired. \CFA processors are object wrappers around kernel threads, specifically pthreads in the current implementation of \CFA. Indeed, any parallelism must go through operatiing system librairies. However, \gls{cfathread} are still the main source of concurrency, processors are simply the underlying source of parallelism. Indeed, processor kernel threads simply fetch a user-level thread from the scheduler and run, they are effectively executers for user-threads. The main benefit of this approach is that it offers a well defined boundary between kernel code and user-code, for example kernel thread quiescing, scheduling and interrupt handling. Processors internally use coroutines to take advantage of the existing context-switching semantics.
+Parallelism in \CFA is built around using processors to specify how much parallelism is desired. \CFA processors are object wrappers around kernel threads, specifically pthreads in the current implementation of \CFA. Indeed, any parallelism must go through operating-system librairies. However, \glspl{uthread} are still the main source of concurrency, processors are simply the underlying source of parallelism. Indeed, processor \glspl{kthread} simply fetch a \glspl{uthread} from the scheduler and run, they are effectively executers for user-threads. The main benefit of this approach is that it offers a well defined boundary between kernel code and user code, for example, kernel thread quiescing, scheduling and interrupt handling. Processors internally use coroutines to take advantage of the existing context-switching semantics.
 
 \subsection{Stack management}
 One of the challenges of this system is to reduce the footprint as much as possible. Specifically, all pthreads created also have a stack created with them, which should be used as much as possible. Normally, coroutines also create there own stack to run on, however, in the case of the coroutines used for processors, these coroutines run directly on the kernel thread stack, effectively stealing the processor stack. The exception to this rule is the Main Processor, i.e. the initial kernel thread that is given to any program. In order to respect user expectations, the stack of the initial kernel thread, the main stack of the program, is used by the main user thread rather than the main processor.
 
-\subsection{Preemption}
-Finally, an important aspect for any complete threading system is preemption. As mentionned in chapter \ref{basics}, preemption introduces an extra degree of unceretainty, which enables users to have multiple threads interleave transparrently between eachother, rather than having to cooperate between thread for proper scheduling and CPU distribution. Indeed, preemption is desireable because it adds a degree of isolation between tasks. In a fully cooperative system, any thread that runs into a long loop can starve other threads, while in a preemptive system starvation can still occur but it does not rely on every thread having to yield or block on a regular basis, which reduces significantly programmer burden. Obviously, preemption is not optimal for every workload, however any preemptive system can become a cooperative system by making the time-slices extremely large. Which is why \CFA uses a preemptive threading system.
-
-Preemption in \CFA is based on kernel timers which are used to run a discreet event simulation. Every processor keeps track of the current time and registers an expiration time with the preemption system. When the preemption system receives a change in preemption it sorts these expiration times in a list and sets a kernel timer for the closest one, effectiveling stepping between preemption events on each signals sent by the timer. These timers use the linux signal {\tt SIGALRM}, which is delivered to the process. This is important because when delivering signals to a process, the kernel documentation states that the signal can be delivered to any kernel thread for which the signal isn't block i.e. :
+\subsection{Preemption} \label{preemption}
+Finally, an important aspect for any complete threading system is preemption. As mentionned in chapter \ref{basics}, preemption introduces an extra degree of uncertainty, which enables users to have multiple threads interleave transparently, rather than having to cooperate among threads for proper scheduling and CPU distribution. Indeed, preemption is desireable because it adds a degree of isolation among threads. In a fully cooperative system, any thread that runs into a long loop can starve other threads, while in a preemptive system starvation can still occur but it does not rely on every thread having to yield or block on a regular basis, which reduces significantly a programmer burden. Obviously, preemption is not optimal for every workload, however any preemptive system can become a cooperative system by making the time-slices extremely large. Which is why \CFA uses a preemptive threading system.
+
+Preemption in \CFA is based on kernel timers, which are used to run a discrete-event simulation. Every processor keeps track of the current time and registers an expiration time with the preemption system. When the preemption system receives a change in preemption, it sorts these expiration times in a list and sets a kernel timer for the closest one, effectively stepping between preemption events on each signals sent by the timer. These timers use the linux signal {\tt SIGALRM}, which is delivered to the process rather than the kernel-thread. This results in an implementation problem,because when delivering signals to a process, the kernel documentation states that the signal can be delivered to any kernel thread for which the signal is not blocked i.e. :
 \begin{quote}
 A process-directed signal may be delivered to any one of the threads that does not currently have the signal blocked. If more than one of the threads has the signal unblocked, then the kernel chooses an arbitrary thread to which to deliver the signal.
@@ -148 +148 @@
 For the sake of simplicity and in order to prevent the case of having two threads receiving alarms simultaneously, \CFA programs block the {\tt SIGALRM} signal on every thread except one. Now because of how involontary context-switches are handled, the kernel thread handling {\tt SIGALRM} cannot also be a processor thread.
 
-Involontary context-switching is done by sending {\tt SIGUSER1} to the corresponding processor and having the thread yield from inside the signal handler. Effectively context-switch away from the signal-handler back to the kernel and the signal-handler frame will be unwound when the thread is scheduled again. This means that a signal-handler can start on one kernel thread and terminate on a second kernel thread (but the same user thread). It is important to note that signal-handlers save and restore signal masks because user-thread migration can cause signal mask to migrate from one kernel thread to another. This is only a problem if all kernel threads among which a user thread can migrate differ in terms of signal masks. However, since the kernel thread hanlding preemption requires a different signal mask, executing user threads on the kernel alarm thread can cause deadlocks. For this reason, the alarm thread is on a tight loop around a system call to \code{sigwait} or more specifically \code{sigwaitinfo}, requiring very little CPU time for preemption. One final detail about the alarm thread is how to wake it when additional communication is required (e.g. on thread termination). This is also done using {\tt SIGALRM}, but sent throught the \code{pthread_sigqueue}. Indeed, \code{sigwait} can differentiate signals sent from \code{pthread_sigqueue} from signals sent from alarms or the kernel.
-
-\subsection{Scheduler} \footnote{ I'm not sure what to write here, is this section even needed. }
-Finally, an aspect that was not mentionned yet is the scheduling algorithm. Currently, the \CFA scheduler uses a single ready queue for all processors. Will this is not the highest performance algorithm, it has the significant advantage of being robust to heterogenous workloads. This is a very simple scheduling approach but is sufficient to for the context of this thesis.
-
-What to do here?
-
-However, when
-As will be mentionned \ref{futur:sched} it needs to be updated when clusters will be
-
-clusters
-
-
-
-Among the most pressing updates to the \CFA
-uses single queue
-in future should move to multiple queues with workstealing
-general purpouse means robust > fast
-worksharing can higher standard deviation in performance
-
+Involuntary context-switching is done by sending signal {\tt SIGUSER1} to the corresponding processor and having the thread yield from inside the signal handler. Effectively context-switching away from the signal-handler back to the kernel and the signal-handler frame is eventually unwound when the thread is scheduled again. This approach means that a signal-handler can start on one kernel thread and terminate on a second kernel thread (but the same user thread). It is important to note that signal-handlers save and restore signal masks because user-thread migration can cause signal mask to migrate from one kernel thread to another. This behaviour is only a problem if all kernel threads among which a user thread can migrate differ in terms of signal masks\footnote{Sadly, official POSIX documentation is silent on what distiguishes ``async-signal-safe'' functions from other functions}. However, since the kernel thread hanlding preemption requires a different signal mask, executing user threads on the kernel alarm thread can cause deadlocks. For this reason, the alarm thread is on a tight loop around a system call to \code{sigwaitinfo}, requiring very little CPU time for preemption. One final detail about the alarm thread is how to wake it when additional communication is required (e.g., on thread termination). This unblocking is also done using {\tt SIGALRM}, but sent throught the \code{pthread_sigqueue}. Indeed, \code{sigwait} can differentiate signals sent from \code{pthread_sigqueue} from signals sent from alarms or the kernel.
+
+\subsection{Scheduler}
+Finally, an aspect that was not mentionned yet is the scheduling algorithm. Currently, the \CFA scheduler uses a single ready queue for all processors, which is the simplest approach to scheduling. Further discussion on scheduling is present in section \label{futur:sched}.
 
 % ======================================================================
@@ -174 +158 @@
 % ======================================================================
 % ======================================================================
-To ease the understanding of monitors, like many other concepts, they are generelly represented graphically. While non-scheduled monitors are simple enough for a graphical representation to be useful, internal scheduling is complex enough to justify a visual representation. The following figure is the traditionnal illustration of a monitor :
-
+The following figure is the traditional illustration of a monitor (repeated from page~\pageref{fig:monitor} for convenience) :
+
+\begin{figure}[H]
 \begin{center}
 {\resizebox{0.4\textwidth}{!}{\input{monitor}}}
 \end{center}
-
-This picture has several components, the two most important being the entry-queue and the AS-stack. The entry-queue is a (almost) FIFO list where threads waiting to enter are parked, while the AS-stack is a FILO list used for threads that have been signaled or otherwise marked as running next. For \CFA, the previous picture does not have support for blocking multiple monitors on a single condition. To support \gls{bulk-acq} two changes to this picture are required. First, it doesn't make sense to tie the condition to a single monitor since blocking two monitors as one would require arbitrarily picking a monitor to hold the condition. Secondly, the object waiting on the conditions and AS-stack cannot simply contain the waiting thread since a single thread can potentially wait on multiple monitors. As mentionned in section \ref{intsched}, the handling in multiple monitors is done by partially passing, which entails that each concerned monitor needs to have a node object. However, for waiting on the condition, since all threads need to wait together, a single object needs to be queued in the condition. Moving out the condition and updating the node types yields :
-
+\caption{Traditional illustration of a monitor}
+\label{fig:monitor}
+\end{figure}
+
+This picture has several components, the two most important being the entry-queue and the AS-stack. The entry-queue is an (almost) FIFO list where threads waiting to enter are parked, while the acceptor-signalor (AS) stack is a FILO list used for threads that have been signalled or otherwise marked as running next.
+
+For \CFA, this picture does not have support for blocking multiple monitors on a single condition. To support \gls{bulk-acq} two changes to this picture are required. First, it is non longer helpful to attach the condition to a single monitor. Secondly, the thread waiting on the conditions has to be seperated multiple monitors, which yields :
+
+\begin{figure}[H]
 \begin{center}
 {\resizebox{0.8\textwidth}{!}{\input{int_monitor}}}
 \end{center}
-
-This picture and the proper entry and leave algorithms is the fundamental implementation of internal scheduling (see listing \ref{lst:entry2}).
+\caption{Illustration of \CFA monitor}
+\label{fig:monitor_cfa}
+\end{figure}
+
+This picture and the proper entry and leave algorithms is the fundamental implementation of internal scheduling (see listing \ref{lst:entry2}). Note that when threads are moved from the condition to the AS-stack, it splits the thread into to pieces. The thread is woken up when all the pieces have moved from the AS-stacks to the active thread seat. In this picture, the threads are split into halves but this is only because there are two monitors in this picture. For a specific signaling operation every monitor needs a piece of thread on its AS-stack.
 
 \begin{figure}[b]
@@ -219 +213 @@
 \end{figure}
 
-Some important things to notice about the exit routine. The solution discussed in \ref{intsched} can be seen in the exit routine of listing \ref{lst:entry2}. Basically, the solution boils down to having a seperate data structure for the condition queue and the AS-stack, and unconditionally transferring ownership of the monitors but only unblocking the thread when the last monitor has transferred ownership. This solution is deadlock safe as well as preventing any potential barging.
-
-The data structure used for the AS-stack are reused extensively for external scheduling, but in the case of internal scheduling, the data is allocated using variable-length arrays on the callstack of the \code{wait} and \code{signal_block} routines.
+Some important things to notice about the exit routine. The solution discussed in \ref{intsched} can be seen in the exit routine of listing \ref{lst:entry2}. Basically, the solution boils down to having a seperate data structure for the condition queue and the AS-stack, and unconditionally transferring ownership of the monitors but only unblocking the thread when the last monitor has transferred ownership. This solution is deadlock safe as well as preventing any potential barging. The data structure used for the AS-stack are reused extensively for external scheduling, but in the case of internal scheduling, the data is allocated using variable-length arrays on the callstack of the \code{wait} and \code{signal_block} routines.
+
+\begin{figure}[H]
+\begin{center}
+{\resizebox{0.8\textwidth}{!}{\input{monitor_structs.pstex_t}}}
+\end{center}
+\caption{Data structures involved in internal/external scheduling}
+\label{fig:structs}
+\end{figure}
+
+Figure \ref{fig:structs} shows a high level representation of these data-structures. The main idea behind them is that, while figure \ref{fig:monitor_cfa} is a nice illustration in theory, in practice breaking a threads into multiple pieces to put unto intrusive stacks does not make sense. The \code{condition node} is the data structure that is queued into a condition variable and, when signaled, the condition queue is popped and each \code{condition criterion} are moved to the AS-stack. Once all the criterion have be popped from their respective AS-stacks, the thread is woken-up, which is what is shown in listing \ref{lst:entry2}.
 
 % ======================================================================
@@ -228 +230 @@
 % ======================================================================
 % ======================================================================
-Similarly to internal scheduling, external scheduling for multiple monitors relies on the idea that entry-queues are no longer specific to a single monitor, as mentionned in section \ref{extsched}. This means that some kind of entry-queues must be used that is aware of both monitors and which holds threads that are currently waiting to enter the critical section. This challenge is solved for internal scheduling by having the entry-queues in conditions no longer be tied to a monitor, effectively allowing conditions to be moved outside of monitors. However, in the case of external scheduling, acceptable routines must be aware of the entry queues, which means they must be stored inside at least one of the monitors that will be acquired. This in turn adds the requirement that a systematic algorithm of disambiguating which monitor holds the relevant queue regardless of user ordering. The proposed algorithm is to fall back on monitor lock ordering and specify that the monitor that is acquired first is the one with the relevant entry queue. This assumes that the lock acquiring order is static for the lifetime of all concerned objects but that is a reasonable constraint.
-
-This algorithm choice has two consequences, the entry queue of the highest priority monitor is no longer a true FIFO queue and the queue of the lowest priority monitor is both required and probably unused. The queue can no longer be a FIFO queue because instead of simply containing the waiting threads in order of arrival, they also contain a set of monitors. Therefore, another thread whos set contains the same highest priority monitor but different lower priority monitors may arrive first but enter the critical section after a thread with the correct pairing. Secondly, since it is not known at compile time which monitor will be the lowest priority monitor, every monitor needs to have the correct queues even though it is probable that some queues will go unused for the entire duration of the program, for example if a monitor is only used in a pair.
+Similarly to internal scheduling, external scheduling for multiple monitors relies on the idea that waiting-thread queues are no longer specific to a single monitor, as mentionned in section \ref{extsched}. For internal scheduling, these queues are part of condition variables which are still unique for a given scheduling operation (e.g., no single statment uses multiple conditions). However, in the case of external scheduling, there is no equivalent object which is associated with \code{waitfor} statements. This absence means the queues holding the waiting threads must be stored inside at least one of the monitors that is acquired. The monitors being the only objects that have sufficient lifetime and are available on both sides of the \code{waitfor} statment. This requires an algorithm to choose which monitor holds the relevant queue. It is also important that said algorithm be independent of the order in which users list parameters. The proposed algorithm is to fall back on monitor lock ordering and specify that the monitor that is acquired first is the one with the relevant wainting queue. This assumes that the lock acquiring order is static for the lifetime of all concerned objects but that is a reasonable constraint.
+
+This algorithm choice has two consequences :
+\begin{itemize}
+        \item The queue of the highest priority monitor is no longer a true FIFO queue because threads can be moved to the front of the queue. These queues need to contain a set of monitors for each of the waiting threads. Therefore, another thread whose set contains the same highest priority monitor but different lower priority monitors may arrive first but enter the critical section after a thread with the correct pairing.
+        \item The queue of the lowest priority monitor is both required and potentially unused. Indeed, since it is not known at compile time which monitor will be the lowest priority monitor, every monitor needs to have the correct queues even though it is possible that some queues will go unused for the entire duration of the program, for example if a monitor is only used in a specific pair.
+\end{itemize}
 
 Therefore, the following modifications need to be made to support external scheduling :
 \begin{itemize}
-        \item The threads waiting on the entry-queue need to keep track of which routine is trying to enter, and using which set of monitors. The \code{mutex} routine already has all the required information on it's stack so the thread only needs to keep a pointer to that information.
+        \item The threads waiting on the entry-queue need to keep track of which routine is trying to enter, and using which set of monitors. The \code{mutex} routine already has all the required information on its stack so the thread only needs to keep a pointer to that information.
         \item The monitors need to keep a mask of acceptable routines. This mask contains for each acceptable routine, a routine pointer and an array of monitors to go with it. It also needs storage to keep track of which routine was accepted. Since this information is not specific to any monitor, the monitors actually contain a pointer to an integer on the stack of the waiting thread. Note that the complete mask can be pushed to any owned monitors, regardless of \code{when} statements, the \code{waitfor} statement is used in a context where the thread already has full ownership of (at least) every concerned monitor and therefore monitors will refuse all calls no matter what.
         \item The entry/exit routine need to be updated as shown in listing \ref{lst:entry3}.
 \end{itemize}
 
+\subsection{External scheduling - destructors}
 Finally, to support the ordering inversion of destructors, the code generation needs to be modified to use a special entry routine. This routine is needed because of the storage requirements of the call order inversion. Indeed, when waiting for the destructors, storage is need for the waiting context and the lifetime of said storage needs to outlive the waiting operation it is needed for. For regular \code{waitfor} statements, the callstack of the routine itself matches this requirement but it is no longer the case when waiting for the destructor since it is pushed on to the AS-stack for later. The waitfor semantics can then be adjusted correspondingly, as seen in listing \ref{lst:entry-dtor}
 
@@ -250 +257 @@
         continue
 elif matches waitfor mask
-        push waiter to AS-stack
+        push criterions to AS-stack
         continue
 else
@@ -265 +272 @@
                 if all monitors ready
                         wake-up thread
+                endif
+        endif
 
         if entry queue not empty
                 wake-up thread
+        endif
 \end{pseudo}
 \end{multicols}
@@ -295 +305 @@
 Waitfor
 \begin{pseudo}
-lock all monitors
 if matching thread is already there
         if found destructor
@@ -303 +312 @@
                 push self to AS-stack
                 baton pass
+        endif
         return
-
+endif
 if non-blocking
         Unlock all monitors
         Return
+endif
 
 push self to AS-stack

doc/proposals/concurrency/text/parallelism.tex

@@ -15 +15 @@
 Examples of languages that support \glspl{uthread} are Erlang~\cite{Erlang} and \uC~\cite{uC++book}.
 
-\subsection{Fibers : user-level threads without preemption}
+\subsection{Fibers : user-level threads without preemption} \label{fibers}
 A popular varient of \glspl{uthread} is what is often refered to as \glspl{fiber}. However, \glspl{fiber} do not present meaningful semantical differences with \glspl{uthread}. The significant difference between \glspl{uthread} and \glspl{fiber} is the lack of \gls{preemption} in the later one. Advocates of \glspl{fiber} list their high performance and ease of implementation as majors strenghts of \glspl{fiber} but the performance difference between \glspl{uthread} and \glspl{fiber} is controversial, and the ease of implementation, while true, is a weak argument in the context of language design. Therefore this proposal largely ignores fibers.
 
@@ -33 +33 @@
 
 \subsection{Future Work: Machine setup}\label{machine}
-While this was not done in the context of this thesis, another important aspect of clusters is affinity. While many common desktop and laptop PCs have homogeneous CPUs, other devices often have more heteregenous setups. For example, system using \acrshort{numa} configurations may benefit from users being able to tie clusters and/or kernel threads to certains CPU cores. OS support for CPU affinity is now common \cit, which means it is both possible and desirable for \CFA to offer an abstraction mechanism for portable CPU affinity.
+While this was not done in the context of this thesis, another important aspect of clusters is affinity. While many common desktop and laptop PCs have homogeneous CPUs, other devices often have more heteregenous setups. For example, system using \acrshort{numa} configurations may benefit from users being able to tie clusters and\/or kernel threads to certains CPU cores. OS support for CPU affinity is now common \cite{affinityLinux, affinityWindows, affinityFreebsd, affinityNetbsd, affinityMacosx} which means it is both possible and desirable for \CFA to offer an abstraction mechanism for portable CPU affinity.
 
-\subsection{Paradigms}\label{cfaparadigms}
-Given these building blocks, it is possible to reproduce all three of the popular paradigms. Indeed, \glspl{uthread} is the default paradigm in \CFA. However, disabling \gls{preemption} on the \gls{cfacluster} means \glspl{cfathread} effectively become \glspl{fiber}. Since several \glspl{cfacluster} with different scheduling policy can coexist in the same application, this allows \glspl{fiber} and \glspl{uthread} to coexist in the runtime of an application. Finally, it is possible to build executors for thread pools from \glspl{uthread} or \glspl{fiber}.
+% \subsection{Paradigms}\label{cfaparadigms}
+% Given these building blocks, it is possible to reproduce all three of the popular paradigms. Indeed, \glspl{uthread} is the default paradigm in \CFA. However, disabling \gls{preemption} on the \gls{cfacluster} means \glspl{cfathread} effectively become \glspl{fiber}. Since several \glspl{cfacluster} with different scheduling policy can coexist in the same application, this allows \glspl{fiber} and \glspl{uthread} to coexist in the runtime of an application. Finally, it is possible to build executors for thread pools from \glspl{uthread} or \glspl{fiber}.

doc/proposals/concurrency/text/results.tex

@@ -1 +1 @@
 % ======================================================================
 % ======================================================================
-\chapter{Performance results}
+\chapter{Performance results} \label{results}
 % ======================================================================
 % ======================================================================
-
 \section{Machine setup}
-
-\begin{figure}
+Table \ref{tab:machine} shows the characteristiques of the machine used to run the benchmarks. All tests where made on this machine.
+\begin{figure}[H]
 \begin{center}
 \begin{tabular}{| l | r | l | r |}
@@ -37 +36 @@
 
 \section{Micro benchmarks}
+All benchmarks are run using the same harness to produce the results, seen as the \code{BENCH()} macro in the following examples. This macro uses the following logic to benchmark the code :
+\begin{pseudo}
+#define BENCH(run, result)
+        gettime();
+        run;
+        gettime();
+        result = (after - before) / N;
+\end{pseudo}
+The method used to get time is \code{clock_gettime(CLOCK_THREAD_CPUTIME_ID);}. Each benchmark is using many interations of a simple call to measure the cost of the call. The specific number of interation dependes on the specific benchmark.
+
+\subsection{Context-switching}
+The first interesting benchmark is to measure how long context-switches take. The simplest approach to do this is to yield on a thread, which executes a 2-step context switch. In order to make the comparison fair, coroutines also execute a 2-step context-switch, which is a resume/suspend cycle instead of a yield. Listing \ref{lst:ctx-switch} shows the code for coroutines and threads. All omitted tests are functionally identical to one of these tests. The results can be shown in table \ref{tab:ctx-switch}.
+\begin{figure}
+\begin{multicols}{2}
+\CFA Coroutines
+\begin{cfacode}
+coroutine GreatSuspender {};
+void main(GreatSuspender& this) {
+        while(true) { suspend(); }
+}
+int main() {
+        GreatSuspender s;
+        resume(s);
+        BENCH(
+                for(size_t i=0; i<n; i++) {
+                        resume(s);
+                },
+                result
+        )
+        printf("%llu\n", result);
+}
+\end{cfacode}
+\columnbreak
+\CFA Threads
+\begin{cfacode}
+
+
+
+
+int main() {
+
+
+        BENCH(
+                for(size_t i=0; i<n; i++) {
+                        yield();
+                },
+                result
+        )
+        printf("%llu\n", result);
+}
+\end{cfacode}
+\end{multicols}
+\caption{\CFA benchmark code used to measure context-switches for coroutines and threads.}
+\label{lst:ctx-switch}
+\end{figure}
 
 \begin{figure}
@@ -54 +108 @@
 \caption{Context Switch comparaison. All numbers are in nanoseconds(\si{\nano\second})}
 \label{tab:ctx-switch}
+\end{figure}
+
+\subsection{Mutual-exclusion}
+The next interesting benchmark is to measure the overhead to enter/leave a critical-section. For monitors, the simplest appraoch is to measure how long it takes enter and leave a monitor routine. Listing \ref{lst:mutex} shows the code for \CFA. To put the results in context, the cost of entering a non-inline function and the cost of acquiring and releasing a pthread mutex lock are also mesured. The results can be shown in table \ref{tab:mutex}.
+
+\begin{figure}
+\begin{cfacode}
+monitor M {};
+void __attribute__((noinline)) call( M & mutex m /*, m2, m3, m4*/ ) {}
+
+int main() {
+        M m/*, m2, m3, m4*/;
+        BENCH(
+                for(size_t i=0; i<n; i++) {
+                        call(m/*, m2, m3, m4*/);
+                },
+                result
+        )
+        printf("%llu\n", result);
+}
+\end{cfacode}
+\caption{\CFA benchmark code used to measure mutex routines.}
+\label{lst:mutex}
 \end{figure}
 
@@ -75 +152 @@
 \end{figure}
 
+\subsection{Internal scheduling}
+The Internal scheduling benchmark measures the cost of waiting on and signaling a condition variable. Listing \ref{lst:int-sched} shows the code for \CFA. The results can be shown in table \ref{tab:int-sched}. As with all other benchmarks, all omitted tests are functionally identical to one of these tests.
+
+\begin{figure}
+\begin{cfacode}
+volatile int go = 0;
+condition c;
+monitor M {};
+M m1;
+
+void __attribute__((noinline)) do_call( M & mutex a1 ) { signal(c); }
+
+thread T {};
+void ^?{}( T & mutex this ) {}
+void main( T & this ) {
+        while(go == 0) { yield(); }
+        while(go == 1) { do_call(m1); }
+}
+int  __attribute__((noinline)) do_wait( M & mutex a1 ) {
+        go = 1;
+        BENCH(
+                for(size_t i=0; i<n; i++) {
+                        wait(c);
+                },
+                result
+        )
+        printf("%llu\n", result);
+        go = 0;
+        return 0;
+}
+int main() {
+        T t;
+        return do_wait(m1);
+}
+\end{cfacode}
+\caption{Benchmark code for internal scheduling}
+\label{lst:int-sched}
+\end{figure}
+
 \begin{figure}
 \begin{center}
@@ -92 +208 @@
 \end{figure}
 
+\subsection{External scheduling}
+The Internal scheduling benchmark measures the cost of the \code{waitfor} statement (\code{_Accept} in \uC). Listing \ref{lst:ext-sched} shows the code for \CFA. The results can be shown in table \ref{tab:ext-sched}. As with all other benchmarks, all omitted tests are functionally identical to one of these tests.
+
+\begin{figure}
+\begin{cfacode}
+volatile int go = 0;
+monitor M {};
+M m1;
+thread T {};
+
+void __attribute__((noinline)) do_call( M & mutex a1 ) {}
+
+void ^?{}( T & mutex this ) {}
+void main( T & this ) {
+        while(go == 0) { yield(); }
+        while(go == 1) { do_call(m1); }
+}
+int  __attribute__((noinline)) do_wait( M & mutex a1 ) {
+        go = 1;
+        BENCH(
+                for(size_t i=0; i<n; i++) {
+                        waitfor(call, a1);
+                },
+                result
+        )
+        printf("%llu\n", result);
+        go = 0;
+        return 0;
+}
+int main() {
+        T t;
+        return do_wait(m1);
+}
+\end{cfacode}
+\caption{Benchmark code for external scheduling}
+\label{lst:ext-sched}
+\end{figure}
+
 \begin{figure}
 \begin{center}
@@ -109 +263 @@
 \end{figure}
 
-\begin{figure}
-\begin{center}
-\begin{tabular}{| l | S[table-format=5.2,table-number-alignment=right] | S[table-format=5.2,table-number-alignment=right] | S[table-format=5.2,table-number-alignment=right] |}
-\cline{2-4}
-\multicolumn{1}{c |}{} & \multicolumn{1}{c |}{ Median } &\multicolumn{1}{c |}{ Average } & \multicolumn{1}{c |}{ Standard Deviation} \\
-\hline
-Pthreads                & 26974.5       & 26977 & 124.12 \\
-\CFA Coroutines & 5             & 5             & 0      \\
-\CFA Threads    & 1122.5        & 1109.86       & 36.54  \\
-\uC Coroutines  & 106           & 107.04        & 1.61   \\
-\uC Threads             & 525.5 & 533.04        & 11.14  \\
+\subsection{Object creation}
+Finaly, the last benchmark measured is the cost of creation for concurrent objects. Listing \ref{lst:creation} shows the code for pthreads and \CFA threads. The results can be shown in table \ref{tab:creation}. As with all other benchmarks, all omitted tests are functionally identical to one of these tests. The only note here is that the callstacks of \CFA coroutines are lazily created, therefore without priming the coroutine, the creation cost is very low.
+
+\begin{figure}
+\begin{multicols}{2}
+pthread
+\begin{cfacode}
+int main() {
+        BENCH(
+                for(size_t i=0; i<n; i++) {
+                        pthread_t thread;
+                        if(pthread_create(
+                                &thread,
+                                NULL,
+                                foo,
+                                NULL
+                        ) < 0) {
+                                perror( "failure" );
+                                return 1;
+                        }
+
+                        if(pthread_join(
+                                thread,
+                                NULL
+                        ) < 0) {
+                                perror( "failure" );
+                                return 1;
+                        }
+                },
+                result
+        )
+        printf("%llu\n", result);
+}
+\end{cfacode}
+\columnbreak
+\CFA Threads
+\begin{cfacode}
+int main() {
+        BENCH(
+                for(size_t i=0; i<n; i++) {
+                        MyThread m;
+                },
+                result
+        )
+
+        printf("%llu\n", result);
+}
+\end{cfacode}
+\end{multicols}
+\caption{Bechmark code for pthreads and \CFA to measure object creation}
+\label{lst:creation}
+\end{figure}
+
+\begin{figure}
+\begin{center}
+\begin{tabular}{| l | S[table-format=5.2,table-number-alignment=right] | S[table-format=5.2,table-number-alignment=right] | S[table-format=5.2,table-number-alignment=right] |}
+\cline{2-4}
+\multicolumn{1}{c |}{} & \multicolumn{1}{c |}{ Median } &\multicolumn{1}{c |}{ Average } & \multicolumn{1}{c |}{ Standard Deviation} \\
+\hline
+Pthreads                        & 26974.5       & 26977 & 124.12 \\
+\CFA Coroutines Lazy    & 5             & 5             & 0      \\
+\CFA Coroutines Eager   & 335.0 & 357.67        & 34.2   \\
+\CFA Threads            & 1122.5        & 1109.86       & 36.54  \\
+\uC Coroutines          & 106           & 107.04        & 1.61   \\
+\uC Threads                     & 525.5 & 533.04        & 11.14  \\
 \hline
 \end{tabular}

doc/proposals/concurrency/text/together.tex

@@ -7 +7 @@
 
 \section{Threads as monitors}
-As it was subtely alluded in section \ref{threads}, \code{threads} in \CFA are in fact monitors. This means that all the monitors features are available when using threads. For example, here is a very simple two thread pipeline that could be used for a simulator of a game engine :
+As it was subtely alluded in section \ref{threads}, \code{threads} in \CFA are in fact monitors, which means that all monitor features are available when using threads. For example, here is a very simple two thread pipeline that could be used for a simulator of a game engine :
 \begin{cfacode}
 // Visualization declaration
@@ -72 +72 @@
         }
 }
+
+// Call destructor for simulator once simulator finishes
+// Call destructor for renderer to signify shutdown
 \end{cfacode}
 
 \section{Fibers \& Threads}
+As mentionned in section \ref{preemption}, \CFA uses preemptive threads by default but can use fibers on demand. Currently, using fibers is done by adding the following line of code to the program~:
+\begin{cfacode}
+unsigned int default_preemption() {
+        return 0;
+}
+\end{cfacode}
+This function is called by the kernel to fetch the default preemption rate, where 0 signifies an infinite time-slice i.e. no preemption. However, once clusters are fully implemented, it will be possible to create fibers and uthreads in on the same system :
+\begin{figure}
+\begin{cfacode}
+//Cluster forward declaration
+struct cluster;
+
+//Processor forward declaration
+struct processor;
+
+//Construct clusters with a preemption rate
+void ?{}(cluster& this, unsigned int rate);
+//Construct processor and add it to cluster
+void ?{}(processor& this, cluster& cluster);
+//Construct thread and schedule it on cluster
+void ?{}(thread& this, cluster& cluster);
+
+//Declare two clusters
+cluster thread_cluster = { 10`ms };                     //Preempt every 10 ms
+cluster fibers_cluster = { 0 };                         //Never preempt
+
+//Construct 4 processors
+processor processors[4] = {
+        //2 for the thread cluster
+        thread_cluster;
+        thread_cluster;
+        //2 for the fibers cluster
+        fibers_cluster;
+        fibers_cluster;
+};
+
+//Declares thread
+thread UThread {};
+void ?{}(UThread& this) {
+        //Construct underlying thread to automatically
+        //be scheduled on the thread cluster
+        (this){ thread_cluster }
+}
+
+void main(UThread & this);
+
+//Declares fibers
+thread Fiber {};
+void ?{}(Fiber& this) {
+        //Construct underlying thread to automatically
+        //be scheduled on the fiber cluster
+        (this.__thread){ fibers_cluster }
+}
+
+void main(Fiber & this);
+\end{cfacode}
+\end{figure}

doc/proposals/concurrency/version

@@ -1 @@
-0.11.47
+0.11.129