Changeset 27dde72 for doc/proposals


Ignore:
Timestamp:
May 29, 2017, 1:39:37 PM (4 years ago)
Author:
Thierry Delisle <tdelisle@…>
Branches:
aaron-thesis, arm-eh, cleanup-dtors, deferred_resn, demangler, jacob/cs343-translation, jenkins-sandbox, master, new-ast, new-ast-unique-expr, new-env, no_list, persistent-indexer, resolv-new, with_gc
Children:
ff98952
Parents:
eb182b0
Message:

Major update to the concurrency proposal to be based on multiple files

Location:
doc/proposals/concurrency
Files:
3 added
1 deleted
8 edited

Legend:

Unmodified
Added
Removed
  • doc/proposals/concurrency/.gitignore

    reb182b0 r27dde72  
    1 concurrency.aux
    2 concurrency.acn
    3 concurrency.acr
    4 concurrency.alg
    5 concurrency.bbl
    6 concurrency.blg
    7 concurrency.brf
    8 concurrency.dvi
    9 concurrency.glg
    10 concurrency.glo
    11 concurrency.gls
    12 concurrency.idx
    13 concurrency.ind
    14 concurrency.ist
    15 concurrency.log
    16 concurrency.out
    17 concurrency.pdf
    18 concurrency.ps
    19 version.aux
    20 monitor.tex
    21 ext_monitor.tex
     1build/*.aux
     2build/*.acn
     3build/*.acr
     4build/*.alg
     5build/*.bbl
     6build/*.blg
     7build/*.brf
     8build/*.dvi
     9build/*.glg
     10build/*.glo
     11build/*.gls
     12build/*.idx
     13build/*.ind
     14build/*.ist
     15build/*.log
     16build/*.out
     17build/*.ps
     18build/*.tex
     19build/*.toc
     20*.pdf
  • doc/proposals/concurrency/Makefile

    reb182b0 r27dde72  
    11## Define the appropriate configuration variables.
    22
    3 TeXLIB = .:../../LaTeXmacros:../../LaTeXmacros/listings:../../LaTeXmacros/enumitem:~/bibliographies:
    4 LaTeX  = TEXINPUTS=${TeXLIB} && export TEXINPUTS && latex -halt-on-error
    5 BibTeX = BIBINPUTS=${TeXLIB} && export BIBINPUTS && bibtex
     3TeXLIB = .:./style:./text:./annex:./build:../../LaTeXmacros:../../LaTeXmacros/listings:../../LaTeXmacros/enumitem:~/bibliographies:
     4LaTeX  = TEXINPUTS=${TeXLIB} && export TEXINPUTS && latex -halt-on-error -output-directory=build -interaction=nonstopmode
     5BibTeX = BIBINPUTS=${TeXLIB} && export BIBINPUTS && bibtex -terse
    66
    77## Define the text source files.
     
    99SOURCES = ${addsuffix .tex, \
    1010thesis \
    11 style \
    12 cfa-format \
    13 glossary \
     11style/style \
     12style/cfa-format \
     13annex/glossary \
     14text/intro \
     15text/basics \
     16text/concurrency \
     17text/parallelism \
    1418}
    1519
    16 FIGURES = ${addsuffix .tex, \
     20FIGURES = ${addprefix build/, ${addsuffix .tex, \
    1721        monitor \
    1822        ext_monitor \
    19 }
     23}}
    2024
    2125PICTURES = ${addsuffix .pstex, \
     
    3741
    3842clean :
    39         rm -f *.bbl *.aux *.dvi *.idx *.ilg *.ind *.brf *.out *.log *.toc *.blg *.pstex_t *.cf *.glg *.glo *.gls *.ist *.acn *.acr *.alg \
    40                 ${FIGURES} ${PICTURES} ${PROGRAMS} ${GRAPHS} ${basename ${DOCUMENT}}.ps ${DOCUMENT}
     43        @rm -fv ${DOCUMENT} \
     44        build/*.acn     \
     45        build/*.acr     \
     46        build/*.alg     \
     47        build/*.aux     \
     48        build/*.bbl     \
     49        build/*.blg     \
     50        build/*.brf     \
     51        build/*.cf      \
     52        build/*.dvi     \
     53        build/*.glg     \
     54        build/*.glo     \
     55        build/*.gls     \
     56        build/*.ist     \
     57        build/*.idx     \
     58        build/*.ilg     \
     59        build/*.ind     \
     60        build/*.log     \
     61        build/*.out     \
     62        build/*.ps      \
     63        build/*.pstex_t \
     64        build/*.tex     \
     65        build/*.toc     \
     66               
    4167
    4268# File Dependencies #
     
    4571        ps2pdf $<
    4672
    47 build/${basename ${DOCUMENT}}.ps : ${basename ${DOCUMENT}}.dvi
     73build/${basename ${DOCUMENT}}.ps : build/${basename ${DOCUMENT}}.dvi
    4874        dvips $< -o $@
    4975
    50 build/${basename ${DOCUMENT}}.dvi : Makefile ${GRAPHS} ${PROGRAMS} ${PICTURES} ${FIGURES} ${SOURCES} ${basename ${DOCUMENT}}.tex \
    51                 ../../LaTeXmacros/common.tex ../../LaTeXmacros/indexstyle
    52         # Conditionally create the build folder
    53         if [ ! -r build ] ; then mkdir build ; fi
    54         # # Conditionally create an empty *.ind (index) file for inclusion until makeindex is run.
    55         # if [ ! -r ${basename $@}.ind ] ; then touch ${basename $@}.ind ; fi
    56         # # Must have *.aux file containing citations for bibtex
    57         # if [ ! -r ${basename $@}.aux ] ; then ${LaTeX} ${basename $@}.tex ; fi
    58         # -${BibTeX} ${basename $@}
    59         # # Some citations reference others so run steps again to resolve these citations
    60         # ${LaTeX} ${basename $@}.tex
    61         # -${BibTeX} ${basename $@}
    62         # # Make index from *.aux entries and input index at end of document
    63         # makeglossaries ${basename $@}
    64         # #${LaTeX} ${basename $@}.tex
    65         # # Run again to get index title into table of contents
    66         # ${LaTeX} ${basename $@}.tex
    67         # -./bump_ver.sh
    68         # ${LaTeX} ${basename $@}.tex
     76build/${basename ${DOCUMENT}}.dvi : Makefile ${GRAPHS} ${PROGRAMS} ${PICTURES} ${FIGURES} ${SOURCES} ${basename ${DOCUMENT}}.tex ../../LaTeXmacros/common.tex ../../LaTeXmacros/indexstyle
     77
     78        @ if [ ! -r ${basename $@}.ind ] ; then touch ${basename $@}.ind ; fi                           # Conditionally create an empty *.ind (index) file for inclusion until makeindex is run.
     79        @ echo "Citation lookup"                                                                                        # Must have *.aux file containing citations for bibtex
     80        @ if [ ! -r ${basename $@}.aux ] ; then ${LaTeX} ${basename ${notdir $@}}.tex ; fi
     81        @ echo "Citation Pass 1"
     82        @ -${BibTeX} ${basename $@}                                                                                     # Some citations reference others so run steps again to resolve these citations
     83        @ echo "Citation Pass 2"       
     84        @ ${LaTeX} ${basename ${notdir $@}}.tex
     85        @ -${BibTeX} ${basename $@}
     86        @ echo "Glossary"       
     87        makeglossaries -q -s ${basename $@}.ist ${basename $@}                                          # Make index from *.aux entries and input index at end of document
     88        @ echo ".dvi generation"       
     89        @ -build/bump_ver.sh
     90        @ ${LaTeX} ${basename ${notdir $@}}.tex                                                                 # Run again to get index title into table of contents
    6991
    7092
     
    7496## Define the default recipes.
    7597
    76 build/%.tex : %.fig
     98build/%.tex : figures/%.fig
    7799        fig2dev -L eepic $< > $@
    78100
    79 build/%.ps : %.fig
     101build/%.ps : figures/%.fig
    80102        fig2dev -L ps $< > $@
    81103
    82 build/%.pstex : %.fig
     104build/%.pstex : figures/%.fig
    83105        fig2dev -L pstex $< > $@
    84106        fig2dev -L pstex_t -p $@ $< > $@_t
  • doc/proposals/concurrency/build/bump_ver.sh

    reb182b0 r27dde72  
    11#!/bin/bash
    2 if [ ! -f version ]; then
    3     echo "0.0.0" > version
     2if [ ! -f build/version ]; then
     3    echo "0.0.0" > build/version
    44fi
    55
    6 sed -r 's/([0-9]+\.[0-9]+.)([0-9]+)/echo "\1\$((\2+1))" > version/ge' version > /dev/null
     6sed -r 's/([0-9]+\.[0-9]+.)([0-9]+)/echo "\1\$((\2+1))" > version/ge' build/version > /dev/null
  • doc/proposals/concurrency/build/version

    reb182b0 r27dde72  
    1 0.8.3
     10.9.117
  • doc/proposals/concurrency/style/cfa-format.tex

    reb182b0 r27dde72  
    1 \usepackage{xcolor}
     1\usepackage[usenames,dvipsnames]{xcolor}
    22\usepackage{listings}
    33\usepackage{inconsolata}
     
    144144  % moredelim=** allows cumulative application
    145145}
    146 \lstset{
    147   morekeywords=[2]{nomutex,mutex,thread,wait,wait_release,signal,signal_block,accept,monitor,suspend,resume,coroutine}
    148   language = CFA,
    149   style=defaultStyle
    150 }
     146
    151147\lstMakeShortInline[basewidth=0.5em,breaklines=true,basicstyle=\normalsize\ttfamily\color{basicCol}]@  % single-character for \lstinline
    152148
    153 \lstnewenvironment{cfacode}[1][]{ %
    154   \lstset{ %
    155     language = CFA, %
    156     style=defaultStyle, %
    157     morekeywords=[2]{nomutex,mutex,thread,wait,signal,signal_block,accept,monitor,suspend,resume,coroutine}, %
    158     #1 %
    159   } %
     149\lstnewenvironment{ccode}[1][]{
     150  \lstset{
     151    language = C,
     152    style=defaultStyle,
     153    #1
     154  }
     155}{}
     156
     157\lstnewenvironment{cfacode}[1][]{
     158  \lstset{
     159    language = CFA,
     160    style=defaultStyle,
     161    #1
     162  }
    160163}{}
    161164
     
    169172
    170173\lstnewenvironment{cppcode}[1][]{
     174  \lstset{
     175    language = c++,
     176    style=defaultStyle,
     177    #1
     178  }
     179}{}
     180
     181\lstnewenvironment{ucppcode}[1][]{
    171182  \lstset{
    172183    language = c++,
     
    219230\newcommand{\one}{\lstinline{one_t}\xspace}
    220231\newcommand{\ateq}{\lstinline{\@=}\xspace}
     232\newcommand{\code}[1]{\lstinline[language=CFA,style=defaultStyle]{#1}}
     233\newcommand{\pscode}[1]{\lstinline[language=pseudo,style=pseudoStyle]{#1}}
  • doc/proposals/concurrency/style/style.tex

    reb182b0 r27dde72  
    44% \CFADefaultStyle
    55
    6 \lstset{
    7 morekeywords=[2]{nomutex,mutex,thread,wait,wait_release,signal,signal_block,accept,monitor,suspend,resume,coroutine},
    8 keywordstyle=[2]\color{blue},                           % second set of keywords for concurency
    9 basicstyle=\linespread{0.9}\tt\small,           % reduce line spacing and use typewriter font
    10 stringstyle=\sf\color{Mahogany},                        % use sanserif font
    11 commentstyle=\itshape\color{OliveGreen},                % green and italic comments
    12 }%
     6% \lstset{
     7% morekeywords=[2]{nomutex,mutex,thread,wait,wait_release,signal,signal_block,accept,monitor,suspend,resume,coroutine},
     8% keywordstyle=[2]\color{blue},                         % second set of keywords for concurency
     9% basicstyle=\linespread{0.9}\tt\small,         % reduce line spacing and use typewriter font
     10% stringstyle=\sf\color{Mahogany},                      % use sanserif font
     11% commentstyle=\itshape\color{OliveGreen},              % green and italic comments
     12% }%
  • doc/proposals/concurrency/text/intro.tex

    reb182b0 r27dde72  
    1 % ### #     # ####### ######  #######
    2 %  #  ##    #    #    #     # #     #
    3 %  #  # #   #    #    #     # #     #
    4 %  #  #  #  #    #    ######  #     #
    5 %  #  #   # #    #    #   #   #     #
    6 %  #  #    ##    #    #    #  #     #
    7 % ### #     #    #    #     # #######
     1% ======================================================================
     2\chapter{Introduction}
     3% ======================================================================
    84
    9 \chapter{Introduction}
    10 This proposal provides a minimal core concurrency API that is both simple, efficient and can be reused to build higher-level features. The simplest possible concurrency core is a thread and a lock but this low-level approach is hard to master. An easier approach for users is to support higher-level constructs as the basis of the concurrency in \CFA. Indeed, for highly productive parallel programming, high-level approaches are much more popular~\cite{HPP:Study}. Examples are task based, message passing and implicit threading.
     5This proposal provides a minimal concurrency API that is simple, efficient and can be reused to build higher-level features. The simplest possible concurrency core is a thread and a lock but this low-level approach is hard to master. An easier approach for users is to support higher-level constructs as the basis of the concurrency in \CFA. Indeed, for highly productive parallel programming, high-level approaches are much more popular~\cite{HPP:Study}. Examples are task based, message passing and implicit threading.
    116
    12 There are actually two problems that need to be solved in the design of the concurrency for a programming language: which concurrency tools are available to the users and which parallelism tools are available. While these two concepts are often seen together, they are in fact distinct concepts that require different sorts of tools~\cite{Buhr05a}. Concurrency tools need to handle mutual exclusion and synchronization, while parallelism tools are more about performance, cost and resource utilization.
     7There are actually two problems that need to be solved in the design of concurrency for a programming language: which concurrency tools are available to the users and which parallelism tools are available. While these two concepts are often seen together, they are in fact distinct concepts that require different sorts of tools~\cite{Buhr05a}. Concurrency tools need to handle mutual exclusion and synchronization, while parallelism tools are about performance, cost and resource utilization.
  • doc/proposals/concurrency/thesis.tex

    reb182b0 r27dde72  
    2020\usepackage{epic,eepic}
    2121\usepackage{upquote}                                            % switch curled `'" to straight
     22\usepackage{dirtytalk}
    2223\usepackage{calc}
    2324\usepackage{xspace}
     
    6162\newcommand{\uC}{$\mu$\CC}
    6263\newcommand{\cit}{\textsuperscript{[Citation Needed]}\xspace}
    63 \newcommand{\code}[1]{\lstinline[language=CFA]{#1}}
    64 \newcommand{\pscode}[1]{\lstinline[language=pseudo]{#1}}
    6564\newcommand{\TODO}{{\Textbf{TODO}}}
    6665
     
    7170%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    7271
    73 \setcounter{secnumdepth}{3}                             % number subsubsections
    74 \setcounter{tocdepth}{3}                                % subsubsections in table of contents
    75 % \linenumbers                                            % comment out to turn off line numbering
     72\setcounter{secnumdepth}{3}                           % number subsubsections
     73\setcounter{tocdepth}{3}                              % subsubsections in table of contents
     74% \linenumbers                                          % comment out to turn off line numbering
    7675\makeindex
    7776\pagestyle{fancy}
    7877\fancyhf{}
    7978\cfoot{\thepage}
    80 \rfoot{v\input{version}}
     79\rfoot{v\input{build/version}}
    8180
    8281%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     
    9291\maketitle
    9392
    94 % ### #     # ####### ######  #######
    95 %  #  ##    #    #    #     # #     #
    96 %  #  # #   #    #    #     # #     #
    97 %  #  #  #  #    #    ######  #     #
    98 %  #  #   # #    #    #   #   #     #
    99 %  #  #    ##    #    #    #  #     #
    100 % ### #     #    #    #     # #######
     93\tableofcontents
    10194
    102 \chapter{Introduction}
    103 This proposal provides a minimal core concurrency API that is both simple, efficient and can be reused to build higher-level features. The simplest possible concurrency core is a thread and a lock but this low-level approach is hard to master. An easier approach for users is to support higher-level constructs as the basis of the concurrency in \CFA. Indeed, for highly productive parallel programming, high-level approaches are much more popular~\cite{HPP:Study}. Examples are task based, message passing and implicit threading.
     95\input{intro}
    10496
    105 There are actually two problems that need to be solved in the design of the concurrency for a programming language: which concurrency tools are available to the users and which parallelism tools are available. While these two concepts are often seen together, they are in fact distinct concepts that require different sorts of tools~\cite{Buhr05a}. Concurrency tools need to handle mutual exclusion and synchronization, while parallelism tools are more about performance, cost and resource utilization.
     97\input{basics}
    10698
    107 %  #####  ####### #     #  #####  #     # ######  ######  ####### #     #  #####  #     #
    108 % #     # #     # ##    # #     # #     # #     # #     # #       ##    # #     #  #   #
    109 % #       #     # # #   # #       #     # #     # #     # #       # #   # #         # #
    110 % #       #     # #  #  # #       #     # ######  ######  #####   #  #  # #          #
    111 % #       #     # #   # # #       #     # #   #   #   #   #       #   # # #          #
    112 % #     # #     # #    ## #     # #     # #    #  #    #  #       #    ## #     #    #
    113 %  #####  ####### #     #  #####   #####  #     # #     # ####### #     #  #####     #
     99\input{concurrency}
    114100
    115 \chapter{Concurrency}
    116 Several tool can be used to solve concurrency challenges. Since these challenges always appear with the use of mutable shared-state, some languages and libraries simply disallow mutable shared-state (Erlang~\cite{Erlang}, Haskell~\cite{Haskell}, Akka (Scala)~\cite{Akka}). In these paradigms, interaction among concurrent objects relies on message passing~\cite{Thoth,Harmony,V-Kernel} or other paradigms that closely relate to networking concepts (channels\cit for example). However, in languages that use routine calls as their core abstraction mechanism, these approaches force a clear distinction between concurrent and non-concurrent paradigms (i.e., message passing versus routine call). Which in turn means that, in order to be effective, programmers need to learn two sets of designs patterns. This distinction can be hidden away in library code, but effective use of the librairy still has to take both paradigms into account. Approaches based on shared memory are more closely related to non-concurrent paradigms since they often rely on basic constructs like routine calls and objects. At a lower level these can be implemented as locks and atomic operations. Many such mechanisms have been proposed, including semaphores~\cite{Dijkstra68b} and path expressions~\cite{Campbell74}. However, for productivity reasons it is desireable to have a higher-level construct be the core concurrency paradigm~\cite{HPP:Study}. An approach that is worth mentionning because it is gaining in popularity is transactionnal memory~\cite{Dice10}[Check citation]. While this approach is even pursued by system languages like \CC\cit, the performance and feature set is currently too restrictive to add such a paradigm to a language like C or \CC\cit, which is why it was rejected as the core paradigm for concurrency in \CFA. One of the most natural, elegant, and efficient mechanisms for synchronization and communication, especially for shared memory systems, is the \emph{monitor}. Monitors were first proposed by Brinch Hansen~\cite{Hansen73} and later described and extended by C.A.R.~Hoare~\cite{Hoare74}. Many programming languages---e.g., Concurrent Pascal~\cite{ConcurrentPascal}, Mesa~\cite{Mesa}, Modula~\cite{Modula-2}, Turing~\cite{Turing:old}, Modula-3~\cite{Modula-3}, NeWS~\cite{NeWS}, Emerald~\cite{Emerald}, \uC~\cite{Buhr92a} and Java~\cite{Java}---provide monitors as explicit language constructs. In addition, operating-system kernels and device drivers have a monitor-like structure, although they often use lower-level primitives such as semaphores or locks to simulate monitors. For these reasons, this project proposes monitors as the core concurrency construct.
     101\input{parallelism}
    117102
    118 % #     # ####### #     # ### ####### ####### ######   #####
    119 % ##   ## #     # ##    #  #     #    #     # #     # #     #
    120 % # # # # #     # # #   #  #     #    #     # #     # #
    121 % #  #  # #     # #  #  #  #     #    #     # ######   #####
    122 % #     # #     # #   # #  #     #    #     # #   #         #
    123 % #     # #     # #    ##  #     #    #     # #    #  #     #
    124 % #     # ####### #     # ###    #    ####### #     #  #####
    125 
    126 \section{Monitors}
    127 A monitor is a set of routines that ensure mutual exclusion when accessing shared state. This concept is generally associated with Object-Oriented Languages like Java~\cite{Java} or \uC~\cite{uC++book} but does not strictly require OOP semantics. The only requirements is the ability to declare a handle to a shared object and a set of routines that act on it :
    128 \begin{cfacode}
    129         typedef /*some monitor type*/ monitor;
    130         int f(monitor & m);
    131 
    132         int main() {
    133                 monitor m;
    134                 f(m);
    135         }
    136 \end{cfacode}
    137 
    138 %  #####     #    #       #
    139 % #     #   # #   #       #
    140 % #        #   #  #       #
    141 % #       #     # #       #
    142 % #       ####### #       #
    143 % #     # #     # #       #
    144 %  #####  #     # ####### #######
    145 
    146 \subsection{Call semantics} \label{call}
    147 The above monitor example displays some of the intrinsic characteristics. Indeed, it is necessary to use pass-by-reference over pass-by-value for monitor routines. This semantics is important because at their core, monitors are implicit mutual-exclusion objects (locks), and these objects cannot be copied. Therefore, monitors are implicitly non-copyable.
    148 
    149 Another aspect to consider is when a monitor acquires its mutual exclusion. For example, a monitor may need to be passed through multiple helper routines that do not acquire the monitor mutual-exclusion on entry. Pass through can be both generic helper routines (\code{swap}, \code{sort}, etc.) or specific helper routines like the following to implement an atomic counter :
    150 
    151 \begin{cfacode}
    152         monitor counter_t { /*...see section $\ref{data}$...*/ };
    153 
    154         void ?{}(counter_t & nomutex this); //constructor
    155         size_t ++?(counter_t & mutex this); //increment
    156 
    157         //need for mutex is platform dependent here
    158         void ?{}(size_t * this, counter_t & mutex cnt); //conversion
    159 \end{cfacode}
    160 
    161 Here, the constructor(\code{?\{\}}) uses the \code{nomutex} keyword to signify that it does not acquire the monitor mutual exclusion when constructing. This semantics is because an object not yet constructed should never be shared and therefore does not require mutual exclusion. The prefix increment operator uses \code{mutex} to protect the incrementing process from race conditions. Finally, there is a conversion operator from \code{counter_t} to \code{size_t}. This conversion may or may not require the \code{mutex} key word depending on whether or not reading an \code{size_t} is an atomic operation or not.
    162 
    163 Having both \code{mutex} and \code{nomutex} keywords could be argued to be redundant based on the meaning of a routine having neither of these keywords. For example, given a routine without quualifiers \code{void foo(counter_t & this)} then one could argue that it should default to the safest option \code{mutex}. On the other hand, the option of having routine \code{void foo(counter_t & this)} mean \code{nomutex} is unsafe by default and may easily cause subtle errors. It can be argued that \code{nomutex} is the more "normal" behaviour, the \code{nomutex} keyword effectively stating explicitly that "this routine has nothing special". Another alternative is to make having exactly one of these keywords mandatory, which would provide the same semantics but without the ambiguity of supporting routine \code{void foo(counter_t & this)}. Mandatory keywords would also have the added benefice of being self-documented but at the cost of extra typing. However, since \CFA relies heavily on traits as an abstraction mechanism, the distinction between a type that is a monitor and a type that looks like a monitor can become blurred. For this reason, \CFA only has the \code{mutex} keyword.
    164 
    165 
    166 The next semantic decision is to establish when \code{mutex} may be used as a type qualifier. Consider the following declarations:
    167 \begin{cfacode}
    168         int f1(monitor & mutex m);
    169         int f2(const monitor & mutex m);
    170         int f3(monitor ** mutex m);
    171         int f4(monitor *[] mutex m);
    172         int f5(graph(monitor*) & mutex m);
    173 \end{cfacode}
    174 The problem is to indentify which object(s) should be acquired. Furthermore, each object needs to be acquired only once. In the case of simple routines like \code{f1} and \code{f2} it is easy to identify an exhaustive list of objects to acquire on entry. Adding indirections (\code{f3}) still allows the compiler and programmer to indentify which object is acquired. However, adding in arrays (\code{f4}) makes it much harder. Array lengths are not necessarily known in C and even then making sure we only acquire objects once becomes also none trivial. This can be extended to absurd limits like \code{f5}, which uses a graph of monitors. To keep everyone as sane as possible~\cite{Chicken}, this projects imposes the requirement that a routine may only acquire one monitor per parameter and it must be the type of the parameter with one level of indirection (ignoring potential qualifiers). Also note that while routine \code{f3} can be supported, meaning that monitor \code{**m} is be acquired, passing an array to this routine would be type safe and yet result in undefined behavior because only the first element of the array is acquired. However, this ambiguity is part of the C type system with respects to arrays. For this reason, \code{mutex} is disallowed in the context where arrays may be passed.
    175 
    176 Finally, for convenience, monitors support multiple acquireing, that is acquireing a monitor while already holding it does not cause a deadlock. It simply increments an internal counter which is then used to release the monitor after the number of acquires and releases match up.
    177 
    178 % ######     #    #######    #
    179 % #     #   # #      #      # #
    180 % #     #  #   #     #     #   #
    181 % #     # #     #    #    #     #
    182 % #     # #######    #    #######
    183 % #     # #     #    #    #     #
    184 % ######  #     #    #    #     #
    185 
    186 \subsection{Data semantics} \label{data}
    187 Once the call semantics are established, the next step is to establish data semantics. Indeed, until now a monitor is used simply as a generic handle but in most cases monitors contian shared data. This data should be intrinsic to the monitor declaration to prevent any accidental use of data without its appropriate protection. For example, here is a complete version of the counter showed in section \ref{call}:
    188 \begin{cfacode}
    189         monitor counter_t {
    190                 int value;
    191         };
    192 
    193         void ?{}(counter_t & this) {
    194                 this.cnt = 0;
    195         }
    196 
    197         int ++?(counter_t & mutex this) {
    198                 return ++this.value;
    199         }
    200 
    201         //need for mutex is platform dependent here
    202         void ?{}(int * this, counter_t & mutex cnt) {
    203                 *this = (int)cnt;
    204         }
    205 \end{cfacode}
    206 
    207 This simple counter is used as follows:
    208 \begin{center}
    209 \begin{tabular}{c @{\hskip 0.35in} c @{\hskip 0.35in} c}
    210 \begin{cfacode}
    211         //shared counter
    212         counter_t cnt;
    213 
    214         //multiple threads access counter
    215         thread 1 : cnt++;
    216         thread 2 : cnt++;
    217         thread 3 : cnt++;
    218           ...
    219         thread N : cnt++;
    220 \end{cfacode}
    221 \end{tabular}
    222 \end{center}
    223 
    224 Notice how the counter is used without any explicit synchronisation and yet supports thread-safe semantics for both reading and writting. Unlike object-oriented monitors, where calling a mutex member \emph{implicitly} acquires mutual-exclusion, \CFA uses an explicit mechanism to acquire mutual-exclusion. A consequence of this approach is that it extends to multi-monitor calls.
    225 \begin{cfacode}
    226         int f(MonitorA & mutex a, MonitorB & mutex b);
    227 
    228         MonitorA a;
    229         MonitorB b;
    230         f(a,b);
    231 \end{cfacode}
    232 This code acquires both locks before entering the critical section, called \emph{\gls{group-acquire}}. In practice, writing multi-locking routines that do not lead to deadlocks is tricky. Having language support for such a feature is therefore a significant asset for \CFA. In the case presented above, \CFA guarantees that the order of aquisition is consistent across calls to routines using the same monitors as arguments. However, since \CFA monitors use multi-acquisition locks, users can effectively force the acquiring order. For example, notice which routines use \code{mutex}/\code{nomutex} and how this affects aquiring order :
    233 \begin{cfacode}
    234         void foo(A & mutex a, B & mutex b) { //acquire a & b
    235                 //...
    236         }
    237 
    238         void bar(A & mutex a, B & /*nomutex*/ b) { //acquire a
    239                 //...
    240                 foo(a, b); //acquire b
    241                 //...
    242         }
    243 
    244         void baz(A & /*nomutex*/ a, B & mutex b) { //acquire b
    245                 //...
    246                 foo(a, b); //acquire a
    247                 //...
    248         }
    249 \end{cfacode}
    250 
    251 The multi-acquisition monitor lock allows a monitor lock to be acquired by both \code{bar} or \code{baz} and acquired again in \code{foo}. In the calls to \code{bar} and \code{baz} the monitors are acquired in opposite order. such use leads to nested monitor call problems~\cite{Lister77}, which is a more specific variation of the lock acquiring order problem. In the example above, the user uses implicit ordering in the case of function \code{foo} but explicit ordering in the case of \code{bar} and \code{baz}. This subtle mistake means that calling these routines concurrently may lead to deadlock and is therefore undefined behavior. As shown on several occasion\cit, solving this problem requires :
    252 \begin{enumerate}
    253         \item Dynamically tracking of the monitor-call order.
    254         \item Implement rollback semantics.
    255 \end{enumerate}
    256 
    257 While the first requirement is already a significant constraint on the system, implementing a general rollback semantics in a C-like language is prohibitively complex \cit. In \CFA, users simply need to be carefull when acquiring multiple monitors at the same time.
    258 
    259 % ######  ####### #######    #    ### #        #####
    260 % #     # #          #      # #    #  #       #     #
    261 % #     # #          #     #   #   #  #       #
    262 % #     # #####      #    #     #  #  #        #####
    263 % #     # #          #    #######  #  #             #
    264 % #     # #          #    #     #  #  #       #     #
    265 % ######  #######    #    #     # ### #######  #####
    266 %
    267 %             ######  ####### #       #     # #     # ####### ######  #     #
    268 %             #     # #     # #        #   #  ##   ## #     # #     # #     #
    269 %             #     # #     # #         # #   # # # # #     # #     # #     #
    270 %  #####    ######  #     # #          #    #  #  # #     # ######  #######
    271 %             #       #     # #          #    #     # #     # #   #   #     #
    272 %             #       #     # #          #    #     # #     # #    #  #     #
    273 %             #       ####### #######    #    #     # ####### #     # #     #
    274 
    275 \subsection{Implementation Details: Interaction with polymorphism}
    276 At first glance, interaction between monitors and \CFA's concept of polymorphism seems complex to support. However, it is shown that entry-point locking can solve most of the issues.
    277 
    278 Before looking into complex control flow, it is important to present the difference between the two acquiring options : \gls{callsite-locking} and \gls{entry-point-locking}, i.e. acquiring the monitors before making a mutex call or as the first instruction of the mutex call. For example:
    279 
    280 \begin{center}
    281 \begin{tabular}{|c|c|c|}
    282 Code & \gls{callsite-locking} & \gls{entry-point-locking} \\
    283 \CFA & pseudo-code & pseudo-code \\
    284 \hline
    285 \begin{cfacode}[tabsize=3]
    286 void foo(monitor& mutex a){
    287 
    288 
    289 
    290         //Do Work
    291         //...
    292 
    293 }
    294 
    295 void main() {
    296         monitor a;
    297 
    298 
    299 
    300         foo(a);
    301 
    302 }
    303 \end{cfacode} & \begin{pseudo}[tabsize=3]
    304 foo(& a) {
    305 
    306 
    307 
    308         //Do Work
    309         //...
    310 
    311 }
    312 
    313 main() {
    314         monitor a;
    315         //calling routine
    316         //handles concurrency
    317         acquire(a);
    318         foo(a);
    319         release(a);
    320 }
    321 \end{pseudo} & \begin{pseudo}[tabsize=3]
    322 foo(& a) {
    323         //called routine
    324         //handles concurrency
    325         acquire(a);
    326         //Do Work
    327         //...
    328         release(a);
    329 }
    330 
    331 main() {
    332         monitor a;
    333 
    334 
    335 
    336         foo(a);
    337 
    338 }
    339 \end{pseudo}
    340 \end{tabular}
    341 \end{center}
    342 
    343 First of all, interaction between \code{otype} polymorphism and monitors is impossible since monitors do not support copying. Therefore, the main question is how to support \code{dtype} polymorphism. Since a monitor's main purpose is to ensure mutual exclusion when accessing shared data, this implies that mutual exclusion is only required for routines that do in fact access shared data. However, since \code{dtype} polymorphism always handles incomplete types (by definition), no \code{dtype} polymorphic routine can access shared data since the data requires knowledge about the type. Therefore, the only concern when combining \code{dtype} polymorphism and monitors is to protect access to routines. \Gls{callsite-locking} would require a significant amount of work, since any \code{dtype} routine may have to obtain some lock before calling a routine, depending on whether or not the type passed is a monitor. However, with \gls{entry-point-locking} calling a monitor routine becomes exactly the same as calling it from anywhere else. Note that the \code{mutex} keyword relies on the resolver, which mean that in cases where generic monitor routines is actually desired, writing mutex routine is possible with the proper trait.
    344 
    345 
    346 % ### #     # #######         #####   #####  #     # ####### ######
    347 %  #  ##    #    #           #     # #     # #     # #       #     #
    348 %  #  # #   #    #           #       #       #     # #       #     #
    349 %  #  #  #  #    #            #####  #       ####### #####   #     #
    350 %  #  #   # #    #    ###          # #       #     # #       #     #
    351 %  #  #    ##    #    ###    #     # #     # #     # #       #     #
    352 % ### #     #    #    ###     #####   #####  #     # ####### ######
    353 
    354 \section{Internal scheduling} \label{insched}
    355 In addition to mutual exclusion, the monitors at the core of \CFA's concurrency can also be used to achieve synchronisation. With monitors, this is generally achieved with internal or external scheduling as in\cit. Since internal scheduling of single monitors is mostly a solved problem, this proposal concentraits on extending internal scheduling to multiple monitors at once. Indeed, like the \gls{group-acquire} semantics, internal scheduling extends to multiple monitors at once in a way that is natural to the user but requires additional complexity on the implementation side.
    356 
    357 First, Here is a simple example of such a technique :
    358 
    359 \begin{cfacode}
    360         monitor A {
    361                 condition e;
    362         }
    363 
    364         void foo(A & mutex a) {
    365                 // ...
    366                 // We need someone else to do something now
    367                 wait(a.e);
    368                 // ...
    369         }
    370 
    371         void bar(A & mutex a) {
    372                 // Do the thing foo is waiting on
    373                 // ...
    374                 // Signal foo it's done
    375                 signal(a.e);
    376         }
    377 \end{cfacode}
    378 
    379 Note that in \CFA, \code{condition} have no particular need to be stored inside a monitor, beyond any software engineering reasons. Here routine \code{foo} waits for the \code{signal} from \code{bar} before making further progress, effectively ensuring a basic ordering. An important aspect to take into account here is that \CFA does not allow barging, which means that once function \code{bar} releases the monitor, foo is guaranteed to resume immediately after (unless some other function waited on the same condition). This guarantees offers the benefit of not having to loop arount waits in order to guarantee that a condition is still met. The main reason \CFA offers this guarantee is that users can easily introduce barging if it becomes a necessity but adding a barging prevention or barging avoidance is more involved without language support.
    380 
    381 Supporting barging prevention as well as extending internal scheduling to multiple monitors is the main source of complexity in the design of \CFA concurrency.
    382 
    383 \subsection{Internal Scheduling - multi monitor}
    384 It easier to understand the problem of multi monitor scheduling using a series of pseudo code though experiment. Note that in the following snippets of pseudo-code waiting and signalling is done without the use of a condition variable. While \CFA requires condition variables to use signalling, the variable itself only really holds the data needed for the implementation of internal schedulling. Some languages like JAVA\cit simply define an implicit condition variable for every monitor while other languages like \uC use explicit condition variables. Since the following pseudo-codes are simple and focused experiments, all condition variables are implicit.
    385 
    386 \begin{multicols}{2}
    387 \begin{pseudo}
    388 acquire A
    389         wait A
    390 release A
    391 \end{pseudo}
    392 
    393 \columnbreak
    394 
    395 \begin{pseudo}
    396 acquire A
    397         signal A
    398 release A
    399 \end{pseudo}
    400 \end{multicols}
    401 
    402 The previous example shows the simple case of having two threads (one for each column) and a single monitor A. One thread acquires before waiting and the other acquires before signalling. There are a few important things to note here. First, both \code{wait} and \code{signal} must be called with the proper monitor(s) already acquired. This can be hidden on the user side but is a logical requirement for barging prevention. Secondly, as stated above, while it is argued that not all problems regarding single monitors are solved, this paper only regards challenges of \gls{group-acquire} and considers other problems related to monitors as solved.
    403 
    404 An important note about this example is that signalling a monitor is a delayed operation. The ownership of the monitor is transferred only when the monitor would have otherwise been released, not at the point of the \code{signal} statement.
    405 
    406 A direct extension of the previous example is the \gls{group-acquire} version :
    407 
    408 \begin{multicols}{2}
    409 \begin{pseudo}
    410 acquire A & B
    411         wait A & B
    412 release A & B
    413 \end{pseudo}
    414 
    415 \columnbreak
    416 
    417 \begin{pseudo}
    418 acquire A & B
    419         signal A & B
    420 release A & B
    421 \end{pseudo}
    422 \end{multicols}
    423 
    424 This version uses \gls{group-acquire} (denoted using the \& symbol), but the presence of multiple monitors does not add a particularly new meaning. Synchronization will happen between the two threads in exactly the same way and order. The only difference is that mutual exclusion will cover more monitors. On the implementation side, handling multiple monitors at once does add a degree of complexity but it is not significant compared to the next few examples.
    425 
    426 For the sake of completeness, here is another example of the single-monitor case, this time with nesting.
    427 
    428 \begin{multicols}{2}
    429 \begin{pseudo}
    430 acquire A
    431         acquire B
    432                 wait B
    433         release B
    434 release A
    435 \end{pseudo}
    436 
    437 \columnbreak
    438 
    439 \begin{pseudo}
    440 
    441 acquire B
    442         signal B
    443 release B
    444 
    445 \end{pseudo}
    446 \end{multicols}
    447 
    448 While these cases can cause some deadlock issues, we consider that these issues are only a symptom of the fact that locks, and by extension monitors, are not perfectly composable. However, for monitors as for locks, it is possible to write program that using nesting without encountering any problems if they are nested carefully.
    449 
    450 The next example is where \gls{group-acquire} adds a significant layer of complexity to the internal signalling semantics.
    451 
    452 \begin{multicols}{2}
    453 \begin{pseudo}
    454 acquire A
    455         // Code Section 1
    456         acquire A & B
    457                 // Code Section 2
    458                 wait A & B
    459                 // Code Section 3
    460         release A & B
    461         // Code Section 4
    462 release A
    463 \end{pseudo}
    464 
    465 \columnbreak
    466 
    467 \begin{pseudo}
    468 acquire A
    469         // Code Section 5
    470         acquire A & B
    471                 // Code Section 6
    472                 signal A & B
    473                 // Code Section 7
    474         release A & B
    475         // Code Section 8
    476 release A
    477 \end{pseudo}
    478 \end{multicols}
    479 
    480 It is particularly important to pay attention to code sections 8 and 3 which are where the existing semantics of internal scheduling are undefined. The root of the problem is that \gls{group-acquire} is used in a context where one of the monitors is already acquired. As mentionned in previous sections, monitors support multiple acquiring which means the that nesting \gls{group-acquire} can be done safely. However, in the context of internal scheduling it is important to define the behaviour of the previous pseudo-code. When the signaller thread reaches the location where it should "release A \& B", it actually only needs to release the monitor B. Since the other thread is waiting on monitor B, the signaller thread cannot simply release the monitor into the wild. This would mean that the waiting thread would have to reacquire the monitor and would therefore open the door to barging threads. Since the signalling thread still needs the monitor A, simply transferring ownership to the waiting thread is not an option because it would pottentially violate mutual exclusion. We are therefore left with three options :
    481 
    482 \subsubsection{Delaying signals}
    483 The first more obvious solution to solve the problem of multi-monitor scheduling is to keep ownership of all locks until the last lock is ready to be transferred. It can be argued that that moment is the correct time to transfer ownership when the last lock is no longer needed is what fits most closely to the behaviour of single monitor scheduling. However, this solution can become much more complicated depending on the content of the code section 8. Indeed, nothing prevents a user from signalling monitor A on a different condition variable. In that case, if monitor B is transferred with monitor A, then it means the system needs to handle threads having ownership on more monitors than expected and how to tie monitors together. On the other hand if the signalling thread only transfers monitor A then somehow both monitors A and B have to be transferred to the waiting thread from two different threads. While this solution may work, it was not fully explored because there is no apparent upper bound on the complexity of ownership transfer.
    484 
    485 \subsubsection{Dependency graphs}
    486 In the previous pseudo-code, there is a solution which would statisfy both barging prevention and mutual exclusion. If ownership of both monitors is transferred to the waiter when the signaller releases A and then the waiter transfers back ownership of A when it releases it then the problem is solved. This is the second solution. The problem it encounters is that it effectively boils down to resolving a dependency graph of ownership requirements. Here even the simplest of code snippets requires two transfers and it seems to increase in a manner closer to polynomial. For example the following code which is just a direct extension to three monitors requires at least three ownership transfer and has multiple solutions.
    487 
    488 \begin{multicols}{2}
    489 \begin{pseudo}
    490 acquire A
    491         acquire B
    492                 acquire C
    493                         wait A & B & C
    494                 release C
    495         release B
    496 release A
    497 \end{pseudo}
    498 
    499 \columnbreak
    500 
    501 \begin{pseudo}
    502 acquire A
    503         acquire B
    504                 acquire C
    505                         signal A & B & C
    506                 release C
    507         release B
    508 release A
    509 \end{pseudo}
    510 \end{multicols}
    511 
    512 \subsubsection{Partial signalling}
    513 Finally, the solution that was chosen for \CFA is to use partial signalling. Consider the following case :
    514 
    515 \begin{multicols}{2}
    516 \begin{pseudo}[numbers=left]
    517 acquire A
    518         acquire A & B
    519                 wait A & B
    520         release A & B
    521 release A
    522 \end{pseudo}
    523 
    524 \columnbreak
    525 
    526 \begin{pseudo}[numbers=left, firstnumber=6]
    527 acquire A
    528         acquire A & B
    529                 signal A & B
    530         release A & B
    531         // ... More code
    532 release A
    533 \end{pseudo}
    534 \end{multicols}
    535 
    536 The partial signalling solution transfers ownership of monitor B at lines 10 but does not wake the waiting thread since it is still using monitor A. Only when it reaches line 11 does it actually wakeup the waiting thread. This solution has the benefit that complexity is encapsulated in to only two actions, passing monitors to the next owner when they should be release and conditionnaly waking threads if all conditions are met. Contrary to the other solutions, this solution quickly hits an upper bound on complexity of implementation.
    537 
    538 % Hard extension :
    539 
    540 % Incorrect options for the signal :
    541 
    542 % \begin{description}
    543 %  \item[-] Release B and baton pass after Code Section 8 : Passing b without having it
    544 %  \item[-] Keep B during Code Section 8 : Can lead to deadlocks since we secretly keep a lock longer than specified by the user
    545 %  \item[-] Instead of release B transfer A and B to waiter then try to reacquire A before running Code Section 8 : This allows barging
    546 % \end{description}
    547 
    548 % Since we don't want barging we need to pass A \& B and somehow block and get A back.
    549 
    550 % \begin{center}
    551 % \begin{tabular}{ c @{\hskip 0.65in} c }
    552 % \begin{lstlisting}[language=Pseudo]
    553 % acquire A
    554 %       acquire B
    555 %               acquire C
    556 %                       wait A & B & C
    557 %               1: release C
    558 %       2: release B
    559 % 3: release A
    560 % \end{lstlisting}&\begin{lstlisting}[language=Pseudo]
    561 % acquire A
    562 %       acquire B
    563 %               acquire C
    564 %                       signal A & B & C
    565 %               4: release C
    566 %       5: release B
    567 % 6: release A
    568 % \end{lstlisting}
    569 % \end{tabular}
    570 % \end{center}
    571 
    572 % To prevent barging :
    573 
    574 % \begin{description}
    575 %  \item[-] When the signaller hits 4 : pass A, B, C to waiter
    576 %  \item[-] When the waiter hits 2 : pass A, B to signaller
    577 %  \item[-] When the signaller hits 5 : pass A to waiter
    578 % \end{description}
    579 
    580 
    581 % \begin{center}
    582 % \begin{tabular}{ c @{\hskip 0.65in} c }
    583 % \begin{lstlisting}[language=Pseudo]
    584 % acquire A
    585 %       acquire C
    586 %               acquire B
    587 %                       wait A & B & C
    588 %               1: release B
    589 %       2: release C
    590 % 3: release A
    591 % \end{lstlisting}&\begin{lstlisting}[language=Pseudo]
    592 % acquire B
    593 %       acquire A
    594 %               acquire C
    595 %                       signal A & B & C
    596 %               4: release C
    597 %       5: release A
    598 % 6: release B
    599 % \end{lstlisting}
    600 % \end{tabular}
    601 % \end{center}
    602 
    603 % To prevent barging : When the signaller hits 4 : pass A, B, C to waiter. When the waiter hits 1 it must release B,
    604 
    605 % \begin{description}
    606 %  \item[-]
    607 %  \item[-] When the waiter hits 1 : pass A, B to signaller
    608 %  \item[-] When the signaller hits 5 : pass A, B to waiter
    609 %  \item[-] When the waiter hits 2 : pass A to signaller
    610 % \end{description}
    611 
    612 % Monitors also need to schedule waiting threads internally as a mean of synchronization. Internal scheduling is one of the simple examples of such a feature. It allows users to declare condition variables and have threads wait and signaled from them. Here is a simple example of such a technique :
    613 
    614 % \begin{lstlisting}
    615 %       mutex struct A {
    616 %               condition e;
    617 %       }
    618 
    619 %       void foo(A & mutex a) {
    620 %               //...
    621 %               wait(a.e);
    622 %               //...
    623 %       }
    624 
    625 %       void bar(A & mutex a) {
    626 %               signal(a.e);
    627 %       }
    628 % \end{lstlisting}
    629 
    630 % Note that in \CFA, \code{condition} have no particular need to be stored inside a monitor, beyond any software engineering reasons. Here routine \code{foo} waits for the \code{signal} from \code{bar} before making further progress, effectively ensuring a basic ordering.
    631 
    632 % As for simple mutual exclusion, these semantics must also be extended to include \gls{group-acquire} :
    633 % \begin{center}
    634 % \begin{tabular}{ c @{\hskip 0.65in} c }
    635 % Thread 1 & Thread 2 \\
    636 % \begin{lstlisting}
    637 % void foo(A & mutex a,
    638 %            A & mutex b) {
    639 %       //...
    640 %       wait(a.e);
    641 %       //...
    642 % }
    643 
    644 % foo(a, b);
    645 % \end{lstlisting} &\begin{lstlisting}
    646 % void bar(A & mutex a,
    647 %            A & mutex b) {
    648 %       signal(a.e);
    649 % }
    650 
    651 
    652 
    653 % bar(a, b);
    654 % \end{lstlisting}
    655 % \end{tabular}
    656 % \end{center}
    657 
    658 % To define the semantics of internal scheduling, it is important to look at nesting and \gls{group-acquire}. Indeed, beyond concerns about lock ordering, without scheduling the two following pseudo codes are mostly equivalent. In fact, if we assume monitors are ordered alphabetically, these two pseudo codes would probably lead to exactly the same implementation :
    659 
    660 % \begin{table}[h!]
    661 % \centering
    662 % \begin{tabular}{c c}
    663 % \begin{lstlisting}[language=pseudo]
    664 % monitor A, B, C
    665 
    666 % acquire A
    667 %       acquire B & C
    668 
    669 %                       //Do stuff
    670 
    671 %       release B & C
    672 % release A
    673 % \end{lstlisting} &\begin{lstlisting}[language=pseudo]
    674 % monitor A, B, C
    675 
    676 % acquire A
    677 %       acquire B
    678 %               acquire C
    679 %                       //Do stuff
    680 %               release C
    681 %       release B
    682 % release A
    683 % \end{lstlisting}
    684 % \end{tabular}
    685 % \end{table}
    686 
    687 % Once internal scheduling is introduce however, semantics of \gls{group-acquire} become relevant. For example, let us look into the semantics of the following pseudo-code :
    688 
    689 % \begin{lstlisting}[language=Pseudo]
    690 % 1: monitor A, B, C
    691 % 2: condition c1
    692 % 3:
    693 % 4: acquire A
    694 % 5:            acquire A & B & C
    695 % 6:                            signal c1
    696 % 7:            release A & B & C
    697 % 8: release A
    698 % \end{lstlisting}
    699 
    700 % Without \gls{group-acquire} signal simply baton passes the monitor lock on the next release. In the case above, we therefore need to indentify the next release. If line 8 is picked at the release point, then the signal will attempt to pass A \& B \& C, without having ownership of B \& C. Since this violates mutual exclusion, we conclude that line 7 is the only valid location where signalling can occur. The traditionnal meaning of signalling is to transfer ownership of the monitor(s) and immediately schedule the longest waiting task. However, in the discussed case, the signalling thread expects to maintain ownership of monitor A. This can be expressed in two differents ways : 1) the thread transfers ownership of all locks and reacquires A when it gets schedulled again or 2) it transfers ownership of all three monitors and then expects the ownership of A to be transferred back.
    701 
    702 % However, the question is does these behavior motivate supporting acquireing non-disjoint set of monitors. Indeed, if the previous example was modified to only acquire B \& C at line 5 (an release the accordingly) then in respects to scheduling, we could add the simplifying constraint that all monitors in a bulk will behave the same way, simplifying the problem back to a single monitor problem which has already been solved. For this constraint to be acceptble however, we need to demonstrate that in does not prevent any meaningful possibilities. And, indeed, we can look at the two previous interpretation of the above pseudo-code and conclude that supporting the acquiring of non-disjoint set of monitors does not add any expressiveness to the language.
    703 
    704 % Option 1 reacquires the lock after the signal statement, this can be rewritten as follows without the need for non-disjoint sets :
    705 % \begin{lstlisting}[language=Pseudo]
    706 % monitor A, B, C
    707 % condition c1
    708 
    709 % acquire A & B & C
    710 %       signal c1
    711 % release A & B & C
    712 % acquire A
    713 
    714 % release A
    715 % \end{lstlisting}
    716 
    717 % This pseudo code has almost exaclty the same semantics as the code acquiring intersecting sets of monitors.
    718 
    719 % Option 2 uses two-way lock ownership transferring instead of reacquiring monitor A. Two-way monitor ownership transfer is normally done using signalBlock semantics, which immedietely transfers ownership of a monitor before getting the ownership back when the other thread no longer needs the monitor. While the example pseudo-code for Option 2 seems toe transfer ownership of A, B and C and only getting A back, this is not a requirement. Getting back all 3 monitors and releasing B and C differs only in performance. For this reason, the second option could arguably be rewritten as :
    720 
    721 % \begin{lstlisting}[language=Pseudo]
    722 % monitor A, B, C
    723 % condition c1
    724 
    725 % acquire A
    726 %       acquire B & C
    727 %               signalBlock c1
    728 %       release B & C
    729 % release A
    730 % \end{lstlisting}
    731 
    732 % Obviously, the difference between these two snippets of pseudo code is that the first one transfers ownership of A, B and C while the second one only transfers ownership of B and C. However, this limitation can be removed by allowing user to release extra monitors when using internal scheduling, referred to as extended internal scheduling (pattent pending) from this point on. Extended internal scheduling means the two following pseudo-codes are functionnaly equivalent :
    733 % \begin{table}[h!]
    734 % \centering
    735 % \begin{tabular}{c @{\hskip 0.65in} c}
    736 % \begin{lstlisting}[language=pseudo]
    737 % monitor A, B, C
    738 % condition c1
    739 
    740 % acquire A
    741 %       acquire B & C
    742 %               signalBlock c1 with A
    743 %       release B & C
    744 % release A
    745 % \end{lstlisting} &\begin{lstlisting}[language=pseudo]
    746 % monitor A, B, C
    747 % condition c1
    748 
    749 % acquire A
    750 %       acquire A & B & C
    751 %               signal c1
    752 %       release A & B & C
    753 % release A
    754 % \end{lstlisting}
    755 % \end{tabular}
    756 % \end{table}
    757 
    758 % It must be stated that the extended internal scheduling only makes sense when using wait and signalBlock, since they need to prevent barging, which cannot be done in the context of signal since the ownership transfer is strictly one-directionnal.
    759 
    760 % One critic that could arise is that extended internal schedulling is not composable since signalBlock must be explicitly aware of which context it is in. However, this argument is not relevant since acquire A, B and C in a context where a subset of them is already acquired cannot be achieved without spurriously releasing some locks or having an oracle aware of all monitors. Therefore, composability of internal scheduling is no more an issue than composability of monitors in general.
    761 
    762 % The main benefit of using extended internal scheduling is that it offers the same expressiveness as intersecting monitor set acquiring but greatly simplifies the selection of a leader (or representative) for a group of monitor. Indeed, when using intersecting sets, it is not obvious which set intersects with other sets which means finding a leader representing only the smallest scope is a hard problem. Where as when using disjoint sets, any monitor that would be intersecting must be specified in the extended set, the leader can be chosen as any monitor in the primary set.
    763 
    764 % We need to make sure the semantics for internally scheduling N monitors are a natural extension of the single monitor semantics. For this reason, we introduce the concept of \gls{mon-ctx}. In terms of context internal scheduling means "releasing a \gls{mon-ctx} and waiting for an other thread to acquire the same \gls{mon-ctx} and baton-pass it back to the initial thread". This definitions requires looking into what a \gls{mon-ctx} is and what the semantics of waiting and baton-passing are.
    765 
    766 % \subsubsection{Internal scheduling: Context} \label{insched-context}
    767 % Monitor scheduling operations are defined in terms of the context they are in. In languages that only supports operations on a single monitor at once, the context is completly defined by which most recently acquired monitors. Indeed, acquiring several monitors will form a stack of monitors which will be released in FILO order. In \CFA, a \gls{mon-ctx} cannot be simply defined by the last monitor that was acquired since \gls{group-acquire} means multiple monitors can be "the last monitor acquired". The \gls{mon-ctx} is therefore defined as the last set of monitors to have been acquired. This means taht when any new monitor is acquired, the group it belongs to is the new \gls{mon-ctx}. Correspondingly, if any monitor is released, the \gls{mon-ctx} reverts back to the context that was used prior to the monitor being acquired. In the most common case, \gls{group-acquire} means every monitor of a group will be acquired in released at the same time. However, since every monitor has its own recursion level, \gls{group-acquire} does not prevent users from reacquiring certain monitors while acquireing new monitors in the same operation. For example :
    768 
    769 % \begin{lstlisting}
    770 % //Forward declarations
    771 % monitor a, b, c
    772 % void foo( monitor & mutex a,
    773 %             monitor & mutex b);
    774 % void bar( monitor & mutex a,
    775 %             monitor & mutex b);
    776 % void baz( monitor & mutex a,
    777 %             monitor & mutex b,
    778 %             monitor & mutex c);
    779 
    780 % //Routines defined inline to illustrate context changed compared to the stack
    781 
    782 % //main thread
    783 % foo(a, b) {
    784 %       //thread calls foo
    785 %       //acquiring context a & b
    786 
    787 %       baz(a, b) {
    788 %               //thread calls baz
    789 %               //no context change
    790 
    791 %               bar(a, b, c) {
    792 %                       //thread calls bar
    793 %                       //acquiring context a & b & c
    794 
    795 %                       //Do stuff
    796 
    797 %                       return;             
    798 %                       //call to bar returns
    799 %               }
    800 %               //context back to a & b
    801 
    802 %               return;
    803 %               //call to baz returns
    804 %       }
    805 %       //no context change
    806 
    807 %       return;
    808 %       //call to foo returns
    809 % }
    810 % //context back to initial state
    811 
    812 % \end{lstlisting}
    813 
    814 % As illustrated by the previous example, context changes can be caused by only one of the monitors comming into context or going out of context.
    815 
    816 % \subsubsection{Internal scheduling: Waiting} \label{insched-wait}
    817 
    818 % \subsubsection{Internal scheduling: Baton Passing} \label{insched-signal}
    819 % Baton passing in internal scheduling is done in terms of \code{signal} and \code{signalBlock}\footnote{Arguably, \code{signal_now} is a more evocative name and \code{signal} could be changed appropriately. }. While \code{signalBlock} is the more straight forward way of baton passing, transferring ownership immediately, it must rely on \code{signal} which is why t is discussed first.
    820 % \code{signal} has for effect to transfer the current context to another thread when the context would otherwise be released. This means that instead of releasing the concerned monitors, the first thread on the condition ready-queue is scheduled to run. The monitors are not released and when the signalled thread runs, it assumes it regained ownership of all the monitors it had in its context.
    821 
    822 % \subsubsection{Internal scheduling: Implementation} \label{insched-impl}
    823 % Too implement internal scheduling, three things are need : a data structure for waiting tasks, a data structure for signalled task and a leaving procedure to run the signalled task. In the case of both data structures, it is desireable to have to use intrusive data structures in order to prevent the need for any dynamic allocation. However, in both cases being able to queue several items in the same position in a queue is non trivial, even more so in the presence of concurrency. However, within a given \gls{mon-ctx}, all monitors have exactly the same behavior in regards to scheduling. Therefore, the problem of queuing multiple monitors at once can be ignored by choosing one monitor to represent every monitor in a context. While this could prove difficult in other situations, \gls{group-acquire} requires that the monitors be sorted according to some stable predicate. Since monitors are sorted in all contexts, the representative can simply be the first in the list. Choosing a representative means a simple intrusive queue inside the condition is sufficient to implement the data structure for both waiting and signalled monitors.
    824 
    825 % Since \CFA monitors don't have a complete image of the \gls{mon-ctx}, choosing the representative and maintaning the current context information cannot easily be done by any single monitors. However, as discussed in section [Missing section here], monitor mutual exclusion is implemented using an raii object which is already in charge of sorting monitors. This object has a complete picture of the \gls{mon-ctx} which means it is well suited to choose the reprensentative and detect context changes.
    826 
    827 % \newpage
    828 % \begin{lstlisting}
    829 % void ctor( monitor ** _monitors, int _count ) {
    830 %       bool ctx_changed = false;
    831 %       for( mon in _monitors ) {
    832 %               ctx_changed = acquire( mon ) || ctx_changed;
    833 %       }
    834 
    835 %       if( ctx_changed ) {
    836 %               set_representative();
    837 %               set_context();
    838 %       }
    839 % }
    840 
    841 % void dtor( monitor ** _monitors, int _count ) {
    842 %       if( context_will_exit( _monitors, count ) ) {
    843 %               baton_pass();
    844 %               return;
    845 %       }
    846 
    847 %       for( mon in _monitors ) {
    848 %               release( mon );
    849 %       }
    850 % }
    851 
    852 % \end{lstlisting}
    853 
    854 
    855 
    856 % A direct extension of the single monitor semantics is to release all locks when waiting and transferring ownership of all locks when signalling. However, for the purpose of synchronization it may be usefull to only release some of the locks but keep others. It is possible to support internal scheduling and \gls{group-acquire} without any extra syntax by relying on order of acquisition. Here is an example of the different contexts in which internal scheduling can be used. (Note that here the use of helper routines is irrelevant, only routines acquire mutual exclusion have an impact on internal scheduling):
    857 
    858 % \begin{table}[h!]
    859 % \centering
    860 % \begin{tabular}{|c|c|c|}
    861 % Context 1 & Context 2 & Context 3 \\
    862 % \hline
    863 % \begin{lstlisting}
    864 % condition e;
    865 
    866 % //acquire a & b
    867 % void foo(monitor & mutex a,
    868 %            monitor & mutex b) {
    869 
    870 %       wait(e); //release a & b
    871 % }
    872 
    873 
    874 
    875 
    876 
    877 
    878 % foo(a,b);
    879 % \end{lstlisting} &\begin{lstlisting}
    880 % condition e;
    881 
    882 % //acquire a
    883 % void bar(monitor & mutex a,
    884 %            monitor & nomutex b) {
    885 %       foo(a,b);
    886 % }
    887 
    888 % //acquire a & b
    889 % void foo(monitor & mutex a,
    890 %            monitor & mutex b) {
    891 %       wait(e);  //release a & b
    892 % }
    893 
    894 % bar(a, b);
    895 % \end{lstlisting} &\begin{lstlisting}
    896 % condition e;
    897 
    898 % //acquire a
    899 % void bar(monitor & mutex a,
    900 %            monitor & nomutex b) {
    901 %       baz(a,b);
    902 % }
    903 
    904 % //acquire b
    905 % void baz(monitor & nomutex a,
    906 %            monitor & mutex b) {
    907 %       wait(e);  //release b
    908 % }
    909 
    910 % bar(a, b);
    911 % \end{lstlisting}
    912 % \end{tabular}
    913 % \end{table}
    914 
    915 % Context 1 is the simplest way of acquiring more than one monitor (\gls{group-acquire}), using a routine with multiple parameters having the \code{mutex} keyword. Context 2 also uses \gls{group-acquire} as well in routine \code{foo}. However, the routine is called by routine \code{bar}, which only acquires monitor \code{a}. Since monitors can be acquired multiple times this does not cause a deadlock by itself but it does force the acquiring order to \code{a} then \code{b}. Context 3 also forces the acquiring order to be \code{a} then \code{b} but does not use \gls{group-acquire}. The previous example tries to illustrate the semantics that must be established to support releasing monitors in a \code{wait} statement. In all cases, the behavior of the wait statment is to release all the locks that were acquired my the inner-most monitor call. That is \code{a & b} in context 1 and 2 and \code{b} only in context 3. Here are a few other examples of this behavior.
    916 
    917 
    918 % \begin{center}
    919 % \begin{tabular}{|c|c|c|}
    920 % \begin{lstlisting}
    921 % condition e;
    922 
    923 % //acquire b
    924 % void foo(monitor & nomutex a,
    925 %            monitor & mutex b) {
    926 %       bar(a,b);
    927 % }
    928 
    929 % //acquire a
    930 % void bar(monitor & mutex a,
    931 %            monitor & nomutex b) {
    932 
    933 %       wait(e); //release a
    934 %                 //keep b
    935 % }
    936 
    937 % foo(a, b);
    938 % \end{lstlisting} &\begin{lstlisting}
    939 % condition e;
    940 
    941 % //acquire a & b
    942 % void foo(monitor & mutex a,
    943 %            monitor & mutex b) {
    944 %       bar(a,b);
    945 % }
    946 
    947 % //acquire b
    948 % void bar(monitor & mutex a,
    949 %            monitor & nomutex b) {
    950 
    951 %       wait(e); //release b
    952 %                 //keep a
    953 % }
    954 
    955 % foo(a, b);
    956 % \end{lstlisting} &\begin{lstlisting}
    957 % condition e;
    958 
    959 % //acquire a & b
    960 % void foo(monitor & mutex a,
    961 %            monitor & mutex b) {
    962 %       bar(a,b);
    963 % }
    964 
    965 % //acquire none
    966 % void bar(monitor & nomutex a,
    967 %            monitor & nomutex b) {
    968 
    969 %       wait(e); //release a & b
    970 %                 //keep none
    971 % }
    972 
    973 % foo(a, b);
    974 % \end{lstlisting}
    975 % \end{tabular}
    976 % \end{center}
    977 % Note the right-most example is actually a trick pulled on the reader. Monitor state information is stored in thread local storage rather then in the routine context, which means that helper routines and other \code{nomutex} routines are invisible to the runtime system in regards to concurrency. This means that in the right-most example, the routine parameters are completly unnecessary. However, calling this routine from outside a valid monitor context is undefined.
    978 
    979 % These semantics imply that in order to release of subset of the monitors currently held, users must write (and name) a routine that only acquires the desired subset and simply calls wait. While users can use this method, \CFA offers the \code{wait_release}\footnote{Not sure if an overload of \code{wait} would work...} which will release only the specified monitors. In the center previous examples, the code in the center uses the \code{bar} routine to only release monitor \code{b}. Using the \code{wait_release} helper, this can be rewritten without having the name two routines :
    980 % \begin{center}
    981 % \begin{tabular}{ c c c }
    982 % \begin{lstlisting}
    983 %       condition e;
    984 
    985 %       //acquire a & b
    986 %       void foo(monitor & mutex a,
    987 %                  monitor & mutex b) {
    988 %               bar(a,b);
    989 %       }
    990 
    991 %       //acquire b
    992 %       void bar(monitor & mutex a,
    993 %                  monitor & nomutex b) {
    994 
    995 %               wait(e); //release b
    996 %                         //keep a
    997 %       }
    998 
    999 %       foo(a, b);
    1000 % \end{lstlisting} &\begin{lstlisting}
    1001 %       =>
    1002 % \end{lstlisting} &\begin{lstlisting}
    1003 %       condition e;
    1004 
    1005 %       //acquire a & b
    1006 %       void foo(monitor & mutex a,
    1007 %                  monitor & mutex b) {
    1008 %               wait_release(e,b); //release b
    1009 %                                        //keep a
    1010 %       }
    1011 
    1012 %       foo(a, b);
    1013 % \end{lstlisting}
    1014 % \end{tabular}
    1015 % \end{center}
    1016 
    1017 % Regardless of the context in which the \code{wait} statement is used, \code{signal} must be called holding the same set of monitors. In all cases, signal only needs a single parameter, the condition variable that needs to be signalled. But \code{signal} needs to be called from the same monitor(s) that call to \code{wait}. Otherwise, mutual exclusion cannot be properly transferred back to the waiting monitor.
    1018 
    1019 % Finally, an additional semantic which can be very usefull is the \code{signal_block} routine. This routine behaves like signal for all of the semantics discussed above, but with the subtelty that mutual exclusion is transferred to the waiting task immediately rather than wating for the end of the critical section.
    1020 % \\
    1021 
    1022 % ####### #     # #######         #####   #####  #     # ####### ######
    1023 % #        #   #     #           #     # #     # #     # #       #     #
    1024 % #         # #      #           #       #       #     # #       #     #
    1025 % #####      #       #            #####  #       ####### #####   #     #
    1026 % #         # #      #    ###          # #       #     # #       #     #
    1027 % #        #   #     #    ###    #     # #     # #     # #       #     #
    1028 % ####### #     #    #    ###     #####   #####  #     # ####### ######
    1029 \section{External scheduling} \label{extsched}
    1030 An alternative to internal scheduling is to use external scheduling instead. This method is more constrained and explicit which may help users tone down the undeterministic nature of concurrency. Indeed, as the following examples demonstrates, external scheduling allows users to wait for events from other threads without the concern of unrelated events occuring. External scheduling can generally be done either in terms of control flow (ex: \uC) or in terms of data (ex: Go). Of course, both of these paradigms have their own strenghts and weaknesses but for this project control flow semantics where chosen to stay consistent with the rest of the languages semantics. Two challenges specific to \CFA arise when trying to add external scheduling with loose object definitions and multi-monitor routines. The following example shows a simple use \code{accept} versus \code{wait}/\code{signal} and its advantages.
    1031 
    1032 \begin{center}
    1033 \begin{tabular}{|c|c|}
    1034 Internal Scheduling & External Scheduling \\
    1035 \hline
    1036 \begin{lstlisting}
    1037         _Monitor blarg {
    1038                 condition c;
    1039         public:
    1040                 void f() { signal(c)}
    1041                 void g() { wait(c); }
    1042         private:
    1043         }
    1044 \end{lstlisting}&\begin{lstlisting}
    1045         _Monitor blarg {
    1046 
    1047         public:
    1048                 void f() { /*...*/ }
    1049                 void g() { _Accept(f); }
    1050         private:
    1051         }
    1052 \end{lstlisting}
    1053 \end{tabular}
    1054 \end{center}
    1055 
    1056 In the case of internal scheduling, the call to \code{wait} only guarantees that \code{g} is the last routine to access the monitor. This intails that the routine \code{f} may have acquired mutual exclusion several times while routine \code{h} was waiting. On the other hand, external scheduling guarantees that while routine \code{h} was waiting, no routine other than \code{g} could acquire the monitor.
    1057 \\
    1058 
    1059 % #       ####### #######  #####  #######    ####### ######        #  #####
    1060 % #       #     # #     # #     # #          #     # #     #       # #     #
    1061 % #       #     # #     # #       #          #     # #     #       # #
    1062 % #       #     # #     #  #####  #####      #     # ######        #  #####
    1063 % #       #     # #     #       # #          #     # #     # #     #       #
    1064 % #       #     # #     # #     # #          #     # #     # #     # #     #
    1065 % ####### ####### #######  #####  #######    ####### ######   #####   #####
    1066 
    1067 \subsection{Loose object definitions}
    1068 In \uC, monitor declarations include an exhaustive list of monitor operations. Since \CFA is not object oriented it becomes both more difficult to implement but also less clear for the user :
    1069 
    1070 \begin{lstlisting}
    1071         mutex struct A {};
    1072 
    1073         void f(A & mutex a);
    1074         void g(A & mutex a) { accept(f); }
    1075 \end{lstlisting}
    1076 
    1077 However, external scheduling is an example where implementation constraints become visible from the interface. Indeed, ince there is no hard limit to the number of threads trying to acquire a monitor concurrently, performance is a significant concern. Here is the pseudo code for the entering phase of a monitor :
    1078 
    1079 \begin{center}
    1080 \begin{tabular}{l}
    1081 \begin{lstlisting}[language=Pseudo]
    1082         if monitor is free :
    1083                 enter
    1084         elif monitor accepts me :
    1085                 enter
    1086         else :
    1087                 block
    1088 \end{lstlisting}
    1089 \end{tabular}
    1090 \end{center}
    1091 
    1092 For the \pscode{monitor is free} condition it is easy to implement a check that can evaluate the condition in a few instruction. However, a fast check for \pscode{monitor accepts me} is much harder to implement depending on the constraints put on the monitors. Indeed, monitors are often expressed as an entry queue and some acceptor queue as in the following figure :
    1093 
    1094 \begin{center}
    1095 {\resizebox{0.4\textwidth}{!}{\input{monitor}}}
    1096 \end{center}
    1097 
    1098 There are other alternatives to these pictures but in the case of this picture implementing a fast accept check is relatively easy. Indeed simply updating a bitmask when the acceptor queue changes is enough to have a check that executes in a single instruction, even with a fairly large number of acceptor. However, this relies on the fact that all the acceptable routines are declared with the monitor type. For OO languages this doesn't compromise much since monitors already have an exhaustive list of member routines. However, for \CFA this isn't the case, routines can be added to a type anywhere after its declaration. Its important to note that the bitmask approach does not actually require an exhaustive list of routines, but it requires a dense unique ordering of routines with an upper-bound and that ordering must be consistent across translation units.
    1099 The alternative would be to have a picture more like this one:
    1100 
    1101 \begin{center}
    1102 {\resizebox{0.4\textwidth}{!}{\input{ext_monitor}}}
    1103 \end{center}
    1104 
    1105 Not storing the queues inside the monitor means that the storage can vary between routines, allowing for more flexibility and extensions. Storing an array of function-pointers would solve the issue of uniquely identifying acceptable routines. However, the single instruction bitmask compare has been replaced by dereferencing a pointer followed by a linear search. Furthermore, supporting nested external scheduling may now require additionnal searches on calls to accept to check if a routine is already queued in.
    1106 
    1107 At this point we must make a decision between flexibility and performance. Many design decisions in \CFA achieve both flexibility and performance, for example polymorphic routines add significant flexibility but inlining them means the optimizer can easily remove any runtime cost. Here however, the cost of flexibility cannot be trivially removed.
    1108 
    1109 In either cases here are a few alternatives for the different syntaxes this syntax : \\
    1110 \begin{center}
    1111 {\renewcommand{\arraystretch}{1.5}
    1112 \begin{tabular}[t]{l @{\hskip 0.35in} l}
    1113 \hline
    1114 \multicolumn{2}{ c }{\code{accept} on type}\\
    1115 \hline
    1116 Alternative 1 & Alternative 2 \\
    1117 \begin{lstlisting}
    1118 mutex struct A
    1119 accept( void f(A & mutex a) )
    1120 {};
    1121 \end{lstlisting} &\begin{lstlisting}
    1122 mutex struct A {}
    1123 accept( void f(A & mutex a) );
    1124 
    1125 \end{lstlisting} \\
    1126 Alternative 3 & Alternative 4 \\
    1127 \begin{lstlisting}
    1128 mutex struct A {
    1129         accept( void f(A & mutex a) )
    1130 };
    1131 
    1132 \end{lstlisting} &\begin{lstlisting}
    1133 mutex struct A {
    1134         accept :
    1135                 void f(A & mutex a) );
    1136 };
    1137 \end{lstlisting}\\
    1138 \hline
    1139 \multicolumn{2}{ c }{\code{accept} on routine}\\
    1140 \hline
    1141 \begin{lstlisting}
    1142 mutex struct A {};
    1143 
    1144 void f(A & mutex a)
    1145 
    1146 accept( void f(A & mutex a) )
    1147 void g(A & mutex a) {
    1148         /*...*/
    1149 }
    1150 \end{lstlisting}&\\
    1151 \end{tabular}
    1152 }
    1153 \end{center}
    1154 
    1155 An other aspect to consider is what happens if multiple overloads of the same routine are used. For the time being it is assumed that multiple overloads of the same routine should be scheduled regardless of the overload used. However, this could easily be extended in the future.
    1156 
    1157 % #     # #     # #       ####### ###    #     # ####### #     #
    1158 % ##   ## #     # #          #     #     ##   ## #     # ##    #
    1159 % # # # # #     # #          #     #     # # # # #     # # #   #
    1160 % #  #  # #     # #          #     #     #  #  # #     # #  #  #
    1161 % #     # #     # #          #     #     #     # #     # #   # #
    1162 % #     # #     # #          #     #     #     # #     # #    ##
    1163 % #     #  #####  #######    #    ###    #     # ####### #     #
    1164 
    1165 \subsection{Multi-monitor scheduling}
    1166 
    1167 External scheduling, like internal scheduling, becomes orders of magnitude more complex when we start introducing multi-monitor syntax. Even in the simplest possible case some new semantics need to be established :
    1168 \begin{lstlisting}
    1169         accept( void f(mutex struct A & mutex this))
    1170         mutex struct A {};
    1171 
    1172         mutex struct B {};
    1173 
    1174         void g(A & mutex a, B & mutex b) {
    1175                 accept(f); //ambiguous, which monitor
    1176         }
    1177 \end{lstlisting}
    1178 
    1179 The obvious solution is to specify the correct monitor as follows :
    1180 
    1181 \begin{lstlisting}
    1182         accept( void f(mutex struct A & mutex this))
    1183         mutex struct A {};
    1184 
    1185         mutex struct B {};
    1186 
    1187         void g(A & mutex a, B & mutex b) {
    1188                 accept( f, b );
    1189         }
    1190 \end{lstlisting}
    1191 
    1192 This is unambiguous. Both locks will be acquired and kept, when routine \code{f} is called the lock for monitor \code{a} will be temporarily transferred from \code{g} to \code{f} (while \code{g} still holds lock \code{b}). This behavior can be extended to multi-monitor accept statment as follows.
    1193 
    1194 \begin{lstlisting}
    1195         accept( void f(mutex struct A & mutex, mutex struct A & mutex))
    1196         mutex struct A {};
    1197 
    1198         mutex struct B {};
    1199 
    1200         void g(A & mutex a, B & mutex b) {
    1201                 accept( f, b, a );
    1202         }
    1203 \end{lstlisting}
    1204 
    1205 Note that the set of monitors passed to the \code{accept} statement must be entirely contained in the set of monitor already acquired in the routine. \code{accept} used in any other context is Undefined Behaviour.
    1206 
    1207 % ######  ####### #######    #    ### #        #####
    1208 % #     # #          #      # #    #  #       #     #
    1209 % #     # #          #     #   #   #  #       #
    1210 % #     # #####      #    #     #  #  #        #####
    1211 % #     # #          #    #######  #  #             #
    1212 % #     # #          #    #     #  #  #       #     #
    1213 % ######  #######    #    #     # ### #######  #####
    1214 %
    1215 %                #####  #     # ####### #     # #######  #####
    1216 %             #     # #     # #       #     # #       #     #
    1217 %             #     # #     # #       #     # #       #
    1218 %    #####    #     # #     # #####   #     # #####    #####
    1219 %             #   # # #     # #       #     # #             #
    1220 %             #    #  #     # #       #     # #       #     #
    1221 %                #### #  #####  #######  #####  #######  #####
    1222 
    1223 
    1224 \subsection{Implementation Details: External scheduling queues}
    1225 To support multi-monitor external scheduling means that some kind of entry-queues must be used that is aware of both monitors. However, acceptable routines must be aware of the entry queues which means they must be stored inside at least one of the monitors that will be acquired. This in turn adds the requirement a systematic algorithm of disambiguating which queue is relavant regardless of user ordering. The proposed algorithm is to fall back on monitors lock ordering and specify that the monitor that is acquired first is the lock with the relevant entry queue. This assumes that the lock acquiring order is static for the lifetime of all concerned objects but that is a reasonnable constraint. This algorithm choice has two consequences, the entry queue of the highest priority monitor is no longer a true FIFO queue and the queue of the lowest priority monitor is both required and probably unused. The queue can no longer be a FIFO queue because instead of simply containing the waiting threads in order arrival, they also contain the second mutex. Therefore, another thread with the same highest priority monitor but a different lowest priority monitor may arrive first but enter the critical section after a thread with the correct pairing. Secondly, since it may not be known at compile time which monitor will be the lowest priority monitor, every monitor needs to have the correct queues even though it is probable that half the multi-monitor queues will go unused for the entire duration of the program.
    1226 
    1227 \section{Other concurrency tools}
    1228 TO BE CONTINUED...
    1229 
    1230 
    1231 
    1232 
    1233 
    1234 
    1235 
    1236 
    1237 
    1238 
    1239 % ######     #    ######     #    #       #       ####### #       ###  #####  #     #
    1240 % #     #   # #   #     #   # #   #       #       #       #        #  #     # ##   ##
    1241 % #     #  #   #  #     #  #   #  #       #       #       #        #  #       # # # #
    1242 % ######  #     # ######  #     # #       #       #####   #        #   #####  #  #  #
    1243 % #       ####### #   #   ####### #       #       #       #        #        # #     #
    1244 % #       #     # #    #  #     # #       #       #       #        #  #     # #     #
    1245 % #       #     # #     # #     # ####### ####### ####### ####### ###  #####  #     #
    1246 \chapter{Parallelism}
    1247 Historically, computer performance was about processor speeds and instructions count. However, with heat dissipation being a direct consequence of speed increase, parallelism has become the new source for increased performance~\cite{Sutter05, Sutter05b}. In this decade, it is not longer reasonnable to create a high-performance application without caring about parallelism. Indeed, parallelism is an important aspect of performance and more specifically throughput and hardware utilization. The lowest-level approach of parallelism is to use \glspl{kthread} in combination with semantics like \code{fork}, \code{join}, etc. However, since these have significant costs and limitations, \glspl{kthread} are now mostly used as an implementation tool rather than a user oriented one. There are several alternatives to solve these issues that all have strengths and weaknesses. While there are many variations of the presented paradigms, most of these variations do not actually change the guarantees or the semantics, they simply move costs in order to achieve better performance for certain workloads.
    1248 
    1249 \section{Paradigm}
    1250 \subsection{User-level threads}
    1251 A direct improvement on the \gls{kthread} approach is to use \glspl{uthread}. These threads offer most of the same features that the operating system already provide but can be used on a much larger scale. This approach is the most powerfull solution as it allows all the features of multi-threading, while removing several of the more expensives costs of using kernel threads. The down side is that almost none of the low-level threading problems are hidden, users still have to think about data races, deadlocks and synchronization issues. These issues can be somewhat alleviated by a concurrency toolkit with strong garantees but the parallelism toolkit offers very little to reduce complexity in itself.
    1252 
    1253 Examples of languages that support \glspl{uthread} are Erlang~\cite{Erlang} and \uC~\cite{uC++book}.
    1254 
    1255 \subsection{Fibers : user-level threads without preemption}
    1256 A popular varient of \glspl{uthread} is what is often reffered to as \glspl{fiber}. However, \glspl{fiber} do not present meaningful semantical differences with \glspl{uthread}. Advocates of \glspl{fiber} list their high performance and ease of implementation as majors strenghts of \glspl{fiber} but the performance difference between \glspl{uthread} and \glspl{fiber} is controversial and the ease of implementation, while true, is a weak argument in the context of language design. Therefore this proposal largely ignore fibers.
    1257 
    1258 An example of a language that uses fibers is Go~\cite{Go}
    1259 
    1260 \subsection{Jobs and thread pools}
    1261 The approach on the opposite end of the spectrum is to base parallelism on \glspl{pool}. Indeed, \glspl{pool} offer limited flexibility but at the benefit of a simpler user interface. In \gls{pool} based systems, users express parallelism as units of work and a dependency graph (either explicit or implicit) that tie them together. This approach means users need not worry about concurrency but significantly limits the interaction that can occur among jobs. Indeed, any \gls{job} that blocks also blocks the underlying worker, which effectively means the CPU utilization, and therefore throughput, suffers noticeably. It can be argued that a solution to this problem is to use more workers than available cores. However, unless the number of jobs and the number of workers are comparable, having a significant amount of blocked jobs always results in idles cores.
    1262 
    1263 The gold standard of this implementation is Intel's TBB library~\cite{TBB}.
    1264 
    1265 \subsection{Paradigm performance}
    1266 While the choice between the three paradigms listed above may have significant performance implication, it is difficult to pindown the performance implications of chosing a model at the language level. Indeed, in many situations one of these paradigms may show better performance but it all strongly depends on the workload. Having a large amount of mostly independent units of work to execute almost guarantess that the \gls{pool} based system has the best performance thanks to the lower memory overhead. However, interactions between jobs can easily exacerbate contention. User-level threads allow fine-grain context switching, which results in better resource utilisation, but context switches will be more expansive and the extra control means users need to tweak more variables to get the desired performance. Furthermore, if the units of uninterrupted work are large enough the paradigm choice is largely amorticised by the actual work done.
    1267 
    1268 %  #####  #######    #          ####### ######  ######
    1269 % #     # #         # #            #    #     # #     #
    1270 % #       #        #   #           #    #     # #     #
    1271 % #       #####   #     # #####    #    ######  ######
    1272 % #       #       #######          #    #     # #     #
    1273 % #     # #       #     #          #    #     # #     #
    1274 %  #####  #       #     #          #    ######  ######
    1275 
    1276 \section{\CFA 's Thread Building Blocks}
    1277 As a system-level language, \CFA should offer both performance and flexibilty as its primary goals, simplicity and user-friendliness being a secondary concern. Therefore, the core of parallelism in \CFA should prioritize power and efficiency. With this said, deconstructing popular paradigms in order to get simple building blocks yields \glspl{uthread} as the core parallelism block. \Glspl{pool} and other parallelism paradigms can then be built on top of the underlying threading model.
    1278 
    1279 \subsection{Coroutines : A stepping stone}\label{coroutine}
    1280 While the main focus of this proposal is concurrency and paralellism, it is important to adress coroutines which are actually a significant underlying aspect of the concurrency system. Indeed, while having nothing todo with parallelism and arguably very little to do with concurrency, coroutines need to deal with context-switchs and and other context management operations. Therefore, this proposal includes coroutines both as an intermediate step for the implementation of threads and a first class feature of \CFA.
    1281 
    1282 The core API of coroutines revolve around two features : independent stacks and \code{suspend}/\code{resume}.
    1283 Here is an example of a solution to the fibonnaci problem using \CFA coroutines :
    1284 \begin{lstlisting}
    1285         struct Fibonacci {
    1286               int fn; // used for communication
    1287               coroutine_descriptor c;
    1288         };
    1289 
    1290         void ?{}(Fibonacci* this) {
    1291               this->fn = 0;
    1292         }
    1293 
    1294         coroutine_descriptor* get_¶coroutine¶(Fibonacci* this) {
    1295               return &this->c;
    1296         }
    1297 
    1298         void co_main(Fibonacci* this) {
    1299                 int fn1, fn2;           // retained between resumes
    1300                 this->fn = 0;
    1301                 fn1 = this->fn;
    1302                 suspend(this);          // return to last resume
    1303 
    1304                 this->fn = 1;
    1305                 fn2 = fn1;
    1306                 fn1 = this->fn;
    1307                 suspend(this);          // return to last resume
    1308 
    1309                 for ( ;; ) {
    1310                         this->fn = fn1 + fn2;
    1311                         fn2 = fn1;
    1312                         fn1 = this->fn;
    1313                         suspend(this);  // return to last resume
    1314                 }
    1315         }
    1316 
    1317         int next(Fibonacci* this) {
    1318                 resume(this); // transfer to last suspend
    1319                 return this.fn;
    1320         }
    1321 
    1322         void main() {
    1323                 Fibonacci f1, f2;
    1324                 for ( int i = 1; i <= 10; i += 1 ) {
    1325                         sout | next(&f1) | '§\verb+ +§' | next(&f2) | endl;
    1326                 }
    1327         }
    1328 \end{lstlisting}
    1329 
    1330 \subsubsection{Construction}
    1331 One important design challenge for coroutines and threads (shown in section \ref{threads}) is that the runtime system needs to run some code after the user-constructor runs. In the case of the coroutines this challenge is simpler since there is no loss of determinism brough by preemption or scheduling, however, the underlying challenge remains the same for coroutines and threads.
    1332 
    1333 The runtime system needs to create the coroutine's stack and more importantly prepare it for the first resumption. The timing of the creation is non trivial since users both expect to have fully constructed objects once the main is called and to be able to resume the coroutine from the main (Obviously we only solve cases where these two statements don't conflict). There are several solutions to this problem but the chosen options effectively forces the design of the coroutine.
    1334 
    1335 Furthermore, \CFA faces an extra challenge which is that polymorphique routines rely on invisible thunks when casted to non-polymorphic routines and these thunks have function scope, for example :
    1336 
    1337 TODO : Simple case where a thunk would be created.
    1338 
    1339 
    1340 
    1341 \subsubsection{Alternative: Inheritance}
    1342 One solution to this challenge would be to use inheritence,
    1343 
    1344 \begin{lstlisting}
    1345         struct Fibonacci {
    1346               int fn; // used for communication
    1347               coroutine c;
    1348         };
    1349 
    1350         void ?{}(Fibonacci* this) {
    1351               this->fn = 0;
    1352                 (&this->c){};
    1353         }
    1354 \end{lstlisting}
    1355 
    1356 There are two downsides to the approach. The first, which is relatively minor, is that the base class needs to be made aware of the main routine pointer, regardless of whether we use a parameter or a virtual pointer, this means the coroutine data must be made larger to store a value that is actually a compile time constant (The address of the main routine). The second problem which is both subtle but significant, is that now can get the initialisation order of there coroutines wrong. Indeed, every field of a \CFA struct will be constructed but in the order of declaration, unless users explicitly write otherwise. This means that users who forget to initialize a the coroutine at the right time may resume the coroutine at with an uninitilized object. For coroutines, this is unlikely to be a problem, for threads however, this is a significant problem.
    1357 
    1358 \subsubsection{Alternative: Reserved keyword}
    1359 The next alternative is to use language support to annotate coroutines as follows :
    1360 
    1361 \begin{lstlisting}
    1362         coroutine struct Fibonacci {
    1363               int fn; // used for communication
    1364         };
    1365 \end{lstlisting}
    1366 
    1367 This mean the compiler can solve problems by injecting code where needed. The downside of this approach is that it makes coroutine a special case in the language. Users who would want to extend coroutines or build their own for various reasons can only do so in ways offered by the language. Furthermore, implementing coroutines without language supports also displays the power of \CFA.
    1368 
    1369 \subsubsection{Alternative: Lamda Objects}
    1370 
    1371 Boost does not use objects...
    1372 TO BE CONTINUED...
    1373 
    1374 \subsubsection{Trait based coroutines}
    1375 
    1376 Finally the approach chosen, which is the one closest to \CFA idioms, is to use trait-based lazy coroutines, the approach shown in section \ref{coroutine}. This approach defines a coroutine as anything that satisfies the \code{is_coroutine} and is used as a coroutine is a coroutine. This entails the an object is not a coroutine until \code{resume} (and \code{prime}) is called on the object. Correspondingly, any object that is passed to \code{resume} is a coroutine since it must satisfy the \code{is_coroutine} trait to compile.
    1377 
    1378 % ####### #     # ######  #######    #    ######   #####
    1379 %    #    #     # #     # #         # #   #     # #     #
    1380 %    #    #     # #     # #        #   #  #     # #
    1381 %    #    ####### ######  #####   #     # #     #  #####
    1382 %    #    #     # #   #   #       ####### #     #       #
    1383 %    #    #     # #    #  #       #     # #     # #     #
    1384 %    #    #     # #     # ####### #     # ######   #####
    1385 
    1386 \subsection{Thread Interface}\label{threads}
    1387 The basic building blocks of \CFA are \glspl{cfathread}. By default these are implemented as \glspl{uthread}, and as such, offer a flexible and lightweight threading interface (lightweight compared to \glspl{kthread}). A thread can be declared using a struct declaration with prefix \code{thread} as follows :
    1388 
    1389 \begin{lstlisting}
    1390         trait is_¶thread¶(dtype T) {
    1391                 void co_main(T* this);
    1392                 coroutine* get_coroutine(T* this);
    1393         };
    1394 
    1395         thread struct foo {};
    1396 \end{lstlisting}
    1397 
    1398 Obviously, for this thread implementation to be usefull it must run some user code. Several other threading interfaces use a function-pointer representation as the interface of threads (for example : \Csharp~\cite{Csharp} and Scala~\cite{Scala}). However, this proposal considers that statically tying a \code{main} routine to a thread superseeds this approach. Since the \code{main} routine is already a special routine in \CFA (where the program begins), the existing syntax for declaring routines names with special semantics can be extended, i.e. operator overloading. As such the \code{main} routine of a thread can be defined as :
    1399 \begin{lstlisting}
    1400         thread struct foo {};
    1401 
    1402         void ?main(foo* this) {
    1403                 sout | "Hello World!" | endl;
    1404         }
    1405 \end{lstlisting}
    1406 
    1407 In this example, threads of type \code{foo} will start there execution in the \code{void ?main(foo*)} routine which in this case prints \code{"Hello World!"}. While this proposoal encourages this approach which is enforces strongly type programming. Users may prefer to use the routine based thread semantics for the sake of simplicity. With these semantics it is trivial to write a thread type that takes a function pointer as parameter and executes it on its stack asynchronously :
    1408 \begin{lstlisting}
    1409         typedef void (*voidFunc)(void);
    1410 
    1411         thread struct FuncRunner {
    1412                 voidFunc func;
    1413         };
    1414 
    1415         //ctor
    1416         void ?{}(FuncRunner* this, voidFunc inFunc) {
    1417                 func = inFunc;
    1418         }
    1419 
    1420         //main
    1421         void t_main(FuncRunner* this) {
    1422                 this->func();
    1423         }
    1424 \end{lstlisting}
    1425 
    1426 Of course for threads to be useful, it must be possible to start and stop threads and wait for them to complete execution. While using an \acrshort{api} such as \code{fork} and \code{join} is relatively common in the literature, such an interface is unnecessary. Indeed, the simplest approach is to use \acrshort{raii} principles and have threads \code{fork} once the constructor has completed and \code{join} before the destructor runs.
    1427 \begin{lstlisting}
    1428 thread struct World; //FuncRunner declared above
    1429 
    1430 void ?main(thread World* this) {
    1431         sout | "World!" | endl;
    1432 }
    1433 
    1434 void main() {
    1435         World w;
    1436         //Thread run forks here
    1437 
    1438         //Print to "Hello " and "World!" will be run concurrently
    1439         sout | "Hello " | endl;
    1440 
    1441         //Implicit join at end of scope
    1442 }
    1443 \end{lstlisting}
    1444 This semantic has several advantages over explicit semantics : typesafety is guaranteed, a thread is always started and stopped exaclty once and users cannot make any progamming errors. However, one of the apparent drawbacks of this system is that threads now always form a lattice, that is they are always destroyed in opposite order of construction. While this seems like a significant limitation, existing \CFA semantics can solve this problem. Indeed, by using dynamic allocation to create threads will naturally let threads outlive the scope in which the thread was created much like dynamically allocating memory will let objects outlive the scope in which thy were created :
    1445 
    1446 \begin{lstlisting}
    1447         thread struct MyThread {
    1448                 //...
    1449         };
    1450 
    1451         //ctor
    1452         void ?{}(MyThread* this,
    1453                      bool is_special = false) {
    1454                 //...
    1455         }
    1456 
    1457         //main
    1458         void ?main(MyThread* this) {
    1459                 //...
    1460         }
    1461 
    1462         void foo() {
    1463                 MyThread* special_thread;
    1464                 {
    1465                         MyThread thrds = {false};
    1466                         //Start a thread at the beginning of the scope
    1467 
    1468                         DoStuff();
    1469 
    1470                         //create a other thread that will outlive the thread in this scope
    1471                         special_thread = new MyThread{true};
    1472 
    1473                         //Wait for the thread to finish
    1474                 }
    1475                 DoMoreStuff();
    1476 
    1477                 //Now wait for the special
    1478         }
    1479 \end{lstlisting}
    1480 
    1481 Another advantage of this semantic is that it naturally scale to multiple threads meaning basic synchronisation is very simple :
    1482 
    1483 \begin{lstlisting}
    1484         thread struct MyThread {
    1485                 //...
    1486         };
    1487 
    1488         //ctor
    1489         void ?{}(MyThread* this) {}
    1490 
    1491         //main
    1492         void ?main(MyThread* this) {
    1493                 //...
    1494         }
    1495 
    1496         void foo() {
    1497                 MyThread thrds[10];
    1498                 //Start 10 threads at the beginning of the scope
    1499 
    1500                 DoStuff();
    1501 
    1502                 //Wait for the 10 threads to finish
    1503         }
    1504 \end{lstlisting}
    1505 
    1506 \newpage
    1507 \bf{WORK IN PROGRESS}
    1508 \subsection{The \CFA Kernel : Processors, Clusters and Threads}\label{kernel}
    1509 
    1510 
    1511 \subsection{Paradigms}\label{cfaparadigms}
    1512 Given these building blocks we can then reproduce the all three of the popular paradigms. Indeed, we get \glspl{uthread} as the default paradigm in \CFA. However, disabling \glspl{preemption} on the \gls{cfacluster} means \glspl{cfathread} effectively become \glspl{fiber}. Since several \glspl{cfacluster} with different scheduling policy can coexist in the same application, this allows \glspl{fiber} and \glspl{uthread} to coexist in the runtime of an application.
    1513 
    1514 % \subsection{High-level options}\label{tasks}
    1515 %
    1516 % \subsubsection{Thread interface}
    1517 % constructors destructors
    1518 %       initializer lists
    1519 % monitors
    1520 %
    1521 % \subsubsection{Futures}
    1522 %
    1523 % \subsubsection{Implicit threading}
    1524 % Finally, simpler applications can benefit greatly from having implicit parallelism. That is, parallelism that does not rely on the user to write concurrency. This type of parallelism can be achieved both at the language level and at the system level.
    1525 %
    1526 % \begin{center}
    1527 % \begin{tabular}[t]{|c|c|c|}
    1528 % Sequential & System Parallel & Language Parallel \\
    1529 % \begin{lstlisting}
    1530 % void big_sum(int* a, int* b,
    1531 %                int* out,
    1532 %                size_t length)
    1533 % {
    1534 %       for(int i = 0; i < length; ++i ) {
    1535 %               out[i] = a[i] + b[i];
    1536 %       }
    1537 % }
    1538 %
    1539 %
    1540 %
    1541 %
    1542 %
    1543 % int* a[10000];
    1544 % int* b[10000];
    1545 % int* c[10000];
    1546 % //... fill in a and b ...
    1547 % big_sum(a, b, c, 10000);
    1548 % \end{lstlisting} &\begin{lstlisting}
    1549 % void big_sum(int* a, int* b,
    1550 %                int* out,
    1551 %                size_t length)
    1552 % {
    1553 %       range ar(a, a + length);
    1554 %       range br(b, b + length);
    1555 %       range or(out, out + length);
    1556 %       parfor( ai, bi, oi,
    1557 %       [](int* ai, int* bi, int* oi) {
    1558 %               oi = ai + bi;
    1559 %       });
    1560 % }
    1561 %
    1562 % int* a[10000];
    1563 % int* b[10000];
    1564 % int* c[10000];
    1565 % //... fill in a and b ...
    1566 % big_sum(a, b, c, 10000);
    1567 % \end{lstlisting}&\begin{lstlisting}
    1568 % void big_sum(int* a, int* b,
    1569 %                int* out,
    1570 %                size_t length)
    1571 % {
    1572 %       for (ai, bi, oi) in (a, b, out) {
    1573 %               oi = ai + bi;
    1574 %       }
    1575 % }
    1576 %
    1577 %
    1578 %
    1579 %
    1580 %
    1581 % int* a[10000];
    1582 % int* b[10000];
    1583 % int* c[10000];
    1584 % //... fill in a and b ...
    1585 % big_sum(a, b, c, 10000);
    1586 % \end{lstlisting}
    1587 % \end{tabular}
    1588 % \end{center}
    1589 %
    1590 % \subsection{Machine setup}\label{machine}
    1591 % Threads are all good and well but wee still some OS support to fully utilize available hardware.
    1592 %
    1593 % \textbf{\large{Work in progress...}} Do wee need something beyond specifying the number of kernel threads?
    1594 
    1595 %    #    #       #
    1596 %   # #   #       #
    1597 %  #   #  #       #
    1598 % #     # #       #
    1599 % ####### #       #
    1600 % #     # #       #
    1601 % #     # ####### #######
    1602103\chapter{Putting it all together}
    1603 
    1604 
    1605 
    1606 
    1607104
    1608105\chapter{Conclusion}
    1609106
    1610 
    1611 
    1612 
    1613 
    1614 
    1615 % ####### #     # ####### #     # ######  #######
    1616 % #       #     #    #    #     # #     # #
    1617 % #       #     #    #    #     # #     # #
    1618 % #####   #     #    #    #     # ######  #####
    1619 % #       #     #    #    #     # #   #   #
    1620 % #       #     #    #    #     # #    #  #
    1621 % #        #####     #     #####  #     # ######
    1622107\chapter{Future work}
    1623108Concurrency and parallelism is still a very active field that strongly benefits from hardware advances. As such certain features that aren't necessarily mature enough in their current state could become relevant in the lifetime of \CFA.
    1624109\subsection{Transactions}
    1625110
    1626 % ####### #     # ######
    1627 % #       ##    # #     #
    1628 % #       # #   # #     #
    1629 % #####   #  #  # #     #
    1630 % #       #   # # #     #
    1631 % #       #    ## #     #
    1632 % ####### #     # ######
    1633111\section*{Acknowledgements}
    1634112
Note: See TracChangeset for help on using the changeset viewer.