Changes in / [e8261bb:53d34343]

Files:

• Jenkins/Distribute (modified) (2 diffs)
• Jenkins/FullBuild (modified) (2 diffs)
• Jenkinsfile (modified) (2 diffs)
• doc/theses/mubeen_zulfiqar_MMath/benchmarks.tex (modified) (1 diff)
• libcfa/configure.ac (modified) (1 diff)

Jenkins/Distribute

@@ -2 +2 @@
 
 import groovy.transform.Field
-
-// For skipping stages
-import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
 
 //===========================================================================================================
@@ -10 +7 @@
 //===========================================================================================================
 
-node('master') {
-        // Globals
-        BuildDir  = pwd tmp: true
-        SrcDir    = pwd tmp: false
-        Settings  = null
-        Version   = ''
+// Globals
+BuildDir  = null
+SrcDir    = null
+Settings  = null
+Version   = ''
 
-        // Local variables
-        def err = null
-        def log_needed = false
+// Local variables
+def err = null
+def log_needed = false
 
-        currentBuild.result = "SUCCESS"
+currentBuild.result = "SUCCESS"
+
+final commit, build
+node {
 
         //Wrap build to add timestamp to command line
         wrap([$class: 'TimestamperBuildWrapper']) {
+                (commit, build) = prepare_build()
+        }
+}
 
-                final commit, build
-                (commit, build) = prepare_build()
-
-                node('x64') {
-                        BuildDir  = pwd tmp: true
-                        SrcDir    = pwd tmp: false
-
-                        Tools.Clean()
-
-                        Tools.Checkout( commit )
-
-                        Version = GetVersion( build )
-
-                        Configure()
-
-                        Package()
-
-                        Test()
-
-                        Archive()
-                }
-
-                // Update the build directories when exiting the node
+node('x64') {
+        //Wrap build to add timestamp to command line
+        wrap([$class: 'TimestamperBuildWrapper']) {
                 BuildDir  = pwd tmp: true
                 SrcDir    = pwd tmp: false
+
+                Tools.Clean()
+
+                Tools.Checkout( commit )
+
+                Version = GetVersion( build )
+
+                Configure()
+
+                Package()
+
+                Test()
+
+                Archive()
         }
-
 }
 

Jenkins/FullBuild

@@ -5 +5 @@
 //===========================================================================================================
 
-node ('master') {
+node {
         def err = null
 
@@ -144 +144 @@
 //Email notification on a full build failure
 def promote_email(boolean success) {
-        echo('notifying users')
+        node {
+                echo('notifying users')
 
-        def result = success ? "PROMOTE - SUCCESS" : "PROMOTE - FAILURE"
+                def result = success ? "PROMOTE - SUCCESS" : "PROMOTE - FAILURE"
 
-        //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
-        //Configurations for email format
-        def email_subject = "[cforall git][${result}]"
-        def email_body = """<p>This is an automated email from the Jenkins build machine. It was
-generated following the result of the C\u2200 nightly build.</p>
+                //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
+                //Configurations for email format
+                def email_subject = "[cforall git][${result}]"
+                def email_body = """<p>This is an automated email from the Jenkins build machine. It was
+        generated following the result of the C\u2200 nightly build.</p>
 
-<p>Check console output at ${env.BUILD_URL} to view the results.</p>
+        <p>Check console output at ${env.BUILD_URL} to view the results.</p>
 
-<p>- Status --------------------------------------------------------------</p>
+        <p>- Status --------------------------------------------------------------</p>
 
-<p>${result}</p>
+        <p>${result}</p>
 
-<p>- Performance ---------------------------------------------------------</p>
+        <p>- Performance ---------------------------------------------------------</p>
 
-<img src="https://cforall.uwaterloo.ca/jenkins/job/Cforall/job/master/plot/Compilation/getPlot?index=0" >
-<img src="https://cforall.uwaterloo.ca/jenkins/job/Cforall/job/master/plot/Compilation/getPlot?index=1" >
+        <img src="https://cforall.uwaterloo.ca/jenkins/job/Cforall/job/master/plot/Compilation/getPlot?index=0" >
+        <img src="https://cforall.uwaterloo.ca/jenkins/job/Cforall/job/master/plot/Compilation/getPlot?index=1" >
 
-<p>- Logs ----------------------------------------------------------------</p>
-"""
+        <p>- Logs ----------------------------------------------------------------</p>
+        """
 
-        def email_to = "cforall@lists.uwaterloo.ca"
+                def email_to = "cforall@lists.uwaterloo.ca"
 
-        //send email notification
-        emailext body: email_body, subject: email_subject, to: email_to, attachLog: !success
+                //send email notification
+                emailext body: email_body, subject: email_subject, to: email_to, attachLog: !success
+        }
 }

Jenkinsfile

@@ -7 +7 @@
 //===========================================================================================================
 
-node('master') {
-        // Globals
-        BuildDir  = pwd tmp: true
-        SrcDir    = pwd tmp: false
-        Settings  = null
-        Tools     = null
-
-        // Local variables
-        def err = null
-        def log_needed = false
-
-        currentBuild.result = "SUCCESS"
-
-        try {
+// Globals
+BuildDir  = null
+SrcDir    = null
+Settings  = null
+Tools     = null
+
+// Local variables
+def err = null
+def log_needed = false
+
+currentBuild.result = "SUCCESS"
+
+try {
+        node {
                 //Wrap build to add timestamp to command line
                 wrap([$class: 'TimestamperBuildWrapper']) {
-
                         Settings = prepare_build()
-
-                        node(Settings.Architecture.node) {
-                                BuildDir  = pwd tmp: true
-                                SrcDir    = pwd tmp: false
-                                currentBuild.description = "${currentBuild.description} on ${env.NODE_NAME}"
-
-                                Tools.Clean()
-
-                                Tools.Checkout()
-
-                                build()
-
-                                test()
-
-                                benchmark()
-
-                                build_doc()
-
-                                publish()
-                        }
-
-                        // Update the build directories when exiting the node
+                }
+        }
+
+        node(Settings.Architecture.node) {
+                //Wrap build to add timestamp to command line
+                wrap([$class: 'TimestamperBuildWrapper']) {
                         BuildDir  = pwd tmp: true
                         SrcDir    = pwd tmp: false
-                }
-        }
-
-        //If an exception is caught we need to change the status and remember to
-        //attach the build log to the email
-        catch (Exception caughtError) {
-                // Store the result of the build log
-                currentBuild.result = "FAILURE"
-
-                // An error has occured, the build log is relevent
-                log_needed = true
-
-                // rethrow error later
-                err = caughtError
-
-                // print the error so it shows in the log
-                echo err.toString()
-        }
-
-        finally {
-                //Send email with final results if this is not a full build
-                email(log_needed)
-
-                echo 'Build Completed'
-
-                /* Must re-throw exception to propagate error */
-                if (err) {
-                        throw err
-                }
+                        currentBuild.description = "${currentBuild.description} on ${env.NODE_NAME}"
+
+                        Tools.Clean()
+
+                        Tools.Checkout()
+
+                        build()
+
+                        test()
+
+                        benchmark()
+
+                        build_doc()
+
+                        publish()
+                }
+        }
+}
+
+//If an exception is caught we need to change the status and remember to
+//attach the build log to the email
+catch (Exception caughtError) {
+        // Store the result of the build log
+        currentBuild.result = "FAILURE"
+
+        // An error has occured, the build log is relevent
+        log_needed = true
+
+        // rethrow error later
+        err = caughtError
+
+        // print the error so it shows in the log
+        echo err.toString()
+}
+
+finally {
+        //Send email with final results if this is not a full build
+        email(log_needed)
+
+        echo 'Build Completed'
+
+        /* Must re-throw exception to propagate error */
+        if (err) {
+                throw err
         }
 }
@@ -228 +226 @@
 //Standard build email notification
 def email(boolean log) {
-        //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
-        //Configurations for email format
-        echo 'Notifying users of result'
-
-        def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
-        def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
-        def email_body = """<p>This is an automated email from the Jenkins build machine. It was
-generated because of a git hooks/post-receive script following
-a ref change which was pushed to the C\u2200 repository.</p>
-""" + Tools.GitLogMessage()
-
-        def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
-
-        if( Settings && !Settings.Silent ) {
-                //send email notification
-                emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
-        } else {
-                echo "Would send email to: ${email_to}"
-                echo "With title: ${email_subject}"
-                echo "Content: \n${email_body}"
+        node {
+                //Since tokenizer doesn't work, figure stuff out from the environnement variables and command line
+                //Configurations for email format
+                echo 'Notifying users of result'
+
+                def project_name = (env.JOB_NAME =~ /(.+)\/.+/)[0][1].toLowerCase()
+                def email_subject = "[${project_name} git][BUILD# ${env.BUILD_NUMBER} - ${currentBuild.result}] - branch ${env.BRANCH_NAME}"
+                def email_body = """<p>This is an automated email from the Jenkins build machine. It was
+        generated because of a git hooks/post-receive script following
+        a ref change which was pushed to the C\u2200 repository.</p>
+        """ + Tools.GitLogMessage()
+
+                def email_to = !Settings.IsSandbox ? "cforall@lists.uwaterloo.ca" : "tdelisle@uwaterloo.ca"
+
+                if( Settings && !Settings.Silent ) {
+                        //send email notification
+                        emailext body: email_body, subject: email_subject, to: email_to, attachLog: log
+                } else {
+                        echo "Would send email to: ${email_to}"
+                        echo "With title: ${email_subject}"
+                        echo "Content: \n${email_body}"
+                }
         }
 }

doc/theses/mubeen_zulfiqar_MMath/benchmarks.tex

@@ -34 +34 @@
 \noindent
 ====================
+
+\section Performance Matrices of Memory Allocators
+
+When it comes to memory allocators, there are no set standards of performance. Performance of a memory allocator depends highly on the usage pattern of the application. A memory allocator that is the best performer for a certain application X might be the worst for some other application which has completely different memory usage pattern compared to the application X. It is extremely difficult to make one universally best memory allocator which will outperform every other memory allocator for every usage pattern. So, there is a lack of a set of standard benchmarks that are used to evaluate a memory allocators's performance.
+
+If we breakdown the goals of a memory allocator, there are two basic matrices on which a memory allocator's performance is evaluated.
+
+1. Memory Overhead
+2. Speed
+
+        /subsection Memory Overhead
+
+        Memory overhead is the extra memory that a memory allocator takes from OS which is not requested by the application. Ideally, an allocator should get just enough memory from OS that can fulfill application's request and should return this memory to OS as soon as applications frees it. But, allocators retain more memory compared to what application has asked for which causes memory overhead. Memory overhead can happen for various reasons.
+
+                /subsubsection Fragmentation
+                Fragmentation is one of the major reasons behind memory overhead. Fragmentation happens because of situations that are either necassary for proper functioning of the allocator such as internal memory management and book-keeping or are out of allocator's control such as application's usage pattern.
+
+                        /subsubsubsection Internal Fragmentation
+                        For internal book-keeping, allocators divide raw memory given by OS into chunks, blocks, or lists that can fulfill application's requested size. Allocators use memory given by OS for creating headers, footers etc. to store information about these chunks, blocks, or lists. This increases usage of memory in-addition to the memory requested by application as the allocators need to store their book-keeping information. This extra usage of memory for allocator's own book-keeping is called Internal Fragmentation. Although it cases memory overhead but this overhead is necassary for an allocator's proper funtioning.
+
+
+*** Insert a figure of internal fragmentation with explanation
+
+                        /subsubsubsection External Fragmentation
+                        External fragmentation is the free bits of memory between or around chunks of memory that are currently in-use of the application. Segmentation in memory due to application's usage pattern causes external fragmentation. The memory which is part of external fragmentation is completely free as it is neither used by allocator's internal book-keeping nor by the application. Ideally, an allocator should return a segment of memory back to the OS as soon as application frees it. But, this is not always the case. Allocators get memory from OS in one of the two ways.
+
+                        \begin{itemize}
+                        \item
+                        MMap: an allocator can ask OS for whole pages in mmap area. Then, the allocator segments the page internally and fulfills application's request.
+                        \item
+                        Heap: an allocator can ask OS for memory in heap area using system calls such as sbrk. Heap are grows downwards and shrinks upwards.
+                        \begin{itemize}
+
+                        If an allocator uses mmap area, it can only return extra memory back to OS if the whole page is free i.e. no chunk on the page is in-use of the application. Even if one chunk on the whole page is currently in-use of the application, the allocator has to retain the whole page.
+
+                        If an allocator uses the heap area, it can only return the continous free memory at the end of the heap area that is currently in allocator's possession as heap area shrinks upwards. If there are free bits of memory in-between chunks of memory that are currently in-use of the application, the allocator can not return these free bits.
+
+*** Insert a figure of above scenrio with explanation
+
+                        Even if the entire heap area is free except one small chunk at the end of heap area that is being used by the application, the allocator cannot return the free heap area back to the OS as it is not a continous region at the end of heap area.
+
+*** Insert a figure of above scenrio with explanation
+
+                        Such scenerios cause external fragmentation but it is out of the allocator's control and depend on application's usage pattern.
+
+                /subsubsection Internal Memory Management
+                Allocators such as je-malloc (FIX ME: insert reference) pro-actively get some memory from the OS and divide it into chunks of certain sizes that can be used in-future to fulfill application's request. This causes memory overhead as these chunks are made before application's request. There is also the possibility that an application may not even request memory of these sizes during their whole life-time.
+
+*** Insert a figure of above scenrio with explanation
+
+                Allocators such as rp-malloc (FIX ME: insert reference) maintain lists or blocks of sized memory segments that is freed by the application for future use. These lists are maintained without any guarantee that application will even request these sizes again.
+
+                Such tactics are usually used to gain speed as allocator will not have to get raw memory from OS and manage it at the time of application's request but they do cause memory overhead.
+
+        Fragmentation and managed sized chunks of free memory can lead to Heap Blowup as the allocator may not be able to use the fragments or sized free chunks of free memory to fulfill application's requests of other sizes.
+
+        /subsection Speed
+

libcfa/configure.ac

@@ -131 +131 @@
 #io_uring 5.5 uses enum values
 #io_uring 5.6 and later uses probes
+
+AH_TEMPLATE([CFA_HAVE_LINUX_RSEQ_H],[Defined if rseq support is present when compiling libcfathread.])
+AC_CHECK_HEADERS([linux/rseq.h], [AC_DEFINE(CFA_HAVE_LINUX_RSEQ_H)])
+
+AH_TEMPLATE([CFA_HAVE_LINUX_LIBRSEQ],[Defined if librseq support is present when compiling libcfathread.])
+AC_CHECK_LIB([rseq], [rseq_available], [AC_DEFINE(CFA_HAVE_LINUX_RSEQ_H)], [])
 
 AH_TEMPLATE([CFA_HAVE_LINUX_IO_URING_H],[Defined if io_uring support is present when compiling libcfathread.])