- File:
-
- 1 edited
-
libcfa/src/concurrency/kernel.cfa (modified) (16 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/kernel.cfa
rd3605f8 rc33c2af 27 27 extern "C" { 28 28 #include <sys/eventfd.h> 29 #include <sys/uio.h>30 29 } 31 30 … … 35 34 #include "strstream.hfa" 36 35 #include "device/cpu.hfa" 37 #include "io/types.hfa"38 36 39 37 //Private includes … … 126 124 static void __wake_one(cluster * cltr); 127 125 128 static void idle_sleep(processor * proc, io_future_t & future, iovec & iov); 129 static bool mark_idle (__cluster_proc_list & idles, processor & proc); 126 static void mark_idle (__cluster_proc_list & idles, processor & proc); 130 127 static void mark_awake(__cluster_proc_list & idles, processor & proc); 128 static [unsigned idle, unsigned total, * processor] query_idles( & __cluster_proc_list idles ); 131 129 132 130 extern void __cfa_io_start( processor * ); 133 131 extern bool __cfa_io_drain( processor * ); 134 extern bool __cfa_io_flush( processor *, int min_comp);132 extern void __cfa_io_flush( processor * ); 135 133 extern void __cfa_io_stop ( processor * ); 136 134 static inline bool __maybe_io_drain( processor * ); 137 138 #if defined(CFA_WITH_IO_URING_IDLE)139 extern bool __kernel_read(processor * proc, io_future_t & future, iovec &, int fd);140 #endif141 135 142 136 extern void __disable_interrupts_hard(); … … 154 148 /* paranoid */ verify( __preemption_enabled() ); 155 149 } 156 157 150 158 151 //============================================================================================= … … 170 163 verify(this); 171 164 172 io_future_t future; // used for idle sleep when io_uring is present173 future.self.ptr = 1p; // mark it as already fulfilled so we know if there is a pending request or not174 eventfd_t idle_val;175 iovec idle_iovec = { &idle_val, sizeof(idle_val) };176 177 165 __cfa_io_start( this ); 178 166 … … 208 196 209 197 if( !readyThread ) { 210 __cfa_io_flush( this, 0 ); 198 ready_schedule_lock(); 199 __cfa_io_flush( this ); 200 ready_schedule_unlock(); 211 201 212 202 readyThread = __next_thread_slow( this->cltr ); … … 223 213 224 214 // Push self to idle stack 225 if(!mark_idle(this->cltr->procs, * this)) continue MAIN_LOOP;215 mark_idle(this->cltr->procs, * this); 226 216 227 217 // Confirm the ready-queue is empty … … 239 229 } 240 230 241 idle_sleep( this, future, idle_iovec ); 242 243 // We were woken up, remove self from idle 244 mark_awake(this->cltr->procs, * this); 245 246 // DON'T just proceed, start looking again 247 continue MAIN_LOOP; 248 } 249 250 /* paranoid */ verify( readyThread ); 251 252 // Reset io dirty bit 253 this->io.dirty = false; 254 255 // We found a thread run it 256 __run_thread(this, readyThread); 257 258 // Are we done? 259 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; 260 261 if(this->io.pending && !this->io.dirty) { 262 __cfa_io_flush( this, 0 ); 263 } 264 265 #else 266 #warning new kernel loop 267 SEARCH: { 268 /* paranoid */ verify( ! __preemption_enabled() ); 269 270 // First, lock the scheduler since we are searching for a thread 271 ready_schedule_lock(); 272 273 // Try to get the next thread 274 readyThread = pop_fast( this->cltr ); 275 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 276 277 // If we can't find a thread, might as well flush any outstanding I/O 278 if(this->io.pending) { __cfa_io_flush( this, 0 ); } 279 280 // Spin a little on I/O, just in case 281 for(5) { 282 __maybe_io_drain( this ); 283 readyThread = pop_fast( this->cltr ); 284 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 285 } 286 287 // no luck, try stealing a few times 288 for(5) { 289 if( __maybe_io_drain( this ) ) { 290 readyThread = pop_fast( this->cltr ); 291 } else { 292 readyThread = pop_slow( this->cltr ); 231 #if !defined(__CFA_NO_STATISTICS__) 232 if(this->print_halts) { 233 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); 293 234 } 294 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 295 } 296 297 // still no luck, search for a thread 298 readyThread = pop_search( this->cltr ); 299 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 300 301 // Don't block if we are done 302 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) { 303 ready_schedule_unlock(); 304 break MAIN_LOOP; 305 } 306 307 __STATS( __tls_stats()->ready.sleep.halts++; ) 308 309 // Push self to idle stack 310 ready_schedule_unlock(); 311 if(!mark_idle(this->cltr->procs, * this)) goto SEARCH; 312 ready_schedule_lock(); 313 314 // Confirm the ready-queue is empty 315 __maybe_io_drain( this ); 316 readyThread = pop_search( this->cltr ); 317 ready_schedule_unlock(); 318 319 if( readyThread ) { 320 // A thread was found, cancel the halt 321 mark_awake(this->cltr->procs, * this); 322 323 __STATS( __tls_stats()->ready.sleep.cancels++; ) 324 325 // continue the main loop 326 break SEARCH; 327 } 328 329 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); ) 330 __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle_fd); 235 #endif 236 237 __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle); 331 238 332 239 { 333 240 eventfd_t val; 334 ssize_t ret = read( this->idle _fd, &val, sizeof(val) );241 ssize_t ret = read( this->idle, &val, sizeof(val) ); 335 242 if(ret < 0) { 336 243 switch((int)errno) { … … 348 255 } 349 256 350 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); ) 257 #if !defined(__CFA_NO_STATISTICS__) 258 if(this->print_halts) { 259 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); 260 } 261 #endif 351 262 352 263 // We were woken up, remove self from idle … … 357 268 } 358 269 270 /* paranoid */ verify( readyThread ); 271 272 // Reset io dirty bit 273 this->io.dirty = false; 274 275 // We found a thread run it 276 __run_thread(this, readyThread); 277 278 // Are we done? 279 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; 280 281 if(this->io.pending && !this->io.dirty) { 282 ready_schedule_lock(); 283 __cfa_io_flush( this ); 284 ready_schedule_unlock(); 285 } 286 287 #else 288 #warning new kernel loop 289 SEARCH: { 290 /* paranoid */ verify( ! __preemption_enabled() ); 291 292 // First, lock the scheduler since we are searching for a thread 293 ready_schedule_lock(); 294 295 // Try to get the next thread 296 readyThread = pop_fast( this->cltr ); 297 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 298 299 // If we can't find a thread, might as well flush any outstanding I/O 300 if(this->io.pending) { __cfa_io_flush( this ); } 301 302 // Spin a little on I/O, just in case 303 for(5) { 304 __maybe_io_drain( this ); 305 readyThread = pop_fast( this->cltr ); 306 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 307 } 308 309 // no luck, try stealing a few times 310 for(5) { 311 if( __maybe_io_drain( this ) ) { 312 readyThread = pop_fast( this->cltr ); 313 } else { 314 readyThread = pop_slow( this->cltr ); 315 } 316 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 317 } 318 319 // still no luck, search for a thread 320 readyThread = pop_search( this->cltr ); 321 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 322 323 // Don't block if we are done 324 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) { 325 ready_schedule_unlock(); 326 break MAIN_LOOP; 327 } 328 329 __STATS( __tls_stats()->ready.sleep.halts++; ) 330 331 // Push self to idle stack 332 ready_schedule_unlock(); 333 mark_idle(this->cltr->procs, * this); 334 ready_schedule_lock(); 335 336 // Confirm the ready-queue is empty 337 __maybe_io_drain( this ); 338 readyThread = pop_search( this->cltr ); 339 ready_schedule_unlock(); 340 341 if( readyThread ) { 342 // A thread was found, cancel the halt 343 mark_awake(this->cltr->procs, * this); 344 345 __STATS( __tls_stats()->ready.sleep.cancels++; ) 346 347 // continue the main loop 348 break SEARCH; 349 } 350 351 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); ) 352 __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle); 353 354 { 355 eventfd_t val; 356 ssize_t ret = read( this->idle, &val, sizeof(val) ); 357 if(ret < 0) { 358 switch((int)errno) { 359 case EAGAIN: 360 #if EAGAIN != EWOULDBLOCK 361 case EWOULDBLOCK: 362 #endif 363 case EINTR: 364 // No need to do anything special here, just assume it's a legitimate wake-up 365 break; 366 default: 367 abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) ); 368 } 369 } 370 } 371 372 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); ) 373 374 // We were woken up, remove self from idle 375 mark_awake(this->cltr->procs, * this); 376 377 // DON'T just proceed, start looking again 378 continue MAIN_LOOP; 379 } 380 359 381 RUN_THREAD: 360 382 /* paranoid */ verify( ! __preemption_enabled() ); … … 371 393 372 394 if(this->io.pending && !this->io.dirty) { 373 __cfa_io_flush( this , 0);395 __cfa_io_flush( this ); 374 396 } 375 397 … … 381 403 382 404 __cfadbg_print_safe(runtime_core, "Kernel : core %p stopping\n", this); 383 }384 385 for(int i = 0; !available(future); i++) {386 if(i > 1000) __cfaabi_dbg_write( "ERROR: kernel has bin spinning on a flush after exit loop.\n", 60);387 __cfa_io_flush( this, 1 );388 405 } 389 406 … … 749 766 750 767 // Check if there is a sleeping processor 751 int fd = __atomic_load_n(&this->procs.fd, __ATOMIC_SEQ_CST); 768 processor * p; 769 unsigned idle; 770 unsigned total; 771 [idle, total, p] = query_idles(this->procs); 752 772 753 773 // If no one is sleeping, we are done 754 if( fd== 0 ) return;774 if( idle == 0 ) return; 755 775 756 776 // We found a processor, wake it up 757 777 eventfd_t val; 758 778 val = 1; 759 eventfd_write( fd, val );779 eventfd_write( p->idle, val ); 760 780 761 781 #if !defined(__CFA_NO_STATISTICS__) … … 782 802 eventfd_t val; 783 803 val = 1; 784 eventfd_write( this->idle _fd, val );804 eventfd_write( this->idle, val ); 785 805 __enable_interrupts_checked(); 786 806 } 787 807 788 static void idle_sleep(processor * this, io_future_t & future, iovec & iov) { 789 #if !defined(CFA_WITH_IO_URING_IDLE) 790 #if !defined(__CFA_NO_STATISTICS__) 791 if(this->print_halts) { 792 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); 793 } 794 #endif 795 796 __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle_fd); 797 798 { 799 eventfd_t val; 800 ssize_t ret = read( this->idle_fd, &val, sizeof(val) ); 801 if(ret < 0) { 802 switch((int)errno) { 803 case EAGAIN: 804 #if EAGAIN != EWOULDBLOCK 805 case EWOULDBLOCK: 806 #endif 807 case EINTR: 808 // No need to do anything special here, just assume it's a legitimate wake-up 809 break; 810 default: 811 abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) ); 812 } 813 } 814 } 815 816 #if !defined(__CFA_NO_STATISTICS__) 817 if(this->print_halts) { 818 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); 819 } 820 #endif 821 #else 822 // Do we already have a pending read 823 if(available(future)) { 824 // There is no pending read, we need to add one 825 reset(future); 826 827 __kernel_read(this, future, iov, this->idle_fd ); 828 } 829 830 __cfa_io_flush( this, 1 ); 831 #endif 832 } 833 834 static bool mark_idle(__cluster_proc_list & this, processor & proc) { 835 /* paranoid */ verify( ! __preemption_enabled() ); 836 if(!try_lock( this )) return false; 808 static void mark_idle(__cluster_proc_list & this, processor & proc) { 809 /* paranoid */ verify( ! __preemption_enabled() ); 810 lock( this ); 837 811 this.idle++; 838 812 /* paranoid */ verify( this.idle <= this.total ); 839 813 remove(proc); 840 814 insert_first(this.idles, proc); 841 842 __atomic_store_n(&this.fd, proc.idle_fd, __ATOMIC_SEQ_CST);843 815 unlock( this ); 844 816 /* paranoid */ verify( ! __preemption_enabled() ); 845 846 return true;847 817 } 848 818 … … 854 824 remove(proc); 855 825 insert_last(this.actives, proc); 856 857 {858 int fd = 0;859 if(!this.idles`isEmpty) fd = this.idles`first.idle_fd;860 __atomic_store_n(&this.fd, fd, __ATOMIC_SEQ_CST);861 }862 863 826 unlock( this ); 827 /* paranoid */ verify( ! __preemption_enabled() ); 828 } 829 830 static [unsigned idle, unsigned total, * processor] query_idles( & __cluster_proc_list this ) { 831 /* paranoid */ verify( ! __preemption_enabled() ); 832 /* paranoid */ verify( ready_schedule_islocked() ); 833 834 for() { 835 uint64_t l = __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST); 836 if( 1 == (l % 2) ) { Pause(); continue; } 837 unsigned idle = this.idle; 838 unsigned total = this.total; 839 processor * proc = &this.idles`first; 840 // Compiler fence is unnecessary, but gcc-8 and older incorrectly reorder code without it 841 asm volatile("": : :"memory"); 842 if(l != __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST)) { Pause(); continue; } 843 return [idle, total, proc]; 844 } 845 846 /* paranoid */ verify( ready_schedule_islocked() ); 864 847 /* paranoid */ verify( ! __preemption_enabled() ); 865 848 } … … 923 906 if(head == tail) return false; 924 907 #if OLD_MAIN 925 926 927 908 ready_schedule_lock(); 909 ret = __cfa_io_drain( proc ); 910 ready_schedule_unlock(); 928 911 #else 929 912 ret = __cfa_io_drain( proc ); 930 913 #endif 931 914 #endif 932 915 return ret;
Note:
See TracChangeset
for help on using the changeset viewer.
