- File:
-
- 1 edited
-
libcfa/src/concurrency/kernel.cfa (modified) (17 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/kernel.cfa
r6ddef36 r24e321c 34 34 #include "strstream.hfa" 35 35 #include "device/cpu.hfa" 36 #include "io/types.hfa"37 36 38 37 //Private includes … … 125 124 static void __wake_one(cluster * cltr); 126 125 127 static void idle_sleep(processor * proc, io_future_t & future, char buf[]); 128 static bool mark_idle (__cluster_proc_list & idles, processor & proc); 126 static void mark_idle (__cluster_proc_list & idles, processor & proc); 129 127 static void mark_awake(__cluster_proc_list & idles, processor & proc); 128 static [unsigned idle, unsigned total, * processor] query_idles( & __cluster_proc_list idles ); 130 129 131 130 extern void __cfa_io_start( processor * ); 132 131 extern bool __cfa_io_drain( processor * ); 133 extern bool __cfa_io_flush( processor *, bool wait);132 extern void __cfa_io_flush( processor * ); 134 133 extern void __cfa_io_stop ( processor * ); 135 134 static inline bool __maybe_io_drain( processor * ); 136 137 #if defined(IO_URING_IDLE) && defined(CFA_HAVE_LINUX_IO_URING_H)138 extern bool __kernel_read(processor * proc, io_future_t & future, char buf[], int fd);139 #endif140 135 141 136 extern void __disable_interrupts_hard(); … … 153 148 /* paranoid */ verify( __preemption_enabled() ); 154 149 } 155 156 150 157 151 //============================================================================================= … … 169 163 verify(this); 170 164 171 io_future_t future; // used for idle sleep when io_uring is present172 future.self.ptr = 1p; // mark it as already fulfilled so we know if there is a pending request or not173 char buf[sizeof(uint64_t)];174 175 165 __cfa_io_start( this ); 176 166 … … 206 196 207 197 if( !readyThread ) { 208 __cfa_io_flush( this, false ); 209 198 __cfa_io_flush( this ); 210 199 readyThread = __next_thread_slow( this->cltr ); 211 200 } … … 221 210 222 211 // Push self to idle stack 223 if(!mark_idle(this->cltr->procs, * this)) continue MAIN_LOOP;212 mark_idle(this->cltr->procs, * this); 224 213 225 214 // Confirm the ready-queue is empty … … 237 226 } 238 227 239 idle_sleep( this, future, buf ); 240 241 // We were woken up, remove self from idle 242 mark_awake(this->cltr->procs, * this); 243 244 // DON'T just proceed, start looking again 245 continue MAIN_LOOP; 246 } 247 248 /* paranoid */ verify( readyThread ); 249 250 // Reset io dirty bit 251 this->io.dirty = false; 252 253 // We found a thread run it 254 __run_thread(this, readyThread); 255 256 // Are we done? 257 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; 258 259 if(this->io.pending && !this->io.dirty) { 260 __cfa_io_flush( this, false ); 261 } 262 263 #else 264 #warning new kernel loop 265 SEARCH: { 266 /* paranoid */ verify( ! __preemption_enabled() ); 267 268 // First, lock the scheduler since we are searching for a thread 269 ready_schedule_lock(); 270 271 // Try to get the next thread 272 readyThread = pop_fast( this->cltr ); 273 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 274 275 // If we can't find a thread, might as well flush any outstanding I/O 276 if(this->io.pending) { __cfa_io_flush( this, false ); } 277 278 // Spin a little on I/O, just in case 279 for(5) { 280 __maybe_io_drain( this ); 281 readyThread = pop_fast( this->cltr ); 282 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 283 } 284 285 // no luck, try stealing a few times 286 for(5) { 287 if( __maybe_io_drain( this ) ) { 288 readyThread = pop_fast( this->cltr ); 289 } else { 290 readyThread = pop_slow( this->cltr ); 228 #if !defined(__CFA_NO_STATISTICS__) 229 if(this->print_halts) { 230 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); 291 231 } 292 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 293 } 294 295 // still no luck, search for a thread 296 readyThread = pop_search( this->cltr ); 297 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 298 299 // Don't block if we are done 300 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) { 301 ready_schedule_unlock(); 302 break MAIN_LOOP; 303 } 304 305 __STATS( __tls_stats()->ready.sleep.halts++; ) 306 307 // Push self to idle stack 308 ready_schedule_unlock(); 309 if(!mark_idle(this->cltr->procs, * this)) goto SEARCH; 310 ready_schedule_lock(); 311 312 // Confirm the ready-queue is empty 313 __maybe_io_drain( this ); 314 readyThread = pop_search( this->cltr ); 315 ready_schedule_unlock(); 316 317 if( readyThread ) { 318 // A thread was found, cancel the halt 319 mark_awake(this->cltr->procs, * this); 320 321 __STATS( __tls_stats()->ready.sleep.cancels++; ) 322 323 // continue the main loop 324 break SEARCH; 325 } 326 327 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); ) 328 __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle_fd); 232 #endif 233 234 __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle); 329 235 330 236 { 331 237 eventfd_t val; 332 ssize_t ret = read( this->idle _fd, &val, sizeof(val) );238 ssize_t ret = read( this->idle, &val, sizeof(val) ); 333 239 if(ret < 0) { 334 240 switch((int)errno) { … … 346 252 } 347 253 348 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); ) 254 #if !defined(__CFA_NO_STATISTICS__) 255 if(this->print_halts) { 256 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); 257 } 258 #endif 349 259 350 260 // We were woken up, remove self from idle … … 355 265 } 356 266 267 /* paranoid */ verify( readyThread ); 268 269 // Reset io dirty bit 270 this->io.dirty = false; 271 272 // We found a thread run it 273 __run_thread(this, readyThread); 274 275 // Are we done? 276 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; 277 278 if(this->io.pending && !this->io.dirty) { 279 __cfa_io_flush( this ); 280 } 281 282 #else 283 #warning new kernel loop 284 SEARCH: { 285 /* paranoid */ verify( ! __preemption_enabled() ); 286 287 // First, lock the scheduler since we are searching for a thread 288 ready_schedule_lock(); 289 290 // Try to get the next thread 291 readyThread = pop_fast( this->cltr ); 292 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 293 294 // If we can't find a thread, might as well flush any outstanding I/O 295 if(this->io.pending) { __cfa_io_flush( this ); } 296 297 // Spin a little on I/O, just in case 298 for(5) { 299 __maybe_io_drain( this ); 300 readyThread = pop_fast( this->cltr ); 301 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 302 } 303 304 // no luck, try stealing a few times 305 for(5) { 306 if( __maybe_io_drain( this ) ) { 307 readyThread = pop_fast( this->cltr ); 308 } else { 309 readyThread = pop_slow( this->cltr ); 310 } 311 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 312 } 313 314 // still no luck, search for a thread 315 readyThread = pop_search( this->cltr ); 316 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 317 318 // Don't block if we are done 319 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; 320 321 __STATS( __tls_stats()->ready.sleep.halts++; ) 322 323 // Push self to idle stack 324 ready_schedule_unlock(); 325 mark_idle(this->cltr->procs, * this); 326 ready_schedule_lock(); 327 328 // Confirm the ready-queue is empty 329 __maybe_io_drain( this ); 330 readyThread = pop_search( this->cltr ); 331 ready_schedule_unlock(); 332 333 if( readyThread ) { 334 // A thread was found, cancel the halt 335 mark_awake(this->cltr->procs, * this); 336 337 __STATS( __tls_stats()->ready.sleep.cancels++; ) 338 339 // continue the main loop 340 break SEARCH; 341 } 342 343 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); ) 344 __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle); 345 346 { 347 eventfd_t val; 348 ssize_t ret = read( this->idle, &val, sizeof(val) ); 349 if(ret < 0) { 350 switch((int)errno) { 351 case EAGAIN: 352 #if EAGAIN != EWOULDBLOCK 353 case EWOULDBLOCK: 354 #endif 355 case EINTR: 356 // No need to do anything special here, just assume it's a legitimate wake-up 357 break; 358 default: 359 abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) ); 360 } 361 } 362 } 363 364 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); ) 365 366 // We were woken up, remove self from idle 367 mark_awake(this->cltr->procs, * this); 368 369 // DON'T just proceed, start looking again 370 continue MAIN_LOOP; 371 } 372 357 373 RUN_THREAD: 358 374 /* paranoid */ verify( ! __preemption_enabled() ); … … 369 385 370 386 if(this->io.pending && !this->io.dirty) { 371 __cfa_io_flush( this , false);387 __cfa_io_flush( this ); 372 388 } 373 389 … … 742 758 743 759 // Check if there is a sleeping processor 744 int fd = __atomic_load_n(&this->procs.fd, __ATOMIC_SEQ_CST); 760 processor * p; 761 unsigned idle; 762 unsigned total; 763 [idle, total, p] = query_idles(this->procs); 745 764 746 765 // If no one is sleeping, we are done 747 if( fd== 0 ) return;766 if( idle == 0 ) return; 748 767 749 768 // We found a processor, wake it up 750 769 eventfd_t val; 751 770 val = 1; 752 eventfd_write( fd, val );771 eventfd_write( p->idle, val ); 753 772 754 773 #if !defined(__CFA_NO_STATISTICS__) … … 775 794 eventfd_t val; 776 795 val = 1; 777 eventfd_write( this->idle _fd, val );796 eventfd_write( this->idle, val ); 778 797 __enable_interrupts_checked(); 779 798 } 780 799 781 static void idle_sleep(processor * this, io_future_t & future, char buf[]) { 782 #if !defined(IO_URING_IDLE) || !defined(CFA_HAVE_LINUX_IO_URING_H) 783 #if !defined(__CFA_NO_STATISTICS__) 784 if(this->print_halts) { 785 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); 786 } 787 #endif 788 789 __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle_fd); 790 791 { 792 eventfd_t val; 793 ssize_t ret = read( this->idle_fd, &val, sizeof(val) ); 794 if(ret < 0) { 795 switch((int)errno) { 796 case EAGAIN: 797 #if EAGAIN != EWOULDBLOCK 798 case EWOULDBLOCK: 799 #endif 800 case EINTR: 801 // No need to do anything special here, just assume it's a legitimate wake-up 802 break; 803 default: 804 abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) ); 805 } 806 } 807 } 808 809 #if !defined(__CFA_NO_STATISTICS__) 810 if(this->print_halts) { 811 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); 812 } 813 #endif 814 #else 815 #if !defined(CFA_HAVE_IORING_OP_READ) 816 #error this is only implemented if the read is present 817 #endif 818 // Do we already have a pending read 819 if(available(future)) { 820 // There is no pending read, we need to add one 821 reset(future); 822 823 __kernel_read(this, future, buf, this->idle_fd ); 824 } 825 826 __cfa_io_flush( this, true ); 827 #endif 828 } 829 830 static bool mark_idle(__cluster_proc_list & this, processor & proc) { 831 /* paranoid */ verify( ! __preemption_enabled() ); 832 if(!try_lock( this )) return false; 800 static void mark_idle(__cluster_proc_list & this, processor & proc) { 801 /* paranoid */ verify( ! __preemption_enabled() ); 802 lock( this ); 833 803 this.idle++; 834 804 /* paranoid */ verify( this.idle <= this.total ); 835 805 remove(proc); 836 806 insert_first(this.idles, proc); 837 838 __atomic_store_n(&this.fd, proc.idle_fd, __ATOMIC_SEQ_CST);839 807 unlock( this ); 840 808 /* paranoid */ verify( ! __preemption_enabled() ); 841 842 return true;843 809 } 844 810 … … 850 816 remove(proc); 851 817 insert_last(this.actives, proc); 852 853 {854 int fd = 0;855 if(!this.idles`isEmpty) fd = this.idles`first.idle_fd;856 __atomic_store_n(&this.fd, fd, __ATOMIC_SEQ_CST);857 }858 859 818 unlock( this ); 819 /* paranoid */ verify( ! __preemption_enabled() ); 820 } 821 822 static [unsigned idle, unsigned total, * processor] query_idles( & __cluster_proc_list this ) { 823 /* paranoid */ verify( ! __preemption_enabled() ); 824 /* paranoid */ verify( ready_schedule_islocked() ); 825 826 for() { 827 uint64_t l = __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST); 828 if( 1 == (l % 2) ) { Pause(); continue; } 829 unsigned idle = this.idle; 830 unsigned total = this.total; 831 processor * proc = &this.idles`first; 832 // Compiler fence is unnecessary, but gcc-8 and older incorrectly reorder code without it 833 asm volatile("": : :"memory"); 834 if(l != __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST)) { Pause(); continue; } 835 return [idle, total, proc]; 836 } 837 838 /* paranoid */ verify( ready_schedule_islocked() ); 860 839 /* paranoid */ verify( ! __preemption_enabled() ); 861 840 } … … 919 898 if(head == tail) return false; 920 899 #if OLD_MAIN 921 922 923 900 ready_schedule_lock(); 901 ret = __cfa_io_drain( proc ); 902 ready_schedule_unlock(); 924 903 #else 925 904 ret = __cfa_io_drain( proc ); 926 905 #endif 927 906 #endif 928 907 return ret; … … 960 939 /* paranoid */ verifyf( it, "Unexpected null iterator, at index %u of %u\n", i, count); 961 940 /* paranoid */ verify( it->local_data->this_stats ); 962 // __print_stats( it->local_data->this_stats, cltr->print_stats, "Processor", it->name, (void*)it );963 941 __tally_stats( cltr->stats, it->local_data->this_stats ); 964 942 it = &(*it)`next; … … 970 948 // this doesn't solve all problems but does solve many 971 949 // so it's probably good enough 972 disable_interrupts();973 950 uint_fast32_t last_size = ready_mutate_lock(); 974 951 … … 978 955 // Unlock the RWlock 979 956 ready_mutate_unlock( last_size ); 980 enable_interrupts();981 957 } 982 958
Note:
See TracChangeset
for help on using the changeset viewer.
