Changeset 7770cc8 for libcfa/src/concurrency/kernel.cfa
- Timestamp:
- Nov 24, 2021, 9:47:56 PM (2 years ago)
- Branches:
- ADT, ast-experimental, enum, master, pthread-emulation, qualifiedEnum
- Children:
- 5235d49
- Parents:
- 94647b0 (diff), 3cc1111 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the(diff)links above to see all the changes relative to each parent. - File:
-
- 1 edited
-
libcfa/src/concurrency/kernel.cfa (modified) (17 diffs)
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/kernel.cfa
r94647b0 r7770cc8 34 34 #include "strstream.hfa" 35 35 #include "device/cpu.hfa" 36 #include "io/types.hfa" 36 37 37 38 //Private includes … … 124 125 static void __wake_one(cluster * cltr); 125 126 126 static void mark_idle (__cluster_proc_list & idles, processor & proc); 127 static void idle_sleep(processor * proc, io_future_t & future, char buf[]); 128 static bool mark_idle (__cluster_proc_list & idles, processor & proc); 127 129 static void mark_awake(__cluster_proc_list & idles, processor & proc); 128 static [unsigned idle, unsigned total, * processor] query_idles( & __cluster_proc_list idles );129 130 130 131 extern void __cfa_io_start( processor * ); 131 132 extern bool __cfa_io_drain( processor * ); 132 extern void __cfa_io_flush( processor *);133 extern bool __cfa_io_flush( processor *, bool wait ); 133 134 extern void __cfa_io_stop ( processor * ); 134 135 static inline bool __maybe_io_drain( processor * ); 136 137 #if defined(IO_URING_IDLE) && defined(CFA_HAVE_LINUX_IO_URING_H) 138 extern bool __kernel_read(processor * proc, io_future_t & future, char buf[], int fd); 139 #endif 135 140 136 141 extern void __disable_interrupts_hard(); … … 148 153 /* paranoid */ verify( __preemption_enabled() ); 149 154 } 155 150 156 151 157 //============================================================================================= … … 163 169 verify(this); 164 170 171 io_future_t future; // used for idle sleep when io_uring is present 172 future.self.ptr = 1p; // mark it as already fulfilled so we know if there is a pending request or not 173 char buf[sizeof(uint64_t)]; 174 165 175 __cfa_io_start( this ); 166 176 … … 196 206 197 207 if( !readyThread ) { 198 __cfa_io_flush( this ); 208 __cfa_io_flush( this, false ); 209 199 210 readyThread = __next_thread_slow( this->cltr ); 200 211 } … … 210 221 211 222 // Push self to idle stack 212 mark_idle(this->cltr->procs, * this);223 if(!mark_idle(this->cltr->procs, * this)) continue MAIN_LOOP; 213 224 214 225 // Confirm the ready-queue is empty … … 226 237 } 227 238 228 #if !defined(__CFA_NO_STATISTICS__) 229 if(this->print_halts) { 230 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); 239 idle_sleep( this, future, buf ); 240 241 // We were woken up, remove self from idle 242 mark_awake(this->cltr->procs, * this); 243 244 // DON'T just proceed, start looking again 245 continue MAIN_LOOP; 246 } 247 248 /* paranoid */ verify( readyThread ); 249 250 // Reset io dirty bit 251 this->io.dirty = false; 252 253 // We found a thread run it 254 __run_thread(this, readyThread); 255 256 // Are we done? 257 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP; 258 259 if(this->io.pending && !this->io.dirty) { 260 __cfa_io_flush( this, false ); 261 } 262 263 #else 264 #warning new kernel loop 265 SEARCH: { 266 /* paranoid */ verify( ! __preemption_enabled() ); 267 268 // First, lock the scheduler since we are searching for a thread 269 ready_schedule_lock(); 270 271 // Try to get the next thread 272 readyThread = pop_fast( this->cltr ); 273 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 274 275 // If we can't find a thread, might as well flush any outstanding I/O 276 if(this->io.pending) { __cfa_io_flush( this, false ); } 277 278 // Spin a little on I/O, just in case 279 for(5) { 280 __maybe_io_drain( this ); 281 readyThread = pop_fast( this->cltr ); 282 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 283 } 284 285 // no luck, try stealing a few times 286 for(5) { 287 if( __maybe_io_drain( this ) ) { 288 readyThread = pop_fast( this->cltr ); 289 } else { 290 readyThread = pop_slow( this->cltr ); 231 291 } 232 #endif 233 234 __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle); 292 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 293 } 294 295 // still no luck, search for a thread 296 readyThread = pop_search( this->cltr ); 297 if(readyThread) { ready_schedule_unlock(); break SEARCH; } 298 299 // Don't block if we are done 300 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) { 301 ready_schedule_unlock(); 302 break MAIN_LOOP; 303 } 304 305 __STATS( __tls_stats()->ready.sleep.halts++; ) 306 307 // Push self to idle stack 308 ready_schedule_unlock(); 309 if(!mark_idle(this->cltr->procs, * this)) goto SEARCH; 310 ready_schedule_lock(); 311 312 // Confirm the ready-queue is empty 313 __maybe_io_drain( this ); 314 readyThread = pop_search( this->cltr ); 315 ready_schedule_unlock(); 316 317 if( readyThread ) { 318 // A thread was found, cancel the halt 319 mark_awake(this->cltr->procs, * this); 320 321 __STATS( __tls_stats()->ready.sleep.cancels++; ) 322 323 // continue the main loop 324 break SEARCH; 325 } 326 327 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); ) 328 __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle_fd); 235 329 236 330 { 237 331 eventfd_t val; 238 ssize_t ret = read( this->idle , &val, sizeof(val) );332 ssize_t ret = read( this->idle_fd, &val, sizeof(val) ); 239 333 if(ret < 0) { 240 334 switch((int)errno) { … … 252 346 } 253 347 254 #if !defined(__CFA_NO_STATISTICS__) 255 if(this->print_halts) { 256 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); 257 } 258 #endif 348 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); ) 259 349 260 350 // We were woken up, remove self from idle … … 265 355 } 266 356 267 /* paranoid */ verify( readyThread );268 269 // Reset io dirty bit270 this->io.dirty = false;271 272 // We found a thread run it273 __run_thread(this, readyThread);274 275 // Are we done?276 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;277 278 if(this->io.pending && !this->io.dirty) {279 __cfa_io_flush( this );280 }281 282 #else283 #warning new kernel loop284 SEARCH: {285 /* paranoid */ verify( ! __preemption_enabled() );286 287 // First, lock the scheduler since we are searching for a thread288 ready_schedule_lock();289 290 // Try to get the next thread291 readyThread = pop_fast( this->cltr );292 if(readyThread) { ready_schedule_unlock(); break SEARCH; }293 294 // If we can't find a thread, might as well flush any outstanding I/O295 if(this->io.pending) { __cfa_io_flush( this ); }296 297 // Spin a little on I/O, just in case298 for(5) {299 __maybe_io_drain( this );300 readyThread = pop_fast( this->cltr );301 if(readyThread) { ready_schedule_unlock(); break SEARCH; }302 }303 304 // no luck, try stealing a few times305 for(5) {306 if( __maybe_io_drain( this ) ) {307 readyThread = pop_fast( this->cltr );308 } else {309 readyThread = pop_slow( this->cltr );310 }311 if(readyThread) { ready_schedule_unlock(); break SEARCH; }312 }313 314 // still no luck, search for a thread315 readyThread = pop_search( this->cltr );316 if(readyThread) { ready_schedule_unlock(); break SEARCH; }317 318 // Don't block if we are done319 if( __atomic_load_n(&this->do_terminate, __ATOMIC_SEQ_CST) ) break MAIN_LOOP;320 321 __STATS( __tls_stats()->ready.sleep.halts++; )322 323 // Push self to idle stack324 ready_schedule_unlock();325 mark_idle(this->cltr->procs, * this);326 ready_schedule_lock();327 328 // Confirm the ready-queue is empty329 __maybe_io_drain( this );330 readyThread = pop_search( this->cltr );331 ready_schedule_unlock();332 333 if( readyThread ) {334 // A thread was found, cancel the halt335 mark_awake(this->cltr->procs, * this);336 337 __STATS( __tls_stats()->ready.sleep.cancels++; )338 339 // continue the main loop340 break SEARCH;341 }342 343 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); )344 __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle);345 346 {347 eventfd_t val;348 ssize_t ret = read( this->idle, &val, sizeof(val) );349 if(ret < 0) {350 switch((int)errno) {351 case EAGAIN:352 #if EAGAIN != EWOULDBLOCK353 case EWOULDBLOCK:354 #endif355 case EINTR:356 // No need to do anything special here, just assume it's a legitimate wake-up357 break;358 default:359 abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) );360 }361 }362 }363 364 __STATS( if(this->print_halts) __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); )365 366 // We were woken up, remove self from idle367 mark_awake(this->cltr->procs, * this);368 369 // DON'T just proceed, start looking again370 continue MAIN_LOOP;371 }372 373 357 RUN_THREAD: 374 358 /* paranoid */ verify( ! __preemption_enabled() ); … … 385 369 386 370 if(this->io.pending && !this->io.dirty) { 387 __cfa_io_flush( this );371 __cfa_io_flush( this, false ); 388 372 } 389 373 … … 758 742 759 743 // Check if there is a sleeping processor 760 processor * p; 761 unsigned idle; 762 unsigned total; 763 [idle, total, p] = query_idles(this->procs); 744 int fd = __atomic_load_n(&this->procs.fd, __ATOMIC_SEQ_CST); 764 745 765 746 // If no one is sleeping, we are done 766 if( idle== 0 ) return;747 if( fd == 0 ) return; 767 748 768 749 // We found a processor, wake it up 769 750 eventfd_t val; 770 751 val = 1; 771 eventfd_write( p->idle, val );752 eventfd_write( fd, val ); 772 753 773 754 #if !defined(__CFA_NO_STATISTICS__) … … 794 775 eventfd_t val; 795 776 val = 1; 796 eventfd_write( this->idle , val );777 eventfd_write( this->idle_fd, val ); 797 778 __enable_interrupts_checked(); 798 779 } 799 780 800 static void mark_idle(__cluster_proc_list & this, processor & proc) { 801 /* paranoid */ verify( ! __preemption_enabled() ); 802 lock( this ); 781 static void idle_sleep(processor * this, io_future_t & future, char buf[]) { 782 #if !defined(IO_URING_IDLE) || !defined(CFA_HAVE_LINUX_IO_URING_H) 783 #if !defined(__CFA_NO_STATISTICS__) 784 if(this->print_halts) { 785 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 0\n", this->unique_id, rdtscl()); 786 } 787 #endif 788 789 __cfadbg_print_safe(runtime_core, "Kernel : core %p waiting on eventfd %d\n", this, this->idle_fd); 790 791 { 792 eventfd_t val; 793 ssize_t ret = read( this->idle_fd, &val, sizeof(val) ); 794 if(ret < 0) { 795 switch((int)errno) { 796 case EAGAIN: 797 #if EAGAIN != EWOULDBLOCK 798 case EWOULDBLOCK: 799 #endif 800 case EINTR: 801 // No need to do anything special here, just assume it's a legitimate wake-up 802 break; 803 default: 804 abort( "KERNEL : internal error, read failure on idle eventfd, error(%d) %s.", (int)errno, strerror( (int)errno ) ); 805 } 806 } 807 } 808 809 #if !defined(__CFA_NO_STATISTICS__) 810 if(this->print_halts) { 811 __cfaabi_bits_print_safe( STDOUT_FILENO, "PH:%d - %lld 1\n", this->unique_id, rdtscl()); 812 } 813 #endif 814 #else 815 #if !defined(CFA_HAVE_IORING_OP_READ) 816 #error this is only implemented if the read is present 817 #endif 818 // Do we already have a pending read 819 if(available(future)) { 820 // There is no pending read, we need to add one 821 reset(future); 822 823 __kernel_read(this, future, buf, this->idle_fd ); 824 } 825 826 __cfa_io_flush( this, true ); 827 #endif 828 } 829 830 static bool mark_idle(__cluster_proc_list & this, processor & proc) { 831 /* paranoid */ verify( ! __preemption_enabled() ); 832 if(!try_lock( this )) return false; 803 833 this.idle++; 804 834 /* paranoid */ verify( this.idle <= this.total ); 805 835 remove(proc); 806 836 insert_first(this.idles, proc); 837 838 __atomic_store_n(&this.fd, proc.idle_fd, __ATOMIC_SEQ_CST); 807 839 unlock( this ); 808 840 /* paranoid */ verify( ! __preemption_enabled() ); 841 842 return true; 809 843 } 810 844 … … 816 850 remove(proc); 817 851 insert_last(this.actives, proc); 852 853 { 854 int fd = 0; 855 if(!this.idles`isEmpty) fd = this.idles`first.idle_fd; 856 __atomic_store_n(&this.fd, fd, __ATOMIC_SEQ_CST); 857 } 858 818 859 unlock( this ); 819 /* paranoid */ verify( ! __preemption_enabled() );820 }821 822 static [unsigned idle, unsigned total, * processor] query_idles( & __cluster_proc_list this ) {823 /* paranoid */ verify( ! __preemption_enabled() );824 /* paranoid */ verify( ready_schedule_islocked() );825 826 for() {827 uint64_t l = __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST);828 if( 1 == (l % 2) ) { Pause(); continue; }829 unsigned idle = this.idle;830 unsigned total = this.total;831 processor * proc = &this.idles`first;832 // Compiler fence is unnecessary, but gcc-8 and older incorrectly reorder code without it833 asm volatile("": : :"memory");834 if(l != __atomic_load_n(&this.lock, __ATOMIC_SEQ_CST)) { Pause(); continue; }835 return [idle, total, proc];836 }837 838 /* paranoid */ verify( ready_schedule_islocked() );839 860 /* paranoid */ verify( ! __preemption_enabled() ); 840 861 } … … 898 919 if(head == tail) return false; 899 920 #if OLD_MAIN 900 ready_schedule_lock();901 ret = __cfa_io_drain( proc );902 ready_schedule_unlock();921 ready_schedule_lock(); 922 ret = __cfa_io_drain( proc ); 923 ready_schedule_unlock(); 903 924 #else 904 925 ret = __cfa_io_drain( proc ); 905 #endif926 #endif 906 927 #endif 907 928 return ret; … … 939 960 /* paranoid */ verifyf( it, "Unexpected null iterator, at index %u of %u\n", i, count); 940 961 /* paranoid */ verify( it->local_data->this_stats ); 962 // __print_stats( it->local_data->this_stats, cltr->print_stats, "Processor", it->name, (void*)it ); 941 963 __tally_stats( cltr->stats, it->local_data->this_stats ); 942 964 it = &(*it)`next; … … 948 970 // this doesn't solve all problems but does solve many 949 971 // so it's probably good enough 972 disable_interrupts(); 950 973 uint_fast32_t last_size = ready_mutate_lock(); 951 974 … … 955 978 // Unlock the RWlock 956 979 ready_mutate_unlock( last_size ); 980 enable_interrupts(); 957 981 } 958 982
Note: See TracChangeset
for help on using the changeset viewer.
