Changeset 2fafe7e
- Timestamp:
- Aug 14, 2020, 3:12:02 PM (5 years ago)
- Branches:
- ADT, arm-eh, ast-experimental, enum, forall-pointer-decay, jacob/cs343-translation, master, new-ast, new-ast-unique-expr, pthread-emulation, qualifiedEnum
- Children:
- dcb5f8d
- Parents:
- 7fdae38
- Location:
- libcfa/src/concurrency
- Files:
-
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
libcfa/src/concurrency/io.cfa
r7fdae38 r2fafe7e 41 41 #include "kernel/fwd.hfa" 42 42 #include "io/types.hfa" 43 44 // returns true of acquired as leader or second leader 45 static inline bool try_lock( __leaderlock_t & this ) { 46 const uintptr_t thrd = 1z | (uintptr_t)active_thread(); 47 bool block; 48 disable_interrupts(); 49 for() { 50 struct $thread * expected = this.value; 51 if( 1p != expected && 0p != expected ) { 52 /* paranoid */ verify( thrd != (uintptr_t)expected ); // We better not already be the next leader 53 enable_interrupts( __cfaabi_dbg_ctx ); 54 return false; 55 } 56 struct $thread * desired; 57 if( 0p == expected ) { 58 // If the lock isn't locked acquire it, no need to block 59 desired = 1p; 60 block = false; 61 } 62 else { 63 // If the lock is already locked try becomming the next leader 64 desired = (struct $thread *)thrd; 65 block = true; 66 } 67 if( __atomic_compare_exchange_n(&this.value, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) break; 68 } 69 if( block ) { 70 enable_interrupts( __cfaabi_dbg_ctx ); 71 park( __cfaabi_dbg_ctx ); 72 disable_interrupts(); 73 } 74 return true; 75 } 76 77 static inline bool next( __leaderlock_t & this ) { 78 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 79 struct $thread * nextt; 80 for() { 81 struct $thread * expected = this.value; 82 /* paranoid */ verify( (1 & (uintptr_t)expected) == 1 ); // The lock better be locked 83 84 struct $thread * desired; 85 if( 1p == expected ) { 86 // No next leader, just unlock 87 desired = 0p; 88 nextt = 0p; 89 } 90 else { 91 // There is a next leader, remove but keep locked 92 desired = 1p; 93 nextt = (struct $thread *)(~1z & (uintptr_t)expected); 94 } 95 if( __atomic_compare_exchange_n(&this.value, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) ) break; 96 } 97 98 if(nextt) { 99 unpark( nextt __cfaabi_dbg_ctx2 ); 100 enable_interrupts( __cfaabi_dbg_ctx ); 101 return true; 102 } 103 enable_interrupts( __cfaabi_dbg_ctx ); 104 return false; 105 } 43 106 44 107 //============================================================================================= … … 297 360 298 361 block++; 299 if( try_lock(ring.submit_q.lock __cfaabi_dbg_ctx2) ) { 300 __release_consumed_submission( ring ); 301 unlock( ring.submit_q.lock ); 302 } 303 else { 362 363 __u32 released = __release_consumed_submission( ring ); 364 if( released == 0 ) { 304 365 yield(); 305 366 } … … 334 395 __u32 picked = __submit_to_ready_array( ring, idx, mask ); 335 396 336 for() { 337 yield(); 338 339 // If some one else collected our index, we are done 340 #warning ABA problem 341 if( ring.submit_q.ready[picked] != idx ) { 397 #if defined(LEADER_LOCK) 398 if( !try_lock(ring.submit_q.submit_lock) ) { 342 399 __STATS__( false, 343 400 io.submit_q.helped += 1; … … 345 402 return; 346 403 } 347 348 if( try_lock(ring.submit_q.lock __cfaabi_dbg_ctx2) ) { 404 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 405 __STATS__( true, 406 io.submit_q.leader += 1; 407 ) 408 #else 409 for() { 410 yield(); 411 412 if( try_lock(ring.submit_q.submit_lock __cfaabi_dbg_ctx2) ) { 413 __STATS__( false, 414 io.submit_q.leader += 1; 415 ) 416 break; 417 } 418 419 // If some one else collected our index, we are done 420 #warning ABA problem 421 if( ring.submit_q.ready[picked] != idx ) { 422 __STATS__( false, 423 io.submit_q.helped += 1; 424 ) 425 return; 426 } 427 349 428 __STATS__( false, 350 io.submit_q. leader+= 1;429 io.submit_q.busy += 1; 351 430 ) 352 break; 353 } 354 355 __STATS__( false, 356 io.submit_q.busy += 1; 357 ) 358 } 431 } 432 #endif 359 433 360 434 // We got the lock … … 365 439 int ret = __io_uring_enter( ring, to_submit, false ); 366 440 367 unlock(ring.submit_q.lock); 441 #if defined(LEADER_LOCK) 442 /* paranoid */ verify( ! kernelTLS.preemption_state.enabled ); 443 next(ring.submit_q.submit_lock); 444 #else 445 unlock(ring.submit_q.submit_lock); 446 #endif 368 447 if( ret < 0 ) return; 369 448 … … 380 459 else { 381 460 // get mutual exclusion 382 lock(ring.submit_q.lock __cfaabi_dbg_ctx2); 461 #if defined(LEADER_LOCK) 462 while(!try_lock(ring.submit_q.submit_lock)); 463 #else 464 lock(ring.submit_q.submit_lock __cfaabi_dbg_ctx2); 465 #endif 383 466 384 467 /* paranoid */ verifyf( ring.submit_q.sqes[ idx ].user_data != 0, … … 418 501 __release_consumed_submission( ring ); 419 502 420 unlock(ring.submit_q.lock); 503 #if defined(LEADER_LOCK) 504 next(ring.submit_q.submit_lock); 505 #else 506 unlock(ring.submit_q.submit_lock); 507 #endif 421 508 422 509 __cfadbg_print_safe( io, "Kernel I/O : Performed io_submit for %p, returned %d\n", active_thread(), ret ); -
libcfa/src/concurrency/io/setup.cfa
r7fdae38 r2fafe7e 378 378 } 379 379 380 (sq. lock){};380 (sq.submit_lock){}; 381 381 (sq.release_lock){}; 382 382 -
libcfa/src/concurrency/io/types.hfa
r7fdae38 r2fafe7e 22 22 23 23 #include "bits/locks.hfa" 24 25 #define LEADER_LOCK 26 struct __leaderlock_t { 27 struct $thread * volatile value; // ($thread) next_leader | (bool:1) is_locked 28 }; 29 30 static inline void ?{}( __leaderlock_t & this ) { this.value = 0p; } 24 31 25 32 //----------------------------------------------------------------------- … … 49 56 __u32 ready_cnt; 50 57 51 __spinlock_t lock; 52 __spinlock_t release_lock; 58 #if defined(LEADER_LOCK) 59 __leaderlock_t submit_lock; 60 #else 61 __spinlock_t submit_lock; 62 #endif 63 __spinlock_t release_lock; 53 64 54 65 // A buffer of sqes (not the actual ring)
Note: See TracChangeset
for help on using the changeset viewer.