Context Navigation

io.cfa@ 78da4ab

Visit:

ADT arm-eh ast-experimental enum forall-pointer-decay jacob/cs343-translation new-ast-unique-expr pthread-emulation qualifiedEnum

Last change on this file since 78da4ab was 78da4ab, checked in by Thierry Delisle <tdelisle@…>, 5 years ago
New implementation of io based on instance burrowing. Trying to avoid the unbounded growth of the previous flat combining approach.
Property mode set to `100644`
File size: 19.5 KB

Line
1	//
2	// Cforall Version 1.0.0 Copyright (C) 2020 University of Waterloo
3	//
4	// The contents of this file are covered under the licence agreement in the
5	// file "LICENCE" distributed with Cforall.
6	//
7	// io.cfa --
8	//
9	// Author : Thierry Delisle
10	// Created On : Thu Apr 23 17:31:00 2020
11	// Last Modified By :
12	// Last Modified On :
13	// Update Count :
14	//
15
16	#define __cforall_thread__
17
18	#if defined(__CFA_DEBUG__)
19	#define __CFA_DEBUG_PRINT_IO__
20	#define __CFA_DEBUG_PRINT_IO_CORE__
21	#endif
22
23
24	#if defined(CFA_HAVE_LINUX_IO_URING_H)
25	#define _GNU_SOURCE /* See feature_test_macros(7) */
26	#include <errno.h>
27	#include <signal.h>
28	#include <stdint.h>
29	#include <string.h>
30	#include <unistd.h>
31
32	extern "C" {
33	#include <sys/syscall.h>
34
35	#include <linux/io_uring.h>
36	}
37
38	#include "stats.hfa"
39	#include "kernel.hfa"
40	#include "kernel/fwd.hfa"
41	#include "io/types.hfa"
42
43	__attribute__((unused)) static const char * opcodes[] = {
44	"OP_NOP",
45	"OP_READV",
46	"OP_WRITEV",
47	"OP_FSYNC",
48	"OP_READ_FIXED",
49	"OP_WRITE_FIXED",
50	"OP_POLL_ADD",
51	"OP_POLL_REMOVE",
52	"OP_SYNC_FILE_RANGE",
53	"OP_SENDMSG",
54	"OP_RECVMSG",
55	"OP_TIMEOUT",
56	"OP_TIMEOUT_REMOVE",
57	"OP_ACCEPT",
58	"OP_ASYNC_CANCEL",
59	"OP_LINK_TIMEOUT",
60	"OP_CONNECT",
61	"OP_FALLOCATE",
62	"OP_OPENAT",
63	"OP_CLOSE",
64	"OP_FILES_UPDATE",
65	"OP_STATX",
66	"OP_READ",
67	"OP_WRITE",
68	"OP_FADVISE",
69	"OP_MADVISE",
70	"OP_SEND",
71	"OP_RECV",
72	"OP_OPENAT2",
73	"OP_EPOLL_CTL",
74	"OP_SPLICE",
75	"OP_PROVIDE_BUFFERS",
76	"OP_REMOVE_BUFFERS",
77	"OP_TEE",
78	"INVALID_OP"
79	};
80
81	//=============================================================================================
82	// I/O Syscall
83	//=============================================================================================
84	static int __io_uring_enter( struct $io_context & ctx, unsigned to_submit, bool get ) {
85	bool need_sys_to_submit = false;
86	bool need_sys_to_complete = false;
87	unsigned flags = 0;
88
89	TO_SUBMIT:
90	if( to_submit > 0 ) {
91	if( !(ctx.ring_flags & IORING_SETUP_SQPOLL) ) {
92	need_sys_to_submit = true;
93	break TO_SUBMIT;
94	}
95	if( (*ctx.sq.flags) & IORING_SQ_NEED_WAKEUP ) {
96	need_sys_to_submit = true;
97	flags \|= IORING_ENTER_SQ_WAKEUP;
98	}
99	}
100
101	if( get && !(ctx.ring_flags & IORING_SETUP_SQPOLL) ) {
102	flags \|= IORING_ENTER_GETEVENTS;
103	if( (ctx.ring_flags & IORING_SETUP_IOPOLL) ) {
104	need_sys_to_complete = true;
105	}
106	}
107
108	int ret = 0;
109	if( need_sys_to_submit \|\| need_sys_to_complete ) {
110	__cfadbg_print_safe(io_core, "Kernel I/O : IO_URING enter %d %u %u\n", ctx.fd, to_submit, flags);
111	ret = syscall( __NR_io_uring_enter, ctx.fd, to_submit, 0, flags, (sigset_t *)0p, _NSIG / 8);
112	__cfadbg_print_safe(io_core, "Kernel I/O : IO_URING %d returned %d\n", ctx.fd, ret);
113	}
114
115	// Memory barrier
116	__atomic_thread_fence( __ATOMIC_SEQ_CST );
117	return ret;
118	}
119
120	//=============================================================================================
121	// I/O Polling
122	//=============================================================================================
123	static inline unsigned __flush( struct $io_context & );
124	static inline __u32 __release_sqes( struct $io_context & );
125
126	static [int, bool] __drain_io( & struct $io_context ctx ) {
127	unsigned to_submit = __flush( ctx );
128	int ret = __io_uring_enter( ctx, to_submit, true );
129	if( ret < 0 ) {
130	switch((int)errno) {
131	case EAGAIN:
132	case EINTR:
133	case EBUSY:
134	return [0, true];
135	break;
136	default:
137	abort( "KERNEL ERROR: IO_URING SYSCALL - (%d) %s\n", (int)errno, strerror(errno) );
138	}
139	}
140
141	// update statistics
142	if (to_submit > 0) {
143	__STATS__( false,
144	if( to_submit > 0 ) {
145	io.submit_q.submit_avg.rdy += to_submit;
146	io.submit_q.submit_avg.csm += ret;
147	io.submit_q.submit_avg.cnt += 1;
148	}
149	)
150	/* paranoid / verify( ctx.sq.to_submit <= ctx.sq.num );
151
152	/* paranoid */ verify( ctx.sq.to_submit >= ret );
153	ctx.sq.to_submit -= ret;
154
155	/* paranoid / verify( ctx.sq.to_submit <= ctx.sq.num );
156
157	if(ret) {
158	__cfadbg_print_safe(io, "Kernel I/O : %u submitted to io_uring\n", ret);
159	}
160	}
161
162	// Release the consumed SQEs
163	__release_sqes( ctx );
164
165	// Drain the queue
166	unsigned head = *ctx.cq.head;
167	unsigned tail = *ctx.cq.tail;
168	const __u32 mask = *ctx.cq.mask;
169
170	// Nothing was new return 0
171	if (head == tail) {
172	return [0, to_submit > 0];
173	}
174
175	__u32 count = tail - head;
176	/* paranoid */ verify( count != 0 );
177	for(i; count) {
178	unsigned idx = (head + i) & mask;
179	volatile struct io_uring_cqe & cqe = ctx.cq.cqes[idx];
180
181	/* paranoid */ verify(&cqe);
182
183	struct io_future_t * future = (struct io_future_t *)(uintptr_t)cqe.user_data;
184	__cfadbg_print_safe( io, "Kernel I/O : Syscall completed : cqe %p, result %d for %p\n", &cqe, cqe.res, future );
185
186	fulfil( *future, cqe.res );
187	}
188
189	if(count) {
190	__cfadbg_print_safe(io, "Kernel I/O : %u completed\n", count);
191	}
192
193	// Mark to the kernel that the cqe has been seen
194	// Ensure that the kernel only sees the new value of the head index after the CQEs have been read.
195	__atomic_store_n( ctx.cq.head, head + count, __ATOMIC_SEQ_CST );
196
197	return [count, count > 0 \|\| to_submit > 0];
198	}
199
200	void main( $io_context & this ) {
201	__cfadbg_print_safe(io_core, "Kernel I/O : IO poller %d (%p) ready\n", this.fd, &this);
202
203	const int reset_cnt = 5;
204	int reset = reset_cnt;
205	// Then loop until we need to start
206	LOOP:
207	while() {
208	waitfor( ^?{} : this) {
209	break LOOP;
210	}
211	or else {}
212
213	// Drain the io
214	int count;
215	bool again;
216	[count, again] = __drain_io( this );
217
218	if(!again) reset--;
219
220	// Update statistics
221	__STATS__( false,
222	io.complete_q.completed_avg.val += count;
223	io.complete_q.completed_avg.cnt += 1;
224	)
225
226	// If we got something, just yield and check again
227	if(reset > 1) {
228	yield();
229	continue LOOP;
230	}
231
232	// We alread failed to find completed entries a few time.
233	if(reset == 1) {
234	// Rearm the context so it can block
235	// but don't block right away
236	// we need to retry one last time in case
237	// something completed just now
238	__ioctx_prepare_block( this );
239	continue LOOP;
240	}
241
242	__STATS__( false,
243	io.complete_q.blocks += 1;
244	)
245	__cfadbg_print_safe(io_core, "Kernel I/O : Parking io poller %d (%p)\n", this.fd, &this);
246
247	// block this thread
248	wait( this.sem );
249
250	// restore counter
251	reset = reset_cnt;
252	}
253
254	__cfadbg_print_safe(io_core, "Kernel I/O : Fast poller %d (%p) stopping\n", this.fd, &this);
255	}
256
257	//=============================================================================================
258	// I/O Submissions
259	//=============================================================================================
260
261	// Submition steps :
262	// 1 - Allocate a queue entry. The ring already has memory for all entries but only the ones
263	// listed in sq.array are visible by the kernel. For those not listed, the kernel does not
264	// offer any assurance that an entry is not being filled by multiple flags. Therefore, we
265	// need to write an allocator that allows allocating concurrently.
266	//
267	// 2 - Actually fill the submit entry, this is the only simple and straightforward step.
268	//
269	// 3 - Append the entry index to the array and adjust the tail accordingly. This operation
270	// needs to arrive to two concensus at the same time:
271	// A - The order in which entries are listed in the array: no two threads must pick the
272	// same index for their entries
273	// B - When can the tail be update for the kernel. EVERY entries in the array between
274	// head and tail must be fully filled and shouldn't ever be touched again.
275	//
276
277	static $io_context * __ioarbiter_allocate( $io_arbiter & mutex this, processor *, __u32 idxs[], __u32 want );
278	static void __ioarbiter_submit ( $io_arbiter & mutex this, $io_context * , __u32 idxs[], __u32 have );
279	static void __ioarbiter_flush ( $io_arbiter & mutex this, $io_context * );
280	static inline void __ioarbiter_notify( $io_context & ctx );
281
282	//=============================================================================================
283	// Allocation
284	// for user's convenience fill the sqes from the indexes
285	static inline void __fill(struct io_uring_sqe * out_sqes[], __u32 want, __u32 idxs[], struct $io_context * ctx) {
286	struct io_uring_sqe * sqes = ctx->sq.sqes;
287	for(i; want) {
288	out_sqes[i] = &sqes[idxs[i]];
289	}
290	}
291
292	// Try to directly allocate from the a given context
293	// Not thread-safe
294	static inline bool __alloc(struct $io_context * ctx, __u32 idxs[], __u32 want) {
295	__sub_ring_t & sq = ctx->sq;
296	const __u32 mask = *sq.mask;
297	__u32 fhead = sq.free_ring.head; // get the current head of the queue
298	__u32 ftail = sq.free_ring.tail; // get the current tail of the queue
299
300	// If we don't have enough sqes, fail
301	if((ftail - fhead) < want) { return false; }
302
303	// copy all the indexes we want from the available list
304	for(i; want) {
305	idxs[i] = sq.free_ring.array[(fhead + i) & mask];
306	}
307
308	// Advance the head to mark the indexes as consumed
309	__atomic_store_n(&sq.free_ring.head, fhead + want, __ATOMIC_RELEASE);
310
311	// return success
312	return true;
313	}
314
315	// Allocate an submit queue entry.
316	// The kernel cannot see these entries until they are submitted, but other threads must be
317	// able to see which entries can be used and which are already un used by an other thread
318	// for convenience, return both the index and the pointer to the sqe
319	// sqe == &sqes[idx]
320	struct $io_context * cfa_io_allocate(struct io_uring_sqe * sqes[], __u32 idxs[], __u32 want) {
321	__cfadbg_print_safe(io, "Kernel I/O : attempting to allocate %u\n", want);
322
323	disable_interrupts();
324	processor * proc = __cfaabi_tls.this_processor;
325	/* paranoid */ verify( __cfaabi_tls.this_processor );
326	/* paranoid */ verify( proc->io.lock == false );
327
328	__atomic_store_n( &proc->io.lock, true, __ATOMIC_SEQ_CST );
329	$io_context * ctx = proc->io.ctx;
330	$io_arbiter * ioarb = proc->cltr->io.arbiter;
331	/* paranoid */ verify( ioarb );
332
333	// Can we proceed to the fast path
334	if( ctx // We alreay have an instance?
335	&& !ctx->revoked ) // Our instance is still valid?
336	{
337	__cfadbg_print_safe(io, "Kernel I/O : attempting to fast allocation\n");
338
339	// We can proceed to the fast path
340	if( __alloc(ctx, idxs, want) ) {
341	// Allocation was successful
342	// Mark the instance as no longer in-use and re-enable interrupts
343	__atomic_store_n( &proc->io.lock, false, __ATOMIC_RELEASE );
344	enable_interrupts( __cfaabi_dbg_ctx );
345
346	__cfadbg_print_safe(io, "Kernel I/O : fast allocation successful\n");
347
348	__fill( sqes, want, idxs, ctx );
349	return ctx;
350	}
351	// The fast path failed, fallback
352	}
353
354	// Fast path failed, fallback on arbitration
355	__atomic_store_n( &proc->io.lock, false, __ATOMIC_RELEASE );
356	enable_interrupts( __cfaabi_dbg_ctx );
357
358	__cfadbg_print_safe(io, "Kernel I/O : falling back on arbiter for allocation\n");
359
360	struct $io_context * ret = __ioarbiter_allocate(*ioarb, proc, idxs, want);
361
362	__cfadbg_print_safe(io, "Kernel I/O : slow allocation completed\n");
363
364	__fill( sqes, want, idxs,ret );
365	return ret;
366	}
367
368
369	//=============================================================================================
370	// submission
371	static inline void __submit( struct $io_context * ctx, __u32 idxs[], __u32 have) {
372	// We can proceed to the fast path
373	// Get the right objects
374	__sub_ring_t & sq = ctx->sq;
375	const __u32 mask = *sq.mask;
376	__u32 tail = sq.kring.ready;
377
378	// Add the sqes to the array
379	for( i; have ) {
380	sq.kring.array[ (tail + i) & mask ] = idxs[i];
381	}
382
383	// Make the sqes visible to the submitter
384	__atomic_store_n(&sq.kring.ready, tail + have, __ATOMIC_RELEASE);
385
386	// Make sure the poller is awake
387	__cfadbg_print_safe(io, "Kernel I/O : waking the poller\n");
388	post( ctx->sem );
389	}
390
391	void cfa_io_submit( struct $io_context * inctx, __u32 idxs[], __u32 have ) __attribute__((nonnull (1))) {
392	__cfadbg_print_safe(io, "Kernel I/O : attempting to submit %u\n", have);
393
394	disable_interrupts();
395	processor * proc = __cfaabi_tls.this_processor;
396	/* paranoid */ verify( __cfaabi_tls.this_processor );
397	/* paranoid */ verify( proc->io.lock == false );
398
399	__atomic_store_n( &proc->io.lock, true, __ATOMIC_SEQ_CST );
400	$io_context * ctx = proc->io.ctx;
401
402	// Can we proceed to the fast path
403	if( ctx // We alreay have an instance?
404	&& !ctx->revoked // Our instance is still valid?
405	&& ctx == inctx ) // We have the right instance?
406	{
407	__submit(ctx, idxs, have);
408
409	// Mark the instance as no longer in-use, re-enable interrupts and return
410	__atomic_store_n( &proc->io.lock, false, __ATOMIC_RELEASE );
411	enable_interrupts( __cfaabi_dbg_ctx );
412
413	__cfadbg_print_safe(io, "Kernel I/O : submitted on fast path\n");
414	return;
415	}
416
417	// Fast path failed, fallback on arbitration
418	__atomic_store_n( &proc->io.lock, false, __ATOMIC_RELEASE );
419	enable_interrupts( __cfaabi_dbg_ctx );
420
421	__cfadbg_print_safe(io, "Kernel I/O : falling back on arbiter for submission\n");
422
423	__ioarbiter_submit(*inctx->arbiter, inctx, idxs, have);
424	}
425
426	//=============================================================================================
427	// Flushing
428	static unsigned __flush( struct $io_context & ctx ) {
429	// First check for external
430	if( !__atomic_load_n(&ctx.ext_sq.empty, __ATOMIC_SEQ_CST) ) {
431	// We have external submissions, delegate to the arbiter
432	__ioarbiter_flush( *ctx.arbiter, &ctx );
433	}
434
435	__u32 tail = *ctx.sq.kring.tail;
436	__u32 ready = ctx.sq.kring.ready;
437
438	/* paranoid / verify( ctx.sq.to_submit <= ctx.sq.num );
439	ctx.sq.to_submit += (ready - tail);
440	/* paranoid / verify( ctx.sq.to_submit <= ctx.sq.num );
441
442	if(ctx.sq.to_submit) {
443	__cfadbg_print_safe(io, "Kernel I/O : %u ready to submit\n", ctx.sq.to_submit);
444	}
445
446	__atomic_store_n(ctx.sq.kring.tail, ready, __ATOMIC_RELEASE);
447
448	return ctx.sq.to_submit;
449	}
450
451
452	// Go through the ring's submit queue and release everything that has already been consumed
453	// by io_uring
454	// This cannot be done by multiple threads
455	static __u32 __release_sqes( struct $io_context & ctx ) {
456	const __u32 mask = *ctx.sq.mask;
457
458	__attribute__((unused))
459	__u32 ctail = *ctx.sq.kring.tail; // get the current tail of the queue
460	__u32 chead = *ctx.sq.kring.head; // get the current head of the queue
461	__u32 phead = ctx.sq.kring.released; // get the head the last time we were here
462
463	__u32 ftail = ctx.sq.free_ring.tail; // get the current tail of the queue
464
465	// the 3 fields are organized like this diagram
466	// except it's are ring
467	// ---+--------+--------+----
468	// ---+--------+--------+----
469	// ^ ^ ^
470	// phead chead ctail
471
472	// make sure ctail doesn't wrap around and reach phead
473	/* paranoid */ verify(
474	(ctail >= chead && chead >= phead)
475	\|\| (chead >= phead && phead >= ctail)
476	\|\| (phead >= ctail && ctail >= chead)
477	);
478
479	// find the range we need to clear
480	__u32 count = chead - phead;
481
482	if(count == 0) {
483	return 0;
484	}
485
486	// We acquired an previous-head/current-head range
487	// go through the range and release the sqes
488	for( i; count ) {
489	__u32 idx = ctx.sq.kring.array[ (phead + i) & mask ];
490	ctx.sq.free_ring.array[ (ftail + i) & mask ] = idx;
491	}
492
493	ctx.sq.kring.released = chead; // note up to were we processed
494	__atomic_store_n(&ctx.sq.free_ring.tail, ftail + count, __ATOMIC_SEQ_CST);
495
496	__ioarbiter_notify(ctx);
497
498	return count;
499	}
500
501	//=============================================================================================
502	// I/O Arbiter
503	//=============================================================================================
504	static inline void __revoke( $io_arbiter & this, $io_context * ctx ) {
505	if(ctx->revoked) return;
506
507	remove( this.assigned, *ctx );
508
509	// Mark as revoked
510	__atomic_store_n(&ctx->revoked, true, __ATOMIC_SEQ_CST);
511
512	// Wait for the processor to no longer use it
513	while(ctx->proc->io.lock) Pause();
514
515	// Remove the coupling with the processor
516	ctx->proc->io.ctx = 0p;
517	ctx->proc = 0p;
518
519	// add to available contexts
520	addHead( this.available, *ctx );
521	}
522
523	static inline void __assign( $io_arbiter & this, $io_context * ctx, processor * proc ) {
524	remove( this.available, *ctx );
525
526	ctx->revoked = false;
527	ctx->proc = proc;
528	__atomic_store_n(&proc->io.ctx, ctx, __ATOMIC_SEQ_CST);
529
530	// add to assigned contexts
531	addTail( this.assigned, *ctx );
532	}
533
534	static $io_context * __ioarbiter_allocate( $io_arbiter & mutex this, processor * proc, __u32 idxs[], __u32 want ) {
535	__cfadbg_print_safe(io, "Kernel I/O : arbiter allocating\n");
536
537	SeqIter($io_context) iter;
538	$io_context & ci;
539	// Do we already have something available?
540	for( over( iter, this.available ); iter \| ci;) {
541	__cfadbg_print_safe(io, "Kernel I/O : attempting available context\n");
542
543	$io_context * c = &ci;
544	if(__alloc(c, idxs, want)) {
545	__assign( this, c, proc);
546	return c;
547	}
548	}
549
550
551	// Otherwise, we have no choice but to revoke everyone to check if other instance have available data
552	for( over( iter, this.assigned ); iter \| ci; ) {
553	__cfadbg_print_safe(io, "Kernel I/O : revoking context for allocation\n");
554
555	$io_context * c = &ci;
556	__revoke( this, c );
557
558	if(__alloc(c, idxs, want)) {
559	__assign( this, c, proc);
560	return c;
561	}
562	}
563
564	__cfadbg_print_safe(io, "Kernel I/O : waiting for available resources\n");
565
566	// No one has any resources left, wait for something to finish
567	// Mark as pending
568	__atomic_store_n( &this.pending.flag, true, __ATOMIC_SEQ_CST );
569
570	// Wait for our turn to submit
571	wait( this.pending.blocked, want );
572
573	__attribute((unused)) bool ret =
574	__alloc( this.pending.ctx, idxs, want);
575	/* paranoid */ verify( ret );
576
577	__assign( this, this.pending.ctx, proc);
578	return this.pending.ctx;
579	}
580
581	static void __ioarbiter_notify( $io_arbiter & mutex this, $io_context * ctx ) {
582	/* paranoid */ verify( !is_empty(this.pending.blocked) );
583	this.pending.ctx = ctx;
584
585	while( !is_empty(this.pending.blocked) ) {
586	__u32 have = ctx->sq.free_ring.tail - ctx->sq.free_ring.head;
587	__u32 want = front( this.pending.blocked );
588
589	if( have > want ) return;
590
591	signal_block( this.pending.blocked );
592	}
593
594	this.pending.flag = false;
595	}
596
597	static void __ioarbiter_notify( $io_context & ctx ) {
598	if(__atomic_load_n( &ctx.arbiter->pending.flag, __ATOMIC_SEQ_CST)) {
599	__ioarbiter_notify( *ctx.arbiter, &ctx );
600	}
601	}
602
603	// Simply append to the pending
604	static void __ioarbiter_submit( $io_arbiter & mutex this, $io_context * ctx, __u32 idxs[], __u32 have ) {
605	__cfadbg_print_safe(io, "Kernel I/O : submitting %u from the arbiter to context %u\n", have, ctx->fd);
606
607	/* paranoid */ verify( &this == ctx->arbiter );
608
609	// Mark as pending
610	__atomic_store_n( &ctx->ext_sq.empty, false, __ATOMIC_SEQ_CST );
611
612	// Wake-up the poller
613	post( ctx->sem );
614
615	__cfadbg_print_safe(io, "Kernel I/O : waiting to submit %u\n", have);
616
617	// Wait for our turn to submit
618	wait( ctx->ext_sq.blocked );
619
620	// Submit our indexes
621	__submit(ctx, idxs, have);
622
623	__cfadbg_print_safe(io, "Kernel I/O : %u submitted from arbiter\n", have);
624	}
625
626	static void __ioarbiter_flush( $io_arbiter & mutex this, $io_context * ctx ) {
627	/* paranoid */ verify( &this == ctx->arbiter );
628
629	__revoke( this, ctx );
630
631	__cfadbg_print_safe(io, "Kernel I/O : arbiter flushing\n");
632
633	condition & blcked = ctx->ext_sq.blocked;
634	/* paranoid */ verify( ctx->ext_sq.empty == is_empty( blcked ) );
635	while(!is_empty( blcked )) {
636	signal_block( blcked );
637	}
638
639	ctx->ext_sq.empty = true;
640	}
641
642	void __ioarbiter_register( $io_arbiter & mutex this, $io_context & ctx ) {
643	__cfadbg_print_safe(io, "Kernel I/O : registering new context\n");
644
645	ctx.arbiter = &this;
646
647	// add to available contexts
648	addHead( this.available, ctx );
649
650	// Check if this solves pending allocations
651	if(this.pending.flag) {
652	__ioarbiter_notify( ctx );
653	}
654	}
655
656	void __ioarbiter_unregister( $io_arbiter & mutex this, $io_context & ctx ) {
657	/* paranoid */ verify( &this == ctx.arbiter );
658
659	__revoke( this, &ctx );
660
661	remove( this.available, ctx );
662	}
663	#endif

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format