FreeBSD/sys/net/netisr.c
/*
 * Dispatch a packet for netisr processing; direct dispatch is permitted by
 * calling context.
 */
Dispatch a packet for netisr processing; direct dispatch is permitted by
calling context.
Dispatch a packet for netisr processing ; direct dispatch is permitted by calling context .
int
netisr_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m)
{
#ifdef NETISR_LOCKING
	struct rm_priotracker tracker;
#endif
	struct netisr_workstream *nwsp;
	struct netisr_proto *npp;
	struct netisr_work *npwp;
	int dosignal, error;
	u_int cpuid, dispatch_policy;
	KASSERT(proto < NETISR_MAXPROT,
	    ("%s: invalid proto %u", __func__, proto));
#ifdef NETISR_LOCKING
	NETISR_RLOCK(&tracker);
#endif
	npp = &netisr_proto[proto];
	KASSERT(npp->np_handler != NULL, ("%s: invalid proto %u", __func__,
	    proto));
#ifdef VIMAGE
	if (V_netisr_enable[proto] == 0) {
		m_freem(m);
		return (ENOPROTOOPT);
	}
#endif
	dispatch_policy = netisr_get_dispatch(npp);
	if (dispatch_policy == NETISR_DISPATCH_DEFERRED)
		return (netisr_queue_src(proto, source, m));
	if (dispatch_policy == NETISR_DISPATCH_DIRECT) {
		nwsp = DPCPU_PTR(nws);
		npwp = &nwsp->nws_work[proto];
		npwp->nw_dispatched++;
		npwp->nw_handled++;
		netisr_proto[proto].np_handler(m);
		error = 0;
		goto out_unlock;
	}
	KASSERT(dispatch_policy == NETISR_DISPATCH_HYBRID,
	    ("%s: unknown dispatch policy (%u)", __func__, dispatch_policy));
	sched_pin();
	m = netisr_select_cpuid(&netisr_proto[proto], NETISR_DISPATCH_HYBRID,
	    source, m, &cpuid);
	if (m == NULL) {
		error = ENOBUFS;
		goto out_unpin;
	}
	KASSERT(!CPU_ABSENT(cpuid), ("%s: CPU %u absent", __func__, cpuid));
	if (cpuid != curcpu)
		goto queue_fallback;
	nwsp = DPCPU_PTR(nws);
	npwp = &nwsp->nws_work[proto];
	NWS_LOCK(nwsp);
	if (nwsp->nws_flags & (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED)) {
		error = netisr_queue_workstream(nwsp, proto, npwp, m,
		    &dosignal);
		NWS_UNLOCK(nwsp);
		if (dosignal)
			NWS_SIGNAL(nwsp);
		goto out_unpin;
	}
	nwsp->nws_flags |= NWS_DISPATCHING;
	NWS_UNLOCK(nwsp);
	netisr_proto[proto].np_handler(m);
	NWS_LOCK(nwsp);
	nwsp->nws_flags &= ~NWS_DISPATCHING;
	npwp->nw_handled++;
	npwp->nw_hybrid_dispatched++;
	if (nwsp->nws_pendingbits != 0) {
		nwsp->nws_flags |= NWS_SCHEDULED;
		dosignal = 1;
	} else
		dosignal = 0;
	NWS_UNLOCK(nwsp);
	if (dosignal)
		NWS_SIGNAL(nwsp);
	error = 0;
	goto out_unpin;
queue_fallback:
	error = netisr_queue_internal(proto, m, cpuid);
out_unpin:
	sched_unpin();
out_unlock:
#ifdef NETISR_LOCKING
	NETISR_RUNLOCK(&tracker);
#endif
	return (error);
}
int netisr_dispatch_src ( u_int proto , uintptr_t source , struct mbuf * m ) { #ifdef NETISR_LOCKING struct rm_priotracker tracker ; #endif struct netisr_workstream * nwsp ; struct netisr_proto * npp ; struct netisr_work * npwp ; int dosignal , error ; u_int cpuid , dispatch_policy ; KASSERT ( proto < NETISR_MAXPROT , ( " " , __func__ , proto ) ) ; #ifdef NETISR_LOCKING NETISR_RLOCK ( & tracker ) ; #endif npp = & netisr_proto [ proto ] ; KASSERT ( npp -> np_handler != NULL , ( " " , __func__ , proto ) ) ; #ifdef VIMAGE if ( V_netisr_enable [ proto ] == 0 ) { m_freem ( m ) ; return ( ENOPROTOOPT ) ; } #endif dispatch_policy = netisr_get_dispatch ( npp ) ; if ( dispatch_policy == NETISR_DISPATCH_DEFERRED ) return ( netisr_queue_src ( proto , source , m ) ) ; if ( dispatch_policy == NETISR_DISPATCH_DIRECT ) { nwsp = DPCPU_PTR ( nws ) ; npwp = & nwsp -> nws_work [ proto ] ; npwp -> nw_dispatched ++ ; npwp -> nw_handled ++ ; netisr_proto [ proto ] . np_handler ( m ) ; error = 0 ; goto out_unlock ; } KASSERT ( dispatch_policy == NETISR_DISPATCH_HYBRID , ( " " , __func__ , dispatch_policy ) ) ; sched_pin ( ) ; m = netisr_select_cpuid ( & netisr_proto [ proto ] , NETISR_DISPATCH_HYBRID , source , m , & cpuid ) ; if ( m == NULL ) { error = ENOBUFS ; goto out_unpin ; } KASSERT ( ! CPU_ABSENT ( cpuid ) , ( " " , __func__ , cpuid ) ) ; if ( cpuid != curcpu ) goto queue_fallback ; nwsp = DPCPU_PTR ( nws ) ; npwp = & nwsp -> nws_work [ proto ] ; NWS_LOCK ( nwsp ) ; if ( nwsp -> nws_flags & ( NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED ) ) { error = netisr_queue_workstream ( nwsp , proto , npwp , m , & dosignal ) ; NWS_UNLOCK ( nwsp ) ; if ( dosignal ) NWS_SIGNAL ( nwsp ) ; goto out_unpin ; } nwsp -> nws_flags |= NWS_DISPATCHING ; NWS_UNLOCK ( nwsp ) ; netisr_proto [ proto ] . np_handler ( m ) ; NWS_LOCK ( nwsp ) ; nwsp -> nws_flags &= ~ NWS_DISPATCHING ; npwp -> nw_handled ++ ; npwp -> nw_hybrid_dispatched ++ ; if ( nwsp -> nws_pendingbits != 0 ) { nwsp -> nws_flags |= NWS_SCHEDULED ; dosignal = 1 ; } else dosignal = 0 ; NWS_UNLOCK ( nwsp ) ; if ( dosignal ) NWS_SIGNAL ( nwsp ) ; error = 0 ; goto out_unpin ; queue_fallback : error = netisr_queue_internal ( proto , m , cpuid ) ; out_unpin : sched_unpin ( ) ; out_unlock : #ifdef NETISR_LOCKING NETISR_RUNLOCK ( & tracker ) ; #endif return ( error ) ; }
Dispatch a packet for netisr processing; direct dispatch is permitted by
 calling context.
Dispatch a packet for netisr processing ; direct dispatch is permitted by calling context .
/*
	 * If direct dispatch is forced, then unconditionally dispatch
	 * without a formal CPU selection.  Borrow the current CPU's stats,
	 * even if there's no worker on it.  In this case we don't update
	 * nws_flags because all netisr processing will be source ordered due
	 * to always being forced to directly dispatch.
	 */ /*
	 * Otherwise, we execute in a hybrid mode where we will try to direct
	 * dispatch if we're on the right CPU and the netisr worker isn't
	 * already running.
	 */ /*-
	 * We are willing to direct dispatch only if three conditions hold:
	 *
	 * (1) The netisr worker isn't already running,
	 * (2) Another thread isn't already directly dispatching, and
	 * (3) The netisr hasn't already been woken up.
	 */ /*
	 * The current thread is now effectively the netisr worker, so set
	 * the dispatching flag to prevent concurrent processing of the
	 * stream from another thread (even the netisr worker), which could
	 * otherwise lead to effective misordering of the stream.
	 */ /*
	 * If other work was enqueued by another thread while we were direct
	 * dispatching, we need to signal the netisr worker to do that work.
	 * In the future, we might want to do some of that work in the
	 * current thread, rather than trigger further context switches.  If
	 * so, we'll want to establish a reasonable bound on the work done in
	 * the "borrowed" context.
	 */