FreeBSD/sys/dev/cxgbe/t4_sge.c
/*
 * Fills up the freelist by allocating up to 'n' buffers.  Buffers that are
 * recycled do not count towards this allocation budget.
 *
 * Returns non-zero to indicate that this freelist should be added to the list
 * of starving freelists.
 */
Fills up the freelist by allocating up to 'n' buffers.  Buffers that are
recycled do not count towards this allocation budget.

Returns non-zero to indicate that this freelist should be added to the list
of starving freelists.
Fills up the freelist by allocating up to ' n ' buffers . Buffers that are recycled do not count towards this allocation budget . Returns non - zero to indicate that this freelist should be added to the list of starving freelists .
static int
refill_fl(struct adapter *sc, struct sge_fl *fl, int n)
{
	__be64 *d;
	struct fl_sdesc *sd;
	uintptr_t pa;
	caddr_t cl;
	struct cluster_layout *cll;
	struct sw_zone_info *swz;
	struct cluster_metadata *clm;
	uint16_t max_pidx;
	uint16_t hw_cidx = fl->hw_cidx;		
	FL_LOCK_ASSERT_OWNED(fl);
	max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1;
	if (fl->pidx == max_pidx * 8)
		return (0);
	d = &fl->desc[fl->pidx];
	sd = &fl->sdesc[fl->pidx];
	cll = &fl->cll_def;	
	swz = &sc->sge.sw_zone_info[cll->zidx];
	while (n > 0) {
		if (sd->cl != NULL) {
			if (sd->nmbuf == 0) {
				fl->cl_fast_recycled++;
#ifdef INVARIANTS
				clm = cl_metadata(sc, fl, &sd->cll, sd->cl);
				if (clm != NULL)
					MPASS(clm->refcount == 1);
#endif
				goto recycled_fast;
			}
			clm = cl_metadata(sc, fl, &sd->cll, sd->cl);
			MPASS(clm != NULL);
			if (atomic_fetchadd_int(&clm->refcount, -1) == 1) {
				fl->cl_recycled++;
				counter_u64_add(extfree_rels, 1);
				goto recycled;
			}
			sd->cl = NULL;	
		}
		MPASS(sd->cl == NULL);
alloc:
		cl = uma_zalloc(swz->zone, M_NOWAIT);
		if (__predict_false(cl == NULL)) {
			if (cll == &fl->cll_alt || fl->cll_alt.zidx == -1 ||
			    fl->cll_def.zidx == fl->cll_alt.zidx)
				break;
			cll = &fl->cll_alt;
			swz = &sc->sge.sw_zone_info[cll->zidx];
			goto alloc;
		}
		fl->cl_allocated++;
		n--;
		pa = pmap_kextract((vm_offset_t)cl);
		pa += cll->region1;
		sd->cl = cl;
		sd->cll = *cll;
		*d = htobe64(pa | cll->hwidx);
		clm = cl_metadata(sc, fl, cll, cl);
		if (clm != NULL) {
recycled:
#ifdef INVARIANTS
			clm->sd = sd;
#endif
			clm->refcount = 1;
		}
		sd->nmbuf = 0;
recycled_fast:
		d++;
		sd++;
		if (__predict_false(++fl->pidx % 8 == 0)) {
			uint16_t pidx = fl->pidx / 8;
			if (__predict_false(pidx == fl->sidx)) {
				fl->pidx = 0;
				pidx = 0;
				sd = fl->sdesc;
				d = fl->desc;
			}
			if (pidx == max_pidx)
				break;
			if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4)
				ring_fl_db(sc, fl);
		}
	}
	if (fl->pidx / 8 != fl->dbidx)
		ring_fl_db(sc, fl);
	return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
}
static int refill_fl ( struct adapter * sc , struct sge_fl * fl , int n ) { __be64 * d ; struct fl_sdesc * sd ; uintptr_t pa ; caddr_t cl ; struct cluster_layout * cll ; struct sw_zone_info * swz ; struct cluster_metadata * clm ; uint16_t max_pidx ; uint16_t hw_cidx = fl -> hw_cidx ; FL_LOCK_ASSERT_OWNED ( fl ) ; max_pidx = __predict_false ( hw_cidx == 0 ) ? fl -> sidx - 1 : hw_cidx - 1 ; if ( fl -> pidx == max_pidx * 8 ) return ( 0 ) ; d = & fl -> desc [ fl -> pidx ] ; sd = & fl -> sdesc [ fl -> pidx ] ; cll = & fl -> cll_def ; swz = & sc -> sge . sw_zone_info [ cll -> zidx ] ; while ( n > 0 ) { if ( sd -> cl != NULL ) { if ( sd -> nmbuf == 0 ) { fl -> cl_fast_recycled ++ ; #ifdef INVARIANTS clm = cl_metadata ( sc , fl , & sd -> cll , sd -> cl ) ; if ( clm != NULL ) MPASS ( clm -> refcount == 1 ) ; #endif goto recycled_fast ; } clm = cl_metadata ( sc , fl , & sd -> cll , sd -> cl ) ; MPASS ( clm != NULL ) ; if ( atomic_fetchadd_int ( & clm -> refcount , -1 ) == 1 ) { fl -> cl_recycled ++ ; counter_u64_add ( extfree_rels , 1 ) ; goto recycled ; } sd -> cl = NULL ; } MPASS ( sd -> cl == NULL ) ; alloc : cl = uma_zalloc ( swz -> zone , M_NOWAIT ) ; if ( __predict_false ( cl == NULL ) ) { if ( cll == & fl -> cll_alt || fl -> cll_alt . zidx == -1 || fl -> cll_def . zidx == fl -> cll_alt . zidx ) break ; cll = & fl -> cll_alt ; swz = & sc -> sge . sw_zone_info [ cll -> zidx ] ; goto alloc ; } fl -> cl_allocated ++ ; n -- ; pa = pmap_kextract ( ( vm_offset_t ) cl ) ; pa += cll -> region1 ; sd -> cl = cl ; sd -> cll = * cll ; * d = htobe64 ( pa | cll -> hwidx ) ; clm = cl_metadata ( sc , fl , cll , cl ) ; if ( clm != NULL ) { recycled : #ifdef INVARIANTS clm -> sd = sd ; #endif clm -> refcount = 1 ; } sd -> nmbuf = 0 ; recycled_fast : d ++ ; sd ++ ; if ( __predict_false ( ++ fl -> pidx % 8 == 0 ) ) { uint16_t pidx = fl -> pidx / 8 ; if ( __predict_false ( pidx == fl -> sidx ) ) { fl -> pidx = 0 ; pidx = 0 ; sd = fl -> sdesc ; d = fl -> desc ; } if ( pidx == max_pidx ) break ; if ( IDXDIFF ( pidx , fl -> dbidx , fl -> sidx ) >= 4 ) ring_fl_db ( sc , fl ) ; } } if ( fl -> pidx / 8 != fl -> dbidx ) ring_fl_db ( sc , fl ) ; return ( FL_RUNNING_LOW ( fl ) && ! ( fl -> flags & FL_STARVING ) ) ; }
Fills up the freelist by allocating up to 'n' buffers.
Fills up the freelist by allocating up to ' n ' buffers .
/* stable snapshot */ /*
	 * We always stop at the beginning of the hardware descriptor that's just
	 * before the one with the hw cidx.  This is to avoid hw pidx = hw cidx,
	 * which would mean an empty freelist to the chip.
	 */ /* default layout */ /*
				 * Fast recycle without involving any atomics on
				 * the cluster's metadata (if the cluster has
				 * metadata).  This happens when all frames
				 * received in the cluster were small enough to
				 * fit within a single mbuf each.
				 */ /*
			 * Cluster is guaranteed to have metadata.  Clusters
			 * without metadata always take the fast recycle path
			 * when they're recycled.
			 */ /* gave up my reference */ /* fall back to the safe zone */