FreeBSD/sys/dev/ixl/ixl_txrx.c
/*********************************************************************
 *
 *  This routine executes in interrupt context. It replenishes
 *  the mbufs in the descriptor and sends data which has been
 *  dma'ed into host memory to upper layer.
 *
 *  We loop at most count times if count is > 0, or until done if
 *  count < 0.
 *
 *  Return TRUE for more work, FALSE for all clean.
 *********************************************************************/
This routine executes in interrupt context. It replenishes
the mbufs in the descriptor and sends data which has been
dma'ed into host memory to upper layer.

We loop at most count times if count is > 0, or until done if
count < 0.

Return TRUE for more work, FALSE for all clean.
This routine executes in interrupt context . It replenishes the mbufs in the descriptor and sends data which has been dma ' ed into host memory to upper layer . We loop at most count times if count is > 0 or until done if count < 0 . Return TRUE for more work FALSE for all clean .
bool
ixl_rxeof(struct ixl_queue *que, int count)
{
	struct ixl_vsi		*vsi = que->vsi;
	struct rx_ring		*rxr = &que->rxr;
	struct ifnet		*ifp = vsi->ifp;
#if defined(INET6) || defined(INET)
	struct lro_ctrl		*lro = &rxr->lro;
#endif
	int			i, nextp, processed = 0;
	union i40e_rx_desc	*cur;
	struct ixl_rx_buf	*rbuf, *nbuf;
	IXL_RX_LOCK(rxr);
#ifdef DEV_NETMAP
	if (netmap_rx_irq(ifp, que->me, &count)) {
		IXL_RX_UNLOCK(rxr);
		return (FALSE);
	}
#endif 
	for (i = rxr->next_check; count != 0;) {
		struct mbuf	*sendmp, *mh, *mp;
		u32		status, error;
		u16		hlen, plen, vtag;
		u64		qword;
		u8		ptype;
		bool		eop;
		bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
		cur = &rxr->base[i];
		qword = le64toh(cur->wb.qword1.status_error_len);
		status = (qword & I40E_RXD_QW1_STATUS_MASK)
		    >> I40E_RXD_QW1_STATUS_SHIFT;
		error = (qword & I40E_RXD_QW1_ERROR_MASK)
		    >> I40E_RXD_QW1_ERROR_SHIFT;
		plen = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK)
		    >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
		hlen = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK)
		    >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
		ptype = (qword & I40E_RXD_QW1_PTYPE_MASK)
		    >> I40E_RXD_QW1_PTYPE_SHIFT;
		if ((status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) {
			++rxr->not_done;
			break;
		}
		if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
			break;
		count--;
		sendmp = NULL;
		nbuf = NULL;
		cur->wb.qword1.status_error_len = 0;
		rbuf = &rxr->buffers[i];
		mh = rbuf->m_head;
		mp = rbuf->m_pack;
		eop = (status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT));
		if (status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT))
			vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1);
		else
			vtag = 0;
                if (eop && (error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
			rxr->desc_errs++;
			ixl_rx_discard(rxr, i);
			goto next_desc;
		}
		if (!eop) {
			nextp = i + 1;
			if (nextp == que->num_desc)
				nextp = 0;
			nbuf = &rxr->buffers[nextp];
			prefetch(nbuf);
		}
		if (rxr->hdr_split && (rbuf->fmp == NULL)) {
			if (hlen > IXL_RX_HDR)
				hlen = IXL_RX_HDR;
			mh->m_len = hlen;
			mh->m_flags |= M_PKTHDR;
			mh->m_next = NULL;
			mh->m_pkthdr.len = mh->m_len;
			rbuf->m_head = NULL;
			if (plen > 0) {
				mp->m_len = plen;
				mp->m_next = NULL;
				mp->m_flags &= ~M_PKTHDR;
				mh->m_next = mp;
				mh->m_pkthdr.len += mp->m_len;
				rbuf->m_pack = NULL;
				rxr->split++;
			}
                        if (eop == 0) {
                                nbuf->fmp = mh;
                                if (plen)
                                        mp->m_next = nbuf->m_pack;
                                else
                                        mh->m_next = nbuf->m_pack;
                        } else {
                                sendmp = mh;
                                if (vtag) {
                                        sendmp->m_pkthdr.ether_vtag = vtag;
                                        sendmp->m_flags |= M_VLANTAG;
                                }
                        }
		} else {
			mp->m_len = plen;
			sendmp = rbuf->fmp;
			rbuf->m_pack = rbuf->fmp = NULL;
			if (sendmp != NULL) 
				sendmp->m_pkthdr.len += mp->m_len;
			else {
				sendmp = mp;
				sendmp->m_flags |= M_PKTHDR;
				sendmp->m_pkthdr.len = mp->m_len;
                        }
			if (eop == 0) {
				nbuf->fmp = sendmp;
				sendmp = NULL;
				mp->m_next = nbuf->m_pack;
			}
		}
		++processed;
		if (eop) {
			sendmp->m_pkthdr.rcvif = ifp;
			rxr->rx_packets++;
			rxr->rx_bytes += sendmp->m_pkthdr.len;
			rxr->packets++;
			rxr->bytes += sendmp->m_pkthdr.len;
			if (vtag) {
				sendmp->m_pkthdr.ether_vtag = vtag;
				sendmp->m_flags |= M_VLANTAG;
			}
			if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
				ixl_rx_checksum(sendmp, status, error, ptype);
#ifdef RSS
			sendmp->m_pkthdr.flowid =
			    le32toh(cur->wb.qword0.hi_dword.rss);
			M_HASHTYPE_SET(sendmp, ixl_ptype_to_hash(ptype));
#else
			sendmp->m_pkthdr.flowid = que->msix;
			M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE);
#endif
		}
next_desc:
		bus_dmamap_sync(rxr->dma.tag, rxr->dma.map,
		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
		if (++i == que->num_desc)
			i = 0;
		if (sendmp != NULL) {
			rxr->next_check = i;
			ixl_rx_input(rxr, ifp, sendmp, ptype);
			i = rxr->next_check;
		}
		if (processed == 8) {
			ixl_refresh_mbufs(que, i);
			processed = 0;
		}
	}
	if (ixl_rx_unrefreshed(que))
		ixl_refresh_mbufs(que, i);
	rxr->next_check = i;
#if defined(INET6) || defined(INET)
	tcp_lro_flush_all(lro);
#endif
	IXL_RX_UNLOCK(rxr);
	return (FALSE);
}
bool ixl_rxeof ( struct ixl_queue * que , int count ) { struct ixl_vsi * vsi = que -> vsi ; struct rx_ring * rxr = & que -> rxr ; struct ifnet * ifp = vsi -> ifp ; #if defined ( INET6 ) || defined ( INET ) struct lro_ctrl * lro = & rxr -> lro ; #endif int i , nextp , processed = 0 ; union i40e_rx_desc * cur ; struct ixl_rx_buf * rbuf , * nbuf ; IXL_RX_LOCK ( rxr ) ; #ifdef DEV_NETMAP if ( netmap_rx_irq ( ifp , que -> me , & count ) ) { IXL_RX_UNLOCK ( rxr ) ; return ( FALSE ) ; } #endif for ( i = rxr -> next_check ; count != 0 ; ) { struct mbuf * sendmp , * mh , * mp ; u32 status , error ; u16 hlen , plen , vtag ; u64 qword ; u8 ptype ; bool eop ; bus_dmamap_sync ( rxr -> dma . tag , rxr -> dma . map , BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE ) ; cur = & rxr -> base [ i ] ; qword = le64toh ( cur -> wb . qword1 . status_error_len ) ; status = ( qword & I40E_RXD_QW1_STATUS_MASK ) >> I40E_RXD_QW1_STATUS_SHIFT ; error = ( qword & I40E_RXD_QW1_ERROR_MASK ) >> I40E_RXD_QW1_ERROR_SHIFT ; plen = ( qword & I40E_RXD_QW1_LENGTH_PBUF_MASK ) >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT ; hlen = ( qword & I40E_RXD_QW1_LENGTH_HBUF_MASK ) >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT ; ptype = ( qword & I40E_RXD_QW1_PTYPE_MASK ) >> I40E_RXD_QW1_PTYPE_SHIFT ; if ( ( status & ( 1 << I40E_RX_DESC_STATUS_DD_SHIFT ) ) == 0 ) { ++ rxr -> not_done ; break ; } if ( ( ifp -> if_drv_flags & IFF_DRV_RUNNING ) == 0 ) break ; count -- ; sendmp = NULL ; nbuf = NULL ; cur -> wb . qword1 . status_error_len = 0 ; rbuf = & rxr -> buffers [ i ] ; mh = rbuf -> m_head ; mp = rbuf -> m_pack ; eop = ( status & ( 1 << I40E_RX_DESC_STATUS_EOF_SHIFT ) ) ; if ( status & ( 1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT ) ) vtag = le16toh ( cur -> wb . qword0 . lo_dword . l2tag1 ) ; else vtag = 0 ; if ( eop && ( error & ( 1 << I40E_RX_DESC_ERROR_RXE_SHIFT ) ) ) { rxr -> desc_errs ++ ; ixl_rx_discard ( rxr , i ) ; goto next_desc ; } if ( ! eop ) { nextp = i + 1 ; if ( nextp == que -> num_desc ) nextp = 0 ; nbuf = & rxr -> buffers [ nextp ] ; prefetch ( nbuf ) ; } if ( rxr -> hdr_split && ( rbuf -> fmp == NULL ) ) { if ( hlen > IXL_RX_HDR ) hlen = IXL_RX_HDR ; mh -> m_len = hlen ; mh -> m_flags |= M_PKTHDR ; mh -> m_next = NULL ; mh -> m_pkthdr . len = mh -> m_len ; rbuf -> m_head = NULL ; if ( plen > 0 ) { mp -> m_len = plen ; mp -> m_next = NULL ; mp -> m_flags &= ~ M_PKTHDR ; mh -> m_next = mp ; mh -> m_pkthdr . len += mp -> m_len ; rbuf -> m_pack = NULL ; rxr -> split ++ ; } if ( eop == 0 ) { nbuf -> fmp = mh ; if ( plen ) mp -> m_next = nbuf -> m_pack ; else mh -> m_next = nbuf -> m_pack ; } else { sendmp = mh ; if ( vtag ) { sendmp -> m_pkthdr . ether_vtag = vtag ; sendmp -> m_flags |= M_VLANTAG ; } } } else { mp -> m_len = plen ; sendmp = rbuf -> fmp ; rbuf -> m_pack = rbuf -> fmp = NULL ; if ( sendmp != NULL ) sendmp -> m_pkthdr . len += mp -> m_len ; else { sendmp = mp ; sendmp -> m_flags |= M_PKTHDR ; sendmp -> m_pkthdr . len = mp -> m_len ; } if ( eop == 0 ) { nbuf -> fmp = sendmp ; sendmp = NULL ; mp -> m_next = nbuf -> m_pack ; } } ++ processed ; if ( eop ) { sendmp -> m_pkthdr . rcvif = ifp ; rxr -> rx_packets ++ ; rxr -> rx_bytes += sendmp -> m_pkthdr . len ; rxr -> packets ++ ; rxr -> bytes += sendmp -> m_pkthdr . len ; if ( vtag ) { sendmp -> m_pkthdr . ether_vtag = vtag ; sendmp -> m_flags |= M_VLANTAG ; } if ( ( ifp -> if_capenable & IFCAP_RXCSUM ) != 0 ) ixl_rx_checksum ( sendmp , status , error , ptype ) ; #ifdef RSS sendmp -> m_pkthdr . flowid = le32toh ( cur -> wb . qword0 . hi_dword . rss ) ; M_HASHTYPE_SET ( sendmp , ixl_ptype_to_hash ( ptype ) ) ; #else sendmp -> m_pkthdr . flowid = que -> msix ; M_HASHTYPE_SET ( sendmp , M_HASHTYPE_OPAQUE ) ; #endif } next_desc : bus_dmamap_sync ( rxr -> dma . tag , rxr -> dma . map , BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE ) ; if ( ++ i == que -> num_desc ) i = 0 ; if ( sendmp != NULL ) { rxr -> next_check = i ; ixl_rx_input ( rxr , ifp , sendmp , ptype ) ; i = rxr -> next_check ; } if ( processed == 8 ) { ixl_refresh_mbufs ( que , i ) ; processed = 0 ; } } if ( ixl_rx_unrefreshed ( que ) ) ixl_refresh_mbufs ( que , i ) ; rxr -> next_check = i ; #if defined ( INET6 ) || defined ( INET ) tcp_lro_flush_all ( lro ) ; #endif IXL_RX_UNLOCK ( rxr ) ; return ( FALSE ) ; }
This routine executes in interrupt context.
This routine executes in interrupt context .
/* DEV_NETMAP */ /* Sync the ring. */ /*
		** Make sure bad packets are discarded,
		** note that only EOP descriptor has valid
		** error results.
		*/ /* Prefetch the next buffer */ /*
		** The header mbuf is ONLY used when header 
		** split is enabled, otherwise we get normal 
		** behavior, ie, both header and payload
		** are DMA'd into the payload buffer.
		**
		** Rather than using the fmp/lmp global pointers
		** we now keep the head of a packet chain in the
		** buffer struct and pass this along from one
		** descriptor to the next, until we get EOP.
		*/ /* Null buf pointer so it is refreshed */ /*
			** Check the payload length, this
			** could be zero if its a small
			** packet.
			*/ /* Null buf pointer so it is refreshed */ /*
			** Now create the forward
			** chain so when complete 
			** we wont have to.
			*/ /* stash the chain head */ /* Make forward chain */ /* Singlet, prepare to send */ /*
			** Either no header split, or a
			** secondary piece of a fragmented
			** split packet.
			*/ /*
			** See if there is a stored head
			** that determines what we are
			*/ /* secondary frag */ /* first desc of a non-ps chain */ /* Pass the head pointer on */ /* Sending this frame? */ /* gather stats */ /* capture data for dynamic ITR adjustment */ /* Set VLAN tag (field only valid in eop desc) */ /* Advance our pointers to the next descriptor. */ /* Now send to the stack or do LRO */ /* Every 8 descriptors we go to refresh mbufs */ /* Refresh any remaining buf structs */ /*
	 * Flush any outstanding LRO work
	 */