FreeBSD/sys/netinet/tcp_stacks/fastpath.c
/*
 * This tcp-do-segment concentrates on making the fastest
 * ack processing path. It does not have a fast-path for
 * data (it possibly could which would then eliminate the
 * need for fast-slow above). For a content distributor having
 * large outgoing elephants and very very little coming in
 * having no fastpath for data does not really help (since you
 * don't get much data in). The most important thing is 
 * processing ack's quickly and getting the rest of the data
 * output to the peer as quickly as possible. This routine
 * seems to be about an overall 3% faster then the old
 * tcp_do_segment and keeps us in the fast-path for packets
 * much more (by allowing window updates to also stay in the fastpath).
 */
This tcp-do-segment concentrates on making the fastest
ack processing path. It does not have a fast-path for
data (it possibly could which would then eliminate the
need for fast-slow above). For a content distributor having
large outgoing elephants and very very little coming in
having no fastpath for data does not really help (since you
don't get much data in). The most important thing is
processing ack's quickly and getting the rest of the data
output to the peer as quickly as possible. This routine
seems to be about an overall 3% faster then the old
tcp_do_segment and keeps us in the fast-path for packets
much more (by allowing window updates to also stay in the fastpath).
This tcp - do - segment concentrates on making the fastest ack processing path . It does not have a fast - path for data ( it possibly could which would then eliminate the need for fast - slow above ) . For a content distributor having large outgoing elephants and very very little coming in having no fastpath for data does not really help ( since you don ' t get much data in ) . The most important thing is processing ack ' s quickly and getting the rest of the data output to the peer as quickly as possible . This routine seems to be about an overall 3% faster then the old tcp_do_segment and keeps us in the fast - path for packets much more ( by allowing window updates to also stay in the fastpath ) .
void
tcp_do_segment_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so,
		       struct tcpcb *tp, int drop_hdrlen, int tlen, uint8_t iptos,
		       int ti_locked)
{
	int thflags;
	u_long tiwin;
	char *s;
	struct in_conninfo *inc;
	struct tcpopt to;
	thflags = th->th_flags;
	tp->sackhint.last_sack_ack = 0;
	inc = &tp->t_inpcb->inp_inc;
	if ((thflags & (TH_SYN | TH_FIN | TH_RST)) != 0 ||
	    tp->t_state != TCPS_ESTABLISHED) {
		KASSERT(ti_locked == TI_RLOCKED, ("%s ti_locked %d for "
						  "SYN/FIN/RST/!EST", __func__, ti_locked));
		INP_INFO_RLOCK_ASSERT(&V_tcbinfo);
	} else {
#ifdef INVARIANTS
		if (ti_locked == TI_RLOCKED) {
			INP_INFO_RLOCK_ASSERT(&V_tcbinfo);
		} else {
			KASSERT(ti_locked == TI_UNLOCKED, ("%s: EST "
							   "ti_locked: %d", __func__, ti_locked));
			INP_INFO_UNLOCK_ASSERT(&V_tcbinfo);
		}
#endif
	}
	INP_WLOCK_ASSERT(tp->t_inpcb);
	KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN",
					    __func__));
	KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT",
						__func__));
	tp->t_rcvtime = ticks;
	if (TCPS_HAVEESTABLISHED(tp->t_state))
		tcp_timer_activate(tp, TT_KEEP, TP_KEEPIDLE(tp));
	tiwin = th->th_win << tp->snd_scale;
	if (tp->t_flags & TF_ECN_PERMIT) {
		if (thflags & TH_CWR)
			tp->t_flags &= ~TF_ECN_SND_ECE;
		switch (iptos & IPTOS_ECN_MASK) {
		case IPTOS_ECN_CE:
			tp->t_flags |= TF_ECN_SND_ECE;
			TCPSTAT_INC(tcps_ecn_ce);
			break;
		case IPTOS_ECN_ECT0:
			TCPSTAT_INC(tcps_ecn_ect0);
			break;
		case IPTOS_ECN_ECT1:
			TCPSTAT_INC(tcps_ecn_ect1);
			break;
		}
		if (thflags & TH_ECE) {
			cc_cong_signal(tp, th, CC_ECN);
		}
	}
	tcp_dooptions(&to, (u_char *)(th + 1),
		      (th->th_off << 2) - sizeof(struct tcphdr),
		      (thflags & TH_SYN) ? TO_SYN : 0);
	if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) {
		to.to_tsecr -= tp->ts_offset;
		if (TSTMP_GT(to.to_tsecr, tcp_ts_getticks()))
			to.to_tsecr = 0;
	}
	if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS)) {
		if ((s = tcp_log_addrs(inc, th, NULL, NULL))) {
			log(LOG_DEBUG, "%s; %s: Timestamp missing, "
			    "no action\n", s, __func__);
			free(s, M_TCPLOG);
		}
	}
	if (!(tp->t_flags & TF_RCVD_TSTMP) && (to.to_flags & TOF_TS)) {
		if ((s = tcp_log_addrs(inc, th, NULL, NULL))) {
			log(LOG_DEBUG, "%s; %s: Timestamp not expected, "
			    "no action\n", s, __func__);
			free(s, M_TCPLOG);
		}
	}
	if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
		if ((to.to_flags & TOF_SCALE) &&
		    (tp->t_flags & TF_REQ_SCALE)) {
			tp->t_flags |= TF_RCVD_SCALE;
			tp->snd_scale = to.to_wscale;
		}
		tp->snd_wnd = th->th_win;
		if (to.to_flags & TOF_TS) {
			tp->t_flags |= TF_RCVD_TSTMP;
			tp->ts_recent = to.to_tsval;
			tp->ts_recent_age = tcp_ts_getticks();
		}
		if (to.to_flags & TOF_MSS)
			tcp_mss(tp, to.to_mss);
		if ((tp->t_flags & TF_SACK_PERMIT) &&
		    (to.to_flags & TOF_SACKPERM) == 0)
			tp->t_flags &= ~TF_SACK_PERMIT;
	}
	if (__predict_true(tp->t_state == TCPS_ESTABLISHED) &&
	    __predict_true(((to.to_flags & TOF_SACK) == 0)) &&
	    __predict_true(tlen == 0) &&
	    __predict_true((thflags & (TH_SYN|TH_FIN|TH_RST|TH_URG|TH_ACK)) == TH_ACK) &&
	    __predict_true(LIST_EMPTY(&tp->t_segq)) &&
	    __predict_true(th->th_seq == tp->rcv_nxt)) {
		    if (tcp_fastack(m, th, so, tp, &to, drop_hdrlen, tlen, 
				    ti_locked, tiwin)) {
			    return;
		    }
	} 
	tcp_do_slowpath(m, th, so, tp, &to, drop_hdrlen, tlen,
			ti_locked, tiwin, thflags);
}
void tcp_do_segment_fastack ( struct mbuf * m , struct tcphdr * th , struct socket * so , struct tcpcb * tp , int drop_hdrlen , int tlen , uint8_t iptos , int ti_locked ) { int thflags ; u_long tiwin ; char * s ; struct in_conninfo * inc ; struct tcpopt to ; thflags = th -> th_flags ; tp -> sackhint . last_sack_ack = 0 ; inc = & tp -> t_inpcb -> inp_inc ; if ( ( thflags & ( TH_SYN | TH_FIN | TH_RST ) ) != 0 || tp -> t_state != TCPS_ESTABLISHED ) { KASSERT ( ti_locked == TI_RLOCKED , ( " " " " , __func__ , ti_locked ) ) ; INP_INFO_RLOCK_ASSERT ( & V_tcbinfo ) ; } else { #ifdef INVARIANTS if ( ti_locked == TI_RLOCKED ) { INP_INFO_RLOCK_ASSERT ( & V_tcbinfo ) ; } else { KASSERT ( ti_locked == TI_UNLOCKED , ( " " " " , __func__ , ti_locked ) ) ; INP_INFO_UNLOCK_ASSERT ( & V_tcbinfo ) ; } #endif } INP_WLOCK_ASSERT ( tp -> t_inpcb ) ; KASSERT ( tp -> t_state > TCPS_LISTEN , ( " " , __func__ ) ) ; KASSERT ( tp -> t_state != TCPS_TIME_WAIT , ( " " , __func__ ) ) ; tp -> t_rcvtime = ticks ; if ( TCPS_HAVEESTABLISHED ( tp -> t_state ) ) tcp_timer_activate ( tp , TT_KEEP , TP_KEEPIDLE ( tp ) ) ; tiwin = th -> th_win << tp -> snd_scale ; if ( tp -> t_flags & TF_ECN_PERMIT ) { if ( thflags & TH_CWR ) tp -> t_flags &= ~ TF_ECN_SND_ECE ; switch ( iptos & IPTOS_ECN_MASK ) { case IPTOS_ECN_CE : tp -> t_flags |= TF_ECN_SND_ECE ; TCPSTAT_INC ( tcps_ecn_ce ) ; break ; case IPTOS_ECN_ECT0 : TCPSTAT_INC ( tcps_ecn_ect0 ) ; break ; case IPTOS_ECN_ECT1 : TCPSTAT_INC ( tcps_ecn_ect1 ) ; break ; } if ( thflags & TH_ECE ) { cc_cong_signal ( tp , th , CC_ECN ) ; } } tcp_dooptions ( & to , ( u_char * ) ( th + 1 ) , ( th -> th_off << 2 ) - sizeof ( struct tcphdr ) , ( thflags & TH_SYN ) ? TO_SYN : 0 ) ; if ( ( to . to_flags & TOF_TS ) && ( to . to_tsecr != 0 ) ) { to . to_tsecr -= tp -> ts_offset ; if ( TSTMP_GT ( to . to_tsecr , tcp_ts_getticks ( ) ) ) to . to_tsecr = 0 ; } if ( ( tp -> t_flags & TF_RCVD_TSTMP ) && ! ( to . to_flags & TOF_TS ) ) { if ( ( s = tcp_log_addrs ( inc , th , NULL , NULL ) ) ) { log ( LOG_DEBUG , " " " \n " , s , __func__ ) ; free ( s , M_TCPLOG ) ; } } if ( ! ( tp -> t_flags & TF_RCVD_TSTMP ) && ( to . to_flags & TOF_TS ) ) { if ( ( s = tcp_log_addrs ( inc , th , NULL , NULL ) ) ) { log ( LOG_DEBUG , " " " \n " , s , __func__ ) ; free ( s , M_TCPLOG ) ; } } if ( tp -> t_state == TCPS_SYN_SENT && ( thflags & TH_SYN ) ) { if ( ( to . to_flags & TOF_SCALE ) && ( tp -> t_flags & TF_REQ_SCALE ) ) { tp -> t_flags |= TF_RCVD_SCALE ; tp -> snd_scale = to . to_wscale ; } tp -> snd_wnd = th -> th_win ; if ( to . to_flags & TOF_TS ) { tp -> t_flags |= TF_RCVD_TSTMP ; tp -> ts_recent = to . to_tsval ; tp -> ts_recent_age = tcp_ts_getticks ( ) ; } if ( to . to_flags & TOF_MSS ) tcp_mss ( tp , to . to_mss ) ; if ( ( tp -> t_flags & TF_SACK_PERMIT ) && ( to . to_flags & TOF_SACKPERM ) == 0 ) tp -> t_flags &= ~ TF_SACK_PERMIT ; } if ( __predict_true ( tp -> t_state == TCPS_ESTABLISHED ) && __predict_true ( ( ( to . to_flags & TOF_SACK ) == 0 ) ) && __predict_true ( tlen == 0 ) && __predict_true ( ( thflags & ( TH_SYN | TH_FIN | TH_RST | TH_URG | TH_ACK ) ) == TH_ACK ) && __predict_true ( LIST_EMPTY ( & tp -> t_segq ) ) && __predict_true ( th -> th_seq == tp -> rcv_nxt ) ) { if ( tcp_fastack ( m , th , so , tp , & to , drop_hdrlen , tlen , ti_locked , tiwin ) ) { return ; } } tcp_do_slowpath ( m , th , so , tp , & to , drop_hdrlen , tlen , ti_locked , tiwin , thflags ) ; }
This tcp-do-segment concentrates on making the fastest
 ack processing path.
This tcp - do - segment concentrates on making the fastest ack processing path .
/*
	 * If this is either a state-changing packet or current state isn't
	 * established, we require a write lock on tcbinfo.  Otherwise, we
	 * allow the tcbinfo to be in either alocked or unlocked, as the
	 * caller may have unnecessarily acquired a write lock due to a race.
	 */ /*
	 * Segment received on connection.
	 * Reset idle time and keep-alive timer.
	 * XXX: This should be done after segment
	 * validation to ignore broken/spoofed segs.
	 */ /*
	 * Unscale the window into a 32-bit value.
	 * For the SYN_SENT state the scale is zero.
	 */ /*
	 * TCP ECN processing.
	 */ /* Congestion experienced. */ /*
	 * Parse options on any incoming segment.
	 */ /*
	 * If echoed timestamp is later than the current time,
	 * fall back to non RFC1323 RTT calculation.  Normalize
	 * timestamp if syncookies were used when this connection
	 * was established.
	 */ /*
	 * If timestamps were negotiated during SYN/ACK they should
	 * appear on every segment during this session and vice versa.
	 */ /*
	 * Process options only when we get SYN/ACK back. The SYN case
	 * for incoming connections is handled in tcp_syncache.
	 * According to RFC1323 the window field in a SYN (i.e., a <SYN>
	 * or <SYN,ACK>) segment itself is never scaled.
	 * XXX this is traditional behavior, may need to be cleaned up.
	 */ /*
		 * Initial send window.  It will be updated with
		 * the next incoming segment to the scaled value.
		 */ /*
	 * Header prediction: check for the two common cases
	 * of a uni-directional data xfer.  If the packet has
	 * no control flags, is in-sequence, the window didn't
	 * change and we're not retransmitting, it's a
	 * candidate.  If the length is zero and the ack moved
	 * forward, we're the sender side of the xfer.  Just
	 * free the data acked & wake any higher level process
	 * that was blocked waiting for space.  If the length
	 * is non-zero and the ack didn't move, we're the
	 * receiver side.  If we're getting packets in-order
	 * (the reassembly queue is empty), add the data to
	 * the socket buffer and note that we need a delayed ack.
	 * Make sure that the hidden state-flags are also off.
	 * Since we check for TCPS_ESTABLISHED first, it can only
	 * be TH_NEEDSYN.
	 */