linux/kernel/bpf/sockmap.c
/* Locking notes: Concurrent updates, deletes, and lookups are allowed and are
 * done inside rcu critical sections. This ensures on updates that the psock
 * will not be released via smap_release_sock() until concurrent updates/deletes
 * complete. All operations operate on sock_map using cmpxchg and xchg
 * operations to ensure we do not get stale references. Any reads into the
 * map must be done with READ_ONCE() because of this.
 *
 * A psock is destroyed via call_rcu and after any worker threads are cancelled
 * and syncd so we are certain all references from the update/lookup/delete
 * operations as well as references in the data path are no longer in use.
 *
 * Psocks may exist in multiple maps, but only a single set of parse/verdict
 * programs may be inherited from the maps it belongs to. A reference count
 * is kept with the total number of references to the psock from all maps. The
 * psock will not be released until this reaches zero. The psock and sock
 * user data data use the sk_callback_lock to protect critical data structures
 * from concurrent access. This allows us to avoid two updates from modifying
 * the user data in sock and the lock is required anyways for modifying
 * callbacks, we simply increase its scope slightly.
 *
 * Rules to follow,
 *  - psock must always be read inside RCU critical section
 *  - sk_user_data must only be modified inside sk_callback_lock and read
 *    inside RCU critical section.
 *  - psock->maps list must only be read & modified inside sk_callback_lock
 *  - sock_map must use READ_ONCE and (cmp)xchg operations
 *  - BPF verdict/parse programs must use READ_ONCE and xchg operations
 */
Locking notes: Concurrent updates, deletes, and lookups are allowed and are
done inside rcu critical sections. This ensures on updates that the psock
will not be released via smap_release_sock() until concurrent updates/deletes
complete. All operations operate on sock_map using cmpxchg and xchg
operations to ensure we do not get stale references. Any reads into the
map must be done with READ_ONCE() because of this.

A psock is destroyed via call_rcu and after any worker threads are cancelled
and syncd so we are certain all references from the update/lookup/delete
operations as well as references in the data path are no longer in use.

Psocks may exist in multiple maps, but only a single set of parse/verdict
programs may be inherited from the maps it belongs to. A reference count
is kept with the total number of references to the psock from all maps. The
psock will not be released until this reaches zero. The psock and sock
user data data use the sk_callback_lock to protect critical data structures
from concurrent access. This allows us to avoid two updates from modifying
the user data in sock and the lock is required anyways for modifying
callbacks, we simply increase its scope slightly.

Rules to follow,
- psock must always be read inside RCU critical section
- sk_user_data must only be modified inside sk_callback_lock and read
inside RCU critical section.
- psock->maps list must only be read & modified inside sk_callback_lock
- sock_map must use READ_ONCE and (cmp)xchg operations
- BPF verdict/parse programs must use READ_ONCE and xchg operations
Locking notes : Concurrent updates deletes and lookups are allowed and are done inside rcu critical sections . This ensures on updates that the psock will not be released via smap_release_sock () until concurrent updates / deletes complete . All operations operate on sock_map using cmpxchg and xchg operations to ensure we do not get stale references . Any reads into the map must be done with READ_ONCE () because of this . A psock is destroyed via call_rcu and after any worker threads are cancelled and syncd so we are certain all references from the update / lookup / delete operations as well as references in the data path are no longer in use . Psocks may exist in multiple maps but only a single set of parse / verdict programs may be inherited from the maps it belongs to . A reference count is kept with the total number of references to the psock from all maps . The psock will not be released until this reaches zero . The psock and sock user data data use the sk_callback_lock to protect critical data structures from concurrent access . This allows us to avoid two updates from modifying the user data in sock and the lock is required anyways for modifying callbacks we simply increase its scope slightly . Rules to follow - psock must always be read inside RCU critical section - sk_user_data must only be modified inside sk_callback_lock and read inside RCU critical section . - psock - > maps list must only be read & modified inside sk_callback_lock - sock_map must use READ_ONCE and ( cmp ) xchg operations - BPF verdict / parse programs must use READ_ONCE and xchg operations
static int __sock_map_ctx_update_elem(struct bpf_map *map,
				      struct bpf_sock_progs *progs,
				      struct sock *sock,
				      void *key)
{
	struct bpf_prog *verdict, *parse, *tx_msg;
	struct smap_psock *psock;
	bool new = false;
	int err = 0;
	verdict = READ_ONCE(progs->bpf_verdict);
	parse = READ_ONCE(progs->bpf_parse);
	tx_msg = READ_ONCE(progs->bpf_tx_msg);
	if (parse && verdict) {
		verdict = bpf_prog_inc_not_zero(verdict);
		if (IS_ERR(verdict))
			return PTR_ERR(verdict);
		parse = bpf_prog_inc_not_zero(parse);
		if (IS_ERR(parse)) {
			bpf_prog_put(verdict);
			return PTR_ERR(parse);
		}
	}
	if (tx_msg) {
		tx_msg = bpf_prog_inc_not_zero(tx_msg);
		if (IS_ERR(tx_msg)) {
			if (parse && verdict) {
				bpf_prog_put(parse);
				bpf_prog_put(verdict);
			}
			return PTR_ERR(tx_msg);
		}
	}
	psock = smap_psock_sk(sock);
	if (psock) {
		if (!psock_is_smap_sk(sock)) {
			err = -EBUSY;
			goto out_progs;
		}
		if (READ_ONCE(psock->bpf_parse) && parse) {
			err = -EBUSY;
			goto out_progs;
		}
		if (READ_ONCE(psock->bpf_tx_msg) && tx_msg) {
			err = -EBUSY;
			goto out_progs;
		}
		if (!refcount_inc_not_zero(&psock->refcnt)) {
			err = -EAGAIN;
			goto out_progs;
		}
	} else {
		psock = smap_init_psock(sock, map->numa_node);
		if (IS_ERR(psock)) {
			err = PTR_ERR(psock);
			goto out_progs;
		}
		set_bit(SMAP_TX_RUNNING, &psock->state);
		new = true;
	}
	if (tx_msg)
		bpf_tcp_msg_add(psock, sock, tx_msg);
	if (new) {
		err = tcp_set_ulp_id(sock, TCP_ULP_BPF);
		if (err)
			goto out_free;
	}
	if (parse && verdict && !psock->strp_enabled) {
		err = smap_init_sock(psock, sock);
		if (err)
			goto out_free;
		smap_init_progs(psock, verdict, parse);
		write_lock_bh(&sock->sk_callback_lock);
		smap_start_sock(psock, sock);
		write_unlock_bh(&sock->sk_callback_lock);
	}
	return err;
out_free:
	smap_release_sock(psock, sock);
out_progs:
	if (parse && verdict) {
		bpf_prog_put(parse);
		bpf_prog_put(verdict);
	}
	if (tx_msg)
		bpf_prog_put(tx_msg);
	return err;
}
static int __sock_map_ctx_update_elem ( struct bpf_map * map , struct bpf_sock_progs * progs , struct sock * sock , void * key ) { struct bpf_prog * verdict , * parse , * tx_msg ; struct smap_psock * psock ; bool new = false ; int err = 0 ; verdict = READ_ONCE ( progs -> bpf_verdict ) ; parse = READ_ONCE ( progs -> bpf_parse ) ; tx_msg = READ_ONCE ( progs -> bpf_tx_msg ) ; if ( parse && verdict ) { verdict = bpf_prog_inc_not_zero ( verdict ) ; if ( IS_ERR ( verdict ) ) return PTR_ERR ( verdict ) ; parse = bpf_prog_inc_not_zero ( parse ) ; if ( IS_ERR ( parse ) ) { bpf_prog_put ( verdict ) ; return PTR_ERR ( parse ) ; } } if ( tx_msg ) { tx_msg = bpf_prog_inc_not_zero ( tx_msg ) ; if ( IS_ERR ( tx_msg ) ) { if ( parse && verdict ) { bpf_prog_put ( parse ) ; bpf_prog_put ( verdict ) ; } return PTR_ERR ( tx_msg ) ; } } psock = smap_psock_sk ( sock ) ; if ( psock ) { if ( ! psock_is_smap_sk ( sock ) ) { err = - EBUSY ; goto out_progs ; } if ( READ_ONCE ( psock -> bpf_parse ) && parse ) { err = - EBUSY ; goto out_progs ; } if ( READ_ONCE ( psock -> bpf_tx_msg ) && tx_msg ) { err = - EBUSY ; goto out_progs ; } if ( ! refcount_inc_not_zero ( & psock -> refcnt ) ) { err = - EAGAIN ; goto out_progs ; } } else { psock = smap_init_psock ( sock , map -> numa_node ) ; if ( IS_ERR ( psock ) ) { err = PTR_ERR ( psock ) ; goto out_progs ; } set_bit ( SMAP_TX_RUNNING , & psock -> state ) ; new = true ; } if ( tx_msg ) bpf_tcp_msg_add ( psock , sock , tx_msg ) ; if ( new ) { err = tcp_set_ulp_id ( sock , TCP_ULP_BPF ) ; if ( err ) goto out_free ; } if ( parse && verdict && ! psock -> strp_enabled ) { err = smap_init_sock ( psock , sock ) ; if ( err ) goto out_free ; smap_init_progs ( psock , verdict , parse ) ; write_lock_bh ( & sock -> sk_callback_lock ) ; smap_start_sock ( psock , sock ) ; write_unlock_bh ( & sock -> sk_callback_lock ) ; } return err ; out_free : smap_release_sock ( psock , sock ) ; out_progs : if ( parse && verdict ) { bpf_prog_put ( parse ) ; bpf_prog_put ( verdict ) ; } if ( tx_msg ) bpf_prog_put ( tx_msg ) ; return err ; }
Locking notes: Concurrent updates, deletes, and lookups are allowed and are
 done inside rcu critical sections.
Locking notes : Concurrent updates deletes and lookups are allowed and are done inside rcu critical sections .
/* 1. If sock map has BPF programs those will be inherited by the
	 * sock being added. If the sock is already attached to BPF programs
	 * this results in an error.
	 */ /* bpf prog refcnt may be zero if a concurrent attach operation
		 * removes the program after the above READ_ONCE() but before
		 * we increment the refcnt. If this is the case abort with an
		 * error.
		 */ /* 2. Do not allow inheriting programs if psock exists and has
	 * already inherited programs. This would create confusion on
	 * which parser/verdict program is running. If no psock exists
	 * create one. Inside sk_callback_lock to ensure concurrent create
	 * doesn't update user data.
	 */ /* 3. At this point we have a reference to a valid psock that is
	 * running. Attach any BPF programs needed.
	 */