FreeBSD/sys/arm64/arm64/pmap.c
/*
 * This routine is called if the desired page table page does not exist.
 *
 * If page table page allocation fails, this routine may sleep before
 * returning NULL.  It sleeps only if a lock pointer was given.
 *
 * Note: If a page allocation fails at page table level two or three,
 * one or two pages may be held during the wait, only to be released
 * afterwards.  This conservative approach is easily argued to avoid
 * race conditions.
 */
This routine is called if the desired page table page does not exist.
If page table page allocation fails, this routine may sleep before
returning NULL.  It sleeps only if a lock pointer was given.

If a page allocation fails at page table level two or three,
one or two pages may be held during the wait, only to be released
afterwards.  This conservative approach is easily argued to avoid
race conditions.
This routine is called if the desired page table page does not exist . If page table page allocation fails this routine may sleep before returning NULL . It sleeps only if a lock pointer was given . If a page allocation fails at page table level two or three one or two pages may be held during the wait only to be released afterwards . This conservative approach is easily argued to avoid race conditions .
static vm_page_t
_pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
{
	vm_page_t m, l1pg, l2pg;
	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
	if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
	    VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
		if (lockp != NULL) {
			RELEASE_PV_LIST_LOCK(lockp);
			PMAP_UNLOCK(pmap);
			rw_runlock(&pvh_global_lock);
			VM_WAIT;
			rw_rlock(&pvh_global_lock);
			PMAP_LOCK(pmap);
		}
		return (NULL);
	}
	if ((m->flags & PG_ZERO) == 0)
		pmap_zero_page(m);
	if (ptepindex >= (NUL2E + NUL1E)) {
		pd_entry_t *l0;
		vm_pindex_t l0index;
		l0index = ptepindex - (NUL2E + NUL1E);
		l0 = &pmap->pm_l0[l0index];
		pmap_load_store(l0, VM_PAGE_TO_PHYS(m) | L0_TABLE);
		PTE_SYNC(l0);
	} else if (ptepindex >= NUL2E) {
		vm_pindex_t l0index, l1index;
		pd_entry_t *l0, *l1;
		pd_entry_t tl0;
		l1index = ptepindex - NUL2E;
		l0index = l1index >> L0_ENTRIES_SHIFT;
		l0 = &pmap->pm_l0[l0index];
		tl0 = pmap_load(l0);
		if (tl0 == 0) {
			if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index,
			    lockp) == NULL) {
				--m->wire_count;
				atomic_subtract_int(&vm_cnt.v_wire_count, 1);
				vm_page_free_zero(m);
				return (NULL);
			}
		} else {
			l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK);
			l1pg->wire_count++;
		}
		l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK);
		l1 = &l1[ptepindex & Ln_ADDR_MASK];
		pmap_load_store(l1, VM_PAGE_TO_PHYS(m) | L1_TABLE);
		PTE_SYNC(l1);
	} else {
		vm_pindex_t l0index, l1index;
		pd_entry_t *l0, *l1, *l2;
		pd_entry_t tl0, tl1;
		l1index = ptepindex >> Ln_ENTRIES_SHIFT;
		l0index = l1index >> L0_ENTRIES_SHIFT;
		l0 = &pmap->pm_l0[l0index];
		tl0 = pmap_load(l0);
		if (tl0 == 0) {
			if (_pmap_alloc_l3(pmap, NUL2E + l1index,
			    lockp) == NULL) {
				--m->wire_count;
				atomic_subtract_int(&vm_cnt.v_wire_count, 1);
				vm_page_free_zero(m);
				return (NULL);
			}
			tl0 = pmap_load(l0);
			l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK);
			l1 = &l1[l1index & Ln_ADDR_MASK];
		} else {
			l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK);
			l1 = &l1[l1index & Ln_ADDR_MASK];
			tl1 = pmap_load(l1);
			if (tl1 == 0) {
				if (_pmap_alloc_l3(pmap, NUL2E + l1index,
				    lockp) == NULL) {
					--m->wire_count;
					atomic_subtract_int(
					    &vm_cnt.v_wire_count, 1);
					vm_page_free_zero(m);
					return (NULL);
				}
			} else {
				l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK);
				l2pg->wire_count++;
			}
		}
		l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK);
		l2 = &l2[ptepindex & Ln_ADDR_MASK];
		pmap_load_store(l2, VM_PAGE_TO_PHYS(m) | L2_TABLE);
		PTE_SYNC(l2);
	}
	pmap_resident_count_inc(pmap, 1);
	return (m);
}
static vm_page_t _pmap_alloc_l3 ( pmap_t pmap , vm_pindex_t ptepindex , struct rwlock * * lockp ) { vm_page_t m , l1pg , l2pg ; PMAP_LOCK_ASSERT ( pmap , MA_OWNED ) ; if ( ( m = vm_page_alloc ( NULL , ptepindex , VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO ) ) == NULL ) { if ( lockp != NULL ) { RELEASE_PV_LIST_LOCK ( lockp ) ; PMAP_UNLOCK ( pmap ) ; rw_runlock ( & pvh_global_lock ) ; VM_WAIT ; rw_rlock ( & pvh_global_lock ) ; PMAP_LOCK ( pmap ) ; } return ( NULL ) ; } if ( ( m -> flags & PG_ZERO ) == 0 ) pmap_zero_page ( m ) ; if ( ptepindex >= ( NUL2E + NUL1E ) ) { pd_entry_t * l0 ; vm_pindex_t l0index ; l0index = ptepindex - ( NUL2E + NUL1E ) ; l0 = & pmap -> pm_l0 [ l0index ] ; pmap_load_store ( l0 , VM_PAGE_TO_PHYS ( m ) | L0_TABLE ) ; PTE_SYNC ( l0 ) ; } else if ( ptepindex >= NUL2E ) { vm_pindex_t l0index , l1index ; pd_entry_t * l0 , * l1 ; pd_entry_t tl0 ; l1index = ptepindex - NUL2E ; l0index = l1index >> L0_ENTRIES_SHIFT ; l0 = & pmap -> pm_l0 [ l0index ] ; tl0 = pmap_load ( l0 ) ; if ( tl0 == 0 ) { if ( _pmap_alloc_l3 ( pmap , NUL2E + NUL1E + l0index , lockp ) == NULL ) { -- m -> wire_count ; atomic_subtract_int ( & vm_cnt . v_wire_count , 1 ) ; vm_page_free_zero ( m ) ; return ( NULL ) ; } } else { l1pg = PHYS_TO_VM_PAGE ( tl0 & ~ ATTR_MASK ) ; l1pg -> wire_count ++ ; } l1 = ( pd_entry_t * ) PHYS_TO_DMAP ( pmap_load ( l0 ) & ~ ATTR_MASK ) ; l1 = & l1 [ ptepindex & Ln_ADDR_MASK ] ; pmap_load_store ( l1 , VM_PAGE_TO_PHYS ( m ) | L1_TABLE ) ; PTE_SYNC ( l1 ) ; } else { vm_pindex_t l0index , l1index ; pd_entry_t * l0 , * l1 , * l2 ; pd_entry_t tl0 , tl1 ; l1index = ptepindex >> Ln_ENTRIES_SHIFT ; l0index = l1index >> L0_ENTRIES_SHIFT ; l0 = & pmap -> pm_l0 [ l0index ] ; tl0 = pmap_load ( l0 ) ; if ( tl0 == 0 ) { if ( _pmap_alloc_l3 ( pmap , NUL2E + l1index , lockp ) == NULL ) { -- m -> wire_count ; atomic_subtract_int ( & vm_cnt . v_wire_count , 1 ) ; vm_page_free_zero ( m ) ; return ( NULL ) ; } tl0 = pmap_load ( l0 ) ; l1 = ( pd_entry_t * ) PHYS_TO_DMAP ( tl0 & ~ ATTR_MASK ) ; l1 = & l1 [ l1index & Ln_ADDR_MASK ] ; } else { l1 = ( pd_entry_t * ) PHYS_TO_DMAP ( tl0 & ~ ATTR_MASK ) ; l1 = & l1 [ l1index & Ln_ADDR_MASK ] ; tl1 = pmap_load ( l1 ) ; if ( tl1 == 0 ) { if ( _pmap_alloc_l3 ( pmap , NUL2E + l1index , lockp ) == NULL ) { -- m -> wire_count ; atomic_subtract_int ( & vm_cnt . v_wire_count , 1 ) ; vm_page_free_zero ( m ) ; return ( NULL ) ; } } else { l2pg = PHYS_TO_VM_PAGE ( tl1 & ~ ATTR_MASK ) ; l2pg -> wire_count ++ ; } } l2 = ( pd_entry_t * ) PHYS_TO_DMAP ( pmap_load ( l1 ) & ~ ATTR_MASK ) ; l2 = & l2 [ ptepindex & Ln_ADDR_MASK ] ; pmap_load_store ( l2 , VM_PAGE_TO_PHYS ( m ) | L2_TABLE ) ; PTE_SYNC ( l2 ) ; } pmap_resident_count_inc ( pmap , 1 ) ; return ( m ) ; }
This routine is called if the desired page table page does not exist.
This routine is called if the desired page table page does not exist .
/*
	 * Allocate a page table page.
	 */ /*
		 * Indicate the need to retry.  While waiting, the page table
		 * page may have been allocated.
		 */ /*
	 * Map the pagetable page into the process address space, if
	 * it isn't already there.
	 */ /* recurse for allocating page dir */ /* XXX: release mem barrier? */ /* recurse for allocating page dir */ /* recurse for allocating page dir */ /* XXX: release mem barrier? */