FreeBSD/sys/arm64/arm64/pmap.c
/*
 * grow the number of kernel page table entries, if needed
 */
grow the number of kernel page table entries, if needed
grow the number of kernel page table entries if needed
void
pmap_growkernel(vm_offset_t addr)
{
	vm_paddr_t paddr;
	vm_page_t nkpg;
	pd_entry_t *l0, *l1, *l2;
	mtx_assert(&kernel_map->system_mtx, MA_OWNED);
	addr = roundup2(addr, L2_SIZE);
	if (addr - 1 >= kernel_map->max_offset)
		addr = kernel_map->max_offset;
	while (kernel_vm_end < addr) {
		l0 = pmap_l0(kernel_pmap, kernel_vm_end);
		KASSERT(pmap_load(l0) != 0,
		    ("pmap_growkernel: No level 0 kernel entry"));
		l1 = pmap_l0_to_l1(l0, kernel_vm_end);
		if (pmap_load(l1) == 0) {
			nkpg = vm_page_alloc(NULL, kernel_vm_end >> L1_SHIFT,
			    VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ |
			    VM_ALLOC_WIRED | VM_ALLOC_ZERO);
			if (nkpg == NULL)
				panic("pmap_growkernel: no memory to grow kernel");
			if ((nkpg->flags & PG_ZERO) == 0)
				pmap_zero_page(nkpg);
			paddr = VM_PAGE_TO_PHYS(nkpg);
			pmap_load_store(l1, paddr | L1_TABLE);
			PTE_SYNC(l1);
			continue; 
		}
		l2 = pmap_l1_to_l2(l1, kernel_vm_end);
		if ((pmap_load(l2) & ATTR_AF) != 0) {
			kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET;
			if (kernel_vm_end - 1 >= kernel_map->max_offset) {
				kernel_vm_end = kernel_map->max_offset;
				break;                       
			}
			continue;
		}
		nkpg = vm_page_alloc(NULL, kernel_vm_end >> L2_SHIFT,
		    VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
		    VM_ALLOC_ZERO);
		if (nkpg == NULL)
			panic("pmap_growkernel: no memory to grow kernel");
		if ((nkpg->flags & PG_ZERO) == 0)
			pmap_zero_page(nkpg);
		paddr = VM_PAGE_TO_PHYS(nkpg);
		pmap_load_store(l2, paddr | L2_TABLE);
		PTE_SYNC(l2);
		pmap_invalidate_page(kernel_pmap, kernel_vm_end);
		kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET;
		if (kernel_vm_end - 1 >= kernel_map->max_offset) {
			kernel_vm_end = kernel_map->max_offset;
			break;                       
		}
	}
}
void pmap_growkernel ( vm_offset_t addr ) { vm_paddr_t paddr ; vm_page_t nkpg ; pd_entry_t * l0 , * l1 , * l2 ; mtx_assert ( & kernel_map -> system_mtx , MA_OWNED ) ; addr = roundup2 ( addr , L2_SIZE ) ; if ( addr - 1 >= kernel_map -> max_offset ) addr = kernel_map -> max_offset ; while ( kernel_vm_end < addr ) { l0 = pmap_l0 ( kernel_pmap , kernel_vm_end ) ; KASSERT ( pmap_load ( l0 ) != 0 , ( " " ) ) ; l1 = pmap_l0_to_l1 ( l0 , kernel_vm_end ) ; if ( pmap_load ( l1 ) == 0 ) { nkpg = vm_page_alloc ( NULL , kernel_vm_end >> L1_SHIFT , VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO ) ; if ( nkpg == NULL ) panic ( " " ) ; if ( ( nkpg -> flags & PG_ZERO ) == 0 ) pmap_zero_page ( nkpg ) ; paddr = VM_PAGE_TO_PHYS ( nkpg ) ; pmap_load_store ( l1 , paddr | L1_TABLE ) ; PTE_SYNC ( l1 ) ; continue ; } l2 = pmap_l1_to_l2 ( l1 , kernel_vm_end ) ; if ( ( pmap_load ( l2 ) & ATTR_AF ) != 0 ) { kernel_vm_end = ( kernel_vm_end + L2_SIZE ) & ~ L2_OFFSET ; if ( kernel_vm_end - 1 >= kernel_map -> max_offset ) { kernel_vm_end = kernel_map -> max_offset ; break ; } continue ; } nkpg = vm_page_alloc ( NULL , kernel_vm_end >> L2_SHIFT , VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO ) ; if ( nkpg == NULL ) panic ( " " ) ; if ( ( nkpg -> flags & PG_ZERO ) == 0 ) pmap_zero_page ( nkpg ) ; paddr = VM_PAGE_TO_PHYS ( nkpg ) ; pmap_load_store ( l2 , paddr | L2_TABLE ) ; PTE_SYNC ( l2 ) ; pmap_invalidate_page ( kernel_pmap , kernel_vm_end ) ; kernel_vm_end = ( kernel_vm_end + L2_SIZE ) & ~ L2_OFFSET ; if ( kernel_vm_end - 1 >= kernel_map -> max_offset ) { kernel_vm_end = kernel_map -> max_offset ; break ; } } }
grow the number of kernel page table entries, if needed
grow the number of kernel page table entries if needed
/* We need a new PDP entry */ /* try again */