/// Construct a new global memory object from a range of initial memory frames.
/// This is typically invoked quite early (we're setting up support for memory allocation).
///
/// We first chop off a small amount of memory from the frames to construct an early
/// TCache (for every NUMA node). Then we construct the big node-caches (NCache) and
/// populate them with remaining (hopefully a lot) memory.
///
/// When this completes we have a bunch of global NUMA aware memory allocators that
/// are protected by spin-locks. `GlobalMemory` together with the core-local allocators
/// forms the tracking for our memory allocation system.
///
/// # Safety
/// Pretty unsafe as we do lot of conjuring objects from frames to allocate memory
/// for our allocators.
/// A client needs to ensure that our frames are valid memory, and not yet
/// being used anywhere yet.
/// The good news is that we only invoke this once during bootstrap.
Construct a new global memory object from a range of initial memory frames.
This is typically invoked quite early (we're setting up support for memory allocation).

We first chop off a small amount of memory from the frames to construct an early
TCache (for every NUMA node). Then we construct the big node-caches (NCache) and
populate them with remaining (hopefully a lot) memory.

When this completes we have a bunch of global NUMA aware memory allocators that
are protected by spin-locks. `GlobalMemory` together with the core-local allocators
forms the tracking for our memory allocation system.
Construct a new global memory object from a range of initial memory frames . This is typically invoked quite early ( we ' re setting up support for memory allocation ) . We first chop off a small amount of memory from the frames to construct an early TCache ( for every NUMA node ) . Then we construct the big node - caches ( NCache ) and populate them with remaining ( hopefully a lot ) memory . When this completes we have a bunch of global NUMA aware memory allocators that are protected by spin - locks . ` GlobalMemory ` together with the core - local allocators forms the tracking for our memory allocation system .
pub unsafe fn new(
        mut memory: ArrayVec<Frame, MAX_PHYSICAL_REGIONS>,
    ) -> Result<GlobalMemory, KError> {
        assert!(memory.is_sorted_by(|a, b| a.affinity.partial_cmp(&b.affinity)));
        debug_assert!(!memory.is_empty());
        let mut gm = GlobalMemory::default();
        // How many NUMA nodes are there in the system
        let max_affinity: usize = memory
            .iter()
            .map(|f| f.affinity as usize)
            .max()
            .expect("Need at least some frames")
            + 1;
        // Construct the `emem`'s for all NUMA nodes:
        let mut cur_affinity = 0;
        // Top of the frames that we didn't end up using for the `emem` construction
        let mut leftovers: ArrayVec<Frame, MAX_PHYSICAL_REGIONS> = ArrayVec::new();
        for frame in memory.iter_mut() {
            const EMEM_SIZE: usize = 2 * LARGE_PAGE_SIZE + 64 * BASE_PAGE_SIZE;
            if frame.affinity == cur_affinity && frame.size() > EMEM_SIZE {
                // Let's make sure we have a frame that starts at a 2 MiB boundary which makes it easier
                // to populate the TCache
                let (low, large_page_aligned_frame) = frame.split_at_nearest_large_page_boundary();
                *frame = low;
                // Cut-away the top memory if the frame we got is too big
                let (emem, leftover_mem) = large_page_aligned_frame.split_at(EMEM_SIZE);
                if leftover_mem != Frame::empty() {
                    // And safe it for later processing
                    leftovers.push(leftover_mem);
                }
                gm.emem.push(Mutex::new(mcache::TCache::new_with_frame(
                    cur_affinity,
                    emem,
                )));
                cur_affinity += 1;
            }
        }
        // If this fails, memory is really fragmented or some nodes have no/little memory
        assert_eq!(
            gm.emem.len(),
            max_affinity,
            "Added early managers for all NUMA nodes"
        );
        // Construct an NCache for all nodes
        for affinity in 0..max_affinity {
            let mut ncache_memory = gm.emem[affinity].lock().allocate_large_page()?;
            let ncache_memory_addr: PAddr = ncache_memory.base;
            assert!(ncache_memory_addr != PAddr::zero());
            ncache_memory.zero(); // TODO(perf) this happens twice atm?
            let ncache_ptr = ncache_memory.uninitialized::<mcache::NCache>();
            let ncache: &'static mut mcache::NCache = mcache::NCache::init(ncache_ptr, affinity);
            debug_assert_eq!(
                &*ncache as *const _ as u64,
                paddr_to_kernel_vaddr(ncache_memory_addr).as_u64()
            );
            gm.node_caches.push(CachePadded::new(Mutex::new(ncache)));
        }
        // Populate the NCaches with all remaining memory
        // Ideally we fully exhaust all frames and put everything in the NCache
        for (ncache_affinity, ncache) in gm.node_caches.iter().enumerate() {
            let mut ncache_locked = ncache.lock();
            for frame in memory.iter() {
                if frame.affinity == ncache_affinity {
                    trace!("Trying to add {:?} frame to {:?}", frame, ncache_locked);
                    ncache_locked.populate_2m_first(*frame);
                }
            }
            for frame in leftovers.iter() {
                if frame.affinity == ncache_affinity {
                    trace!("Trying to add {:?} frame to {:?}", frame, ncache_locked);
                    ncache_locked.populate_2m_first(*frame);
                }
            }
        }
        Ok(gm)
    }
pub unsafe fn new ( mut memory : ArrayVec < Frame , MAX_PHYSICAL_REGIONS > , ) -> Result < GlobalMemory , KError > { assert ! ( memory is_sorted_by ( a b a affinity partial_cmp ( b affinity ) ) ) ; debug_assert ! ( memory is_empty ( ) ) ; let mut gm = GlobalMemory :: default ( ) ; // How many NUMA nodes are there in the system let max_affinity : usize = memory . iter ( ) . map ( | f | f . affinity as usize ) . max ( ) . expect ( " " ) + 1 ; // Construct the `emem`'s for all NUMA nodes: let mut cur_affinity = 0 ; // Top of the frames that we didn't end up using for the `emem` construction let mut leftovers : ArrayVec < Frame , MAX_PHYSICAL_REGIONS > = ArrayVec :: new ( ) ; for frame in memory . iter_mut ( ) { const EMEM_SIZE : usize = 2 * LARGE_PAGE_SIZE + 64 * BASE_PAGE_SIZE ; if frame . affinity == cur_affinity && frame . size ( ) > EMEM_SIZE { // Let's make sure we have a frame that starts at a 2 MiB boundary which makes it easier // to populate the TCache let ( low , large_page_aligned_frame ) = frame . split_at_nearest_large_page_boundary ( ) ; * frame = low ; // Cut-away the top memory if the frame we got is too big let ( emem , leftover_mem ) = large_page_aligned_frame . split_at ( EMEM_SIZE ) ; if leftover_mem != Frame :: empty ( ) { // And safe it for later processing leftovers . push ( leftover_mem ) ; } gm . emem . push ( Mutex :: new ( mcache :: TCache :: new_with_frame ( cur_affinity , emem , ) ) ) ; cur_affinity += 1 ; } } // If this fails, memory is really fragmented or some nodes have no/little memory assert_eq ! ( gm emem len ( ) max_affinity " " ) ; // Construct an NCache for all nodes for affinity in 0 .. max_affinity { let mut ncache_memory = gm . emem [ affinity ] . lock ( ) . allocate_large_page ( ) ? ; let ncache_memory_addr : PAddr = ncache_memory . base ; assert ! ( ncache_memory_addr PAddr zero ( ) ) ; ncache_memory . zero ( ) ; // TODO(perf) this happens twice atm? let ncache_ptr = ncache_memory . uninitialized :: < mcache :: NCache > ( ) ; let ncache : & ' static mut mcache :: NCache = mcache :: NCache :: init ( ncache_ptr , affinity ) ; debug_assert_eq ! ( ncache as const as u64 paddr_to_kernel_vaddr ( ncache_memory_addr ) as_u64 ( ) ) ; gm . node_caches . push ( CachePadded :: new ( Mutex :: new ( ncache ) ) ) ; } // Populate the NCaches with all remaining memory // Ideally we fully exhaust all frames and put everything in the NCache for ( ncache_affinity , ncache ) in gm . node_caches . iter ( ) . enumerate ( ) { let mut ncache_locked = ncache . lock ( ) ; for frame in memory . iter ( ) { if frame . affinity == ncache_affinity { trace ! ( " " frame ncache_locked ) ; ncache_locked . populate_2m_first ( * frame ) ; } } for frame in leftovers . iter ( ) { if frame . affinity == ncache_affinity { trace ! ( " " frame ncache_locked ) ; ncache_locked . populate_2m_first ( * frame ) ; } } } Ok ( gm ) }
Construct a new global memory object from a range of initial memory frames.
Construct a new global memory object from a range of initial memory frames .
memory
ArrayVec<Frame, MAX_PHYSICAL_REGIONS>
safety
Pretty unsafe as we do lot of conjuring objects from frames to allocate memory
for our allocators.
A client needs to ensure that our frames are valid memory, and not yet
being used anywhere yet.
The good news is that we only invoke this once during bootstrap.
Pretty unsafe as we do lot of conjuring objects from frames to allocate memory for our allocators . A client needs to ensure that our frames are valid memory and not yet being used anywhere yet . The good news is that we only invoke this once during bootstrap .