def __getitem__(self, index):
        """Slice the distributed array"""
        # To be a DistArray, must have an axis across >=2 engines. If the slice
        # result will no longer be distributed, we return a RemoteArray instead
        distaxis = self._distaxis
        if isinstance(index, np.ndarray) and index.dtype.type is np.bool_:
            raise Error('indexing by boolean array not yet implemented')
        if not isinstance(index, Sequence):
            index = (index,) + (slice(None),)*(self.ndim - 1)
        ix_types = tuple(type(x) for x in index)
        if (np.ndarray in ix_types or
                (not isinstance(index, _TupleType) and 
                    _NewaxisType not in ix_types and 
                    _EllipsisType not in ix_types and
                    _SliceType not in ix_types) or
                any(issubclass(T, Sequence) for T in ix_types)):
            basic_slicing = False
        else:
            basic_slicing = True
        # Apply any ellipses
        while _EllipsisType in ix_types:
            pos = ix_types.index(_EllipsisType)
            m = (self.ndim + ix_types.count(_NewaxisType) - len(index) + 1)
            index = index[:pos] + (slice(None),)*m + index[(pos+1):]
            ix_types = tuple(type(x) for x in index)
        # Apply any np.newaxis
        if _NewaxisType in ix_types:
            new_distaxis = distaxis
            subix = [slice(None)] * self.ndim
            while _NewaxisType in ix_types:
                pos = ix_types.index(type(np.newaxis))
                index = index[:pos] + (slice(None),) + index[(pos+1):]
                ix_types = tuple(type(x) for x in index)
                if pos <= distaxis:
                    subix[pos] = np.newaxis
                    new_distaxis += 1
                else:
                    subix[pos - 1] = np.newaxis
            new_subarrays = [ra[tuple(subix)] for ra in self._subarrays]
            return DistArray(new_subarrays, new_distaxis)[index]
        index = tuple(index) + (slice(None),)*(self.ndim - len(index))
        if len(index) > self.ndim:
            raise IndexError('too many indices for array')
        if basic_slicing:
            # separate the index acting on distributed axis from other indices
            distix = index[distaxis]
            if isinstance(distix, numbers.Number):
                # distributed axis has been sliced away: return a RemoteArray
                s, i = self._tosub(distix)
                subix = index[0:distaxis] + (i,) + index[(distaxis+1):]
                return self._subarrays[s][subix]
            else:
                result_ras = []
                ss, ms = self._tosubslices(distix)
                for s, m in zip(ss, ms):
                    subix = index[0:distaxis] + (m,) + index[(distaxis+1):]
                    result_ras.append(self._subarrays[s][subix])
                axes_removed = sum(1 for x in index[:distaxis] if isinstance(
                        x, numbers.Integral))
                new_distaxis = distaxis - axes_removed
        else:
            # advanced integer slicing
            is_fancy = tuple(not isinstance(x, _SliceType) for x in index)
            fancy_pos = tuple(i for i in range(len(index)) if is_fancy[i])
            slice_pos = tuple(i for i in range(len(index)) if not is_fancy[i])
            contiguous = (fancy_pos[-1] - fancy_pos[0] == len(fancy_pos) - 1)
            index = list(index)
            ix_arrays = [index[j] for j in fancy_pos]
            ix_arrays = np.broadcast_arrays(*ix_arrays)
            for j in range(len(fancy_pos)):
                if ix_arrays[j].shape is ():
                    ix_arrays[j] = np.expand_dims(ix_arrays[j], 0)
                index[fancy_pos[j]] = ix_arrays[j]
            index = tuple(index)
            idim = index[fancy_pos[0]].ndim # common ndim of all index arrays
            assert(idim > 0)
            distix = index[distaxis]
            otherix = index[0:distaxis] + (slice(None),) + index[(distaxis+1):]
            if not is_fancy[distaxis]:
                # fancy indexing is only being applied to non-distributed axes
                result_ras = []
                ss, ms = self._tosubslices(distix)
                for s, m in zip(ss, ms):
                    subix = index[0:distaxis] + (m,) + index[(distaxis+1):]
                    result_ras.append(self._subarrays[s][subix])
                # predict where that new axis will be in subarrays post-slicing
                if contiguous:
                    if fancy_pos[0] > distaxis:
                        new_distaxis = distaxis
                    else:
                        new_distaxis = distaxis - len(fancy_pos) + idim
                else:
                    earlier_fancy = len([i for i in fancy_pos if i < distaxis])
                    new_distaxis = distaxis - earlier_fancy + idim
            else:
                # fancy indexing is being applied to the distributed axis
                nonconstant_ix_axes = []
                for j in range(idim):
                    n = distix.shape[j]
                    if n > 1:
                        partix = np.split(distix, n, axis=j)
                        if not all(np.array_equal(
                                partix[0], partix[i]) for i in range(1, n)):
                            nonconstant_ix_axes.append(j)
                if len(nonconstant_ix_axes) <= 1:
                    # then we can apply the indexing without moving data
                    if len(nonconstant_ix_axes) is 0:
                        # implies every entry of indexing array for distaxis is
                        # the same, so all result data is on a single engine.
                        all_same_engine = True
                        iax = idim - 1
                    else:
                        # len(nonconstant_ix_axes) is 1
                        # the result will remain distributed on one output axis
                        all_same_engine = False
                        iax = nonconstant_ix_axes[0]
                    iix = [0] * idim
                    iix[iax] = slice(None)
                    iix = tuple(iix)
                    ixlist = self._placeholders[distix[iix]]
                    if contiguous:
                        new_distaxis = fancy_pos[0] + iax
                    else:
                        new_distaxis = 0 + iax
                    result_ras = []
                    ss, ms, js = self._tosubsj(ixlist)
                    shp = [1] * idim
                    for s, m, jlist in zip(ss, ms, js):
                        # Slice the original indexing arrays into smaller
                        # arrays, suitable for indexing our subarrays.
                        shp[iax] = len(m)
                        m = np.array(m).reshape(shp) # make shape broadcastable
                        sl = [slice(None)] * idim
                        sl[iax] = jlist
                        sl = tuple(sl)
                        subix = list(index)
                        for i in range(len(subix)):
                            if isinstance(subix[i], np.ndarray):
                                if i == distaxis:
                                    subix[i] = m
                                else:
                                    subix[i] = subix[i][sl]
                        subix = tuple(subix)
                        result_ras.append(self._subarrays[s][subix])
                    if all_same_engine:
                        result_ras = [expand_dims(r, new_distaxis) for
                                      r in result_ras]
                else:
                    # Have more than one nonconstant indexing axis in the
                    # indexing array that applies to our distributed axis.
                    msg = (u'The requested slicing operation requires moving '
                            'data between engines. Will fetch data locally.')
                    warnings.warn(msg, RuntimeWarning)
                    return self._ob[index]
        if len(result_ras) == 0:
            # result is an array with 0 in its shape
            subix = index[0:distaxis] + (slice(0),) + index[(distaxis+1):]
            return self._subarrays[0][subix]
        if all(not isinstance(ra, RemoteArray) or ra._id.engine == -1 for
                ra in result_ras):
            # all resulting subarrays are local: return an ordinary ndarray
            return gather(concatenate(result_ras, axis=new_distaxis))
        engines = [ra._id.engine if isinstance(ra, RemoteArray) else -1 for
                   ra in result_ras]
        if all(e == engines[0] for e in engines):
            # no longer distributed: return a RemoteArray
            return concatenate(result_ras, axis=new_distaxis)
        else:
            return DistArray(result_ras, new_distaxis)
def __getitem__ ( self , index ) : distaxis = self . _distaxis if isinstance ( index , np . ndarray ) and index . dtype . type is np . bool_ : raise Error ( 'indexing by boolean array not yet implemented' ) if not isinstance ( index , Sequence ) : index = ( index , ) + ( slice ( None ) , ) * ( self . ndim - 1 ) ix_types = tuple ( type ( x ) for x in index ) if ( np . ndarray in ix_types or ( not isinstance ( index , _TupleType ) and _NewaxisType not in ix_types and _EllipsisType not in ix_types and _SliceType not in ix_types ) or any ( issubclass ( T , Sequence ) for T in ix_types ) ) : basic_slicing = False else : basic_slicing = True while _EllipsisType in ix_types : pos = ix_types . index ( _EllipsisType ) m = ( self . ndim + ix_types . count ( _NewaxisType ) - len ( index ) + 1 ) index = index [ : pos ] + ( slice ( None ) , ) * m + index [ ( pos + 1 ) : ] ix_types = tuple ( type ( x ) for x in index ) if _NewaxisType in ix_types : new_distaxis = distaxis subix = [ slice ( None ) ] * self . ndim while _NewaxisType in ix_types : pos = ix_types . index ( type ( np . newaxis ) ) index = index [ : pos ] + ( slice ( None ) , ) + index [ ( pos + 1 ) : ] ix_types = tuple ( type ( x ) for x in index ) if pos <= distaxis : subix [ pos ] = np . newaxis new_distaxis += 1 else : subix [ pos - 1 ] = np . newaxis new_subarrays = [ ra [ tuple ( subix ) ] for ra in self . _subarrays ] return DistArray ( new_subarrays , new_distaxis ) [ index ] index = tuple ( index ) + ( slice ( None ) , ) * ( self . ndim - len ( index ) ) if len ( index ) > self . ndim : raise IndexError ( 'too many indices for array' ) if basic_slicing : distix = index [ distaxis ] if isinstance ( distix , numbers . Number ) : s , i = self . _tosub ( distix ) subix = index [ 0 : distaxis ] + ( i , ) + index [ ( distaxis + 1 ) : ] return self . _subarrays [ s ] [ subix ] else : result_ras = [ ] ss , ms = self . _tosubslices ( distix ) for s , m in zip ( ss , ms ) : subix = index [ 0 : distaxis ] + ( m , ) + index [ ( distaxis + 1 ) : ] result_ras . append ( self . _subarrays [ s ] [ subix ] ) axes_removed = sum ( 1 for x in index [ : distaxis ] if isinstance ( x , numbers . Integral ) ) new_distaxis = distaxis - axes_removed else : is_fancy = tuple ( not isinstance ( x , _SliceType ) for x in index ) fancy_pos = tuple ( i for i in range ( len ( index ) ) if is_fancy [ i ] ) slice_pos = tuple ( i for i in range ( len ( index ) ) if not is_fancy [ i ] ) contiguous = ( fancy_pos [ - 1 ] - fancy_pos [ 0 ] == len ( fancy_pos ) - 1 ) index = list ( index ) ix_arrays = [ index [ j ] for j in fancy_pos ] ix_arrays = np . broadcast_arrays ( * ix_arrays ) for j in range ( len ( fancy_pos ) ) : if ix_arrays [ j ] . shape is ( ) : ix_arrays [ j ] = np . expand_dims ( ix_arrays [ j ] , 0 ) index [ fancy_pos [ j ] ] = ix_arrays [ j ] index = tuple ( index ) idim = index [ fancy_pos [ 0 ] ] . ndim assert ( idim > 0 ) distix = index [ distaxis ] otherix = index [ 0 : distaxis ] + ( slice ( None ) , ) + index [ ( distaxis + 1 ) : ] if not is_fancy [ distaxis ] : result_ras = [ ] ss , ms = self . _tosubslices ( distix ) for s , m in zip ( ss , ms ) : subix = index [ 0 : distaxis ] + ( m , ) + index [ ( distaxis + 1 ) : ] result_ras . append ( self . _subarrays [ s ] [ subix ] ) if contiguous : if fancy_pos [ 0 ] > distaxis : new_distaxis = distaxis else : new_distaxis = distaxis - len ( fancy_pos ) + idim else : earlier_fancy = len ( [ i for i in fancy_pos if i < distaxis ] ) new_distaxis = distaxis - earlier_fancy + idim else : nonconstant_ix_axes = [ ] for j in range ( idim ) : n = distix . shape [ j ] if n > 1 : partix = np . split ( distix , n , axis = j ) if not all ( np . array_equal ( partix [ 0 ] , partix [ i ] ) for i in range ( 1 , n ) ) : nonconstant_ix_axes . append ( j ) if len ( nonconstant_ix_axes ) <= 1 : if len ( nonconstant_ix_axes ) is 0 : all_same_engine = True iax = idim - 1 else : all_same_engine = False iax = nonconstant_ix_axes [ 0 ] iix = [ 0 ] * idim iix [ iax ] = slice ( None ) iix = tuple ( iix ) ixlist = self . _placeholders [ distix [ iix ] ] if contiguous : new_distaxis = fancy_pos [ 0 ] + iax else : new_distaxis = 0 + iax result_ras = [ ] ss , ms , js = self . _tosubsj ( ixlist ) shp = [ 1 ] * idim for s , m , jlist in zip ( ss , ms , js ) : shp [ iax ] = len ( m ) m = np . array ( m ) . reshape ( shp ) sl = [ slice ( None ) ] * idim sl [ iax ] = jlist sl = tuple ( sl ) subix = list ( index ) for i in range ( len ( subix ) ) : if isinstance ( subix [ i ] , np . ndarray ) : if i == distaxis : subix [ i ] = m else : subix [ i ] = subix [ i ] [ sl ] subix = tuple ( subix ) result_ras . append ( self . _subarrays [ s ] [ subix ] ) if all_same_engine : result_ras = [ expand_dims ( r , new_distaxis ) for r in result_ras ] else : msg = ( u'The requested slicing operation requires moving ' 'data between engines. Will fetch data locally.' ) warnings . warn ( msg , RuntimeWarning ) return self . _ob [ index ] if len ( result_ras ) == 0 : subix = index [ 0 : distaxis ] + ( slice ( 0 ) , ) + index [ ( distaxis + 1 ) : ] return self . _subarrays [ 0 ] [ subix ] if all ( not isinstance ( ra , RemoteArray ) or ra . _id . engine == - 1 for ra in result_ras ) : return gather ( concatenate ( result_ras , axis = new_distaxis ) ) engines = [ ra . _id . engine if isinstance ( ra , RemoteArray ) else - 1 for ra in result_ras ] if all ( e == engines [ 0 ] for e in engines ) : return concatenate ( result_ras , axis = new_distaxis ) else : return DistArray ( result_ras , new_distaxis )
Slice the distributed array