def clear(self):
        """
        Completely resets the database. This means that all information in
        the local cache and on disc will be erased.
        """
        if not os.path.isdir(self.fdir):
            os.makedirs(self.fdir, exist_ok=True)
            return

        for f in os.listdir(self.fdir):
            del self[f]

def clear ( self ) : if not os . path . isdir ( self . fdir ) : os . makedirs ( self . fdir , exist_ok = True ) return for f in os . listdir ( self . fdir ) : del self [ f ]

Completely resets the database . This means that all information in the local cache and on disc will be erased .


def scrape(ctx, url):
    """
    Rip the events from a given rss feed, normalize the data and store.
    """
    data = load_feed(url)

    feed = data['feed']
    entries = data['entries']

    # THIS IS SPECIFIC TO # http://konfery.cz/rss/

    _type = 'community'
    country = 'Czech Republic'
    # title, title_detail, links, link, published, summary, tags
    # unused: summary_detail, guidislink, published_parsed
    for entry in entries:
        _id = sluggify(entry['id'])
        city = entry['tags'][0]['term']
        landing = entry['link']
        start_time = dt_normalize(entry['published_parsed'], local_tz=True)
        title = entry['title']
        summary = entry['summary']
        link = entry['link']

        ipdb.set_trace()

Rip the events from a given rss feed normalize the data and store .


def download_image(self):
        """
        Download the image and return the
        local path to the image file.
        """
        split = urlsplit(self.url)
        filename = split.path.split("/")[-1]

        # Ensure the directory to store the image cache exists
        if not os.path.exists(self.cache_directory):
            os.makedirs(self.cache_directory)

        filepath = os.path.join(self.cache_directory, filename)

        data = urllib_request.urlopen(self.url)
        with open(filepath, "wb") as image:
            image.write(data.read())

        return filepath

Download the image and return the local path to the image file .


def has_changed(self):
        """
        Method to check if an image has changed
        since it was last downloaded. By making
        a head request, this check can be done
        quicker that downloading and processing
        the whole file.
        """
        request = urllib_request.Request(self.url)
        request.get_method = lambda: 'HEAD'

        response = urllib_request.urlopen(request)
        information = response.info()

        if 'Last-Modified' in information:
            last_modified = information['Last-Modified']

            # Return False if the image has not been modified
            if last_modified == self.image_last_modified:
                return False

        self.image_last_modified = last_modified

        # Return True if the image has been modified
        # or if the image has no last-modified header
        return True

Method to check if an image has changed since it was last downloaded . By making a head request this check can be done quicker that downloading and processing the whole file .


def fancy_tag_compiler(params, defaults, takes_var_args, takes_var_kwargs, takes_context, name, node_class, parser, token):
    "Returns a template.Node subclass."
    bits = token.split_contents()[1:]

    if takes_context:
        if 'context' in params[:1]:
            params = params[1:]
        else:
            raise TemplateSyntaxError(
                "Any tag function decorated with takes_context=True "
                "must have a first argument of 'context'")

    # Split args and kwargs
    args = []
    kwargs = {}
    kwarg_found = False
    unhandled_params = list(params)
    handled_params = []
    if len(bits) > 1 and bits[-2] == 'as':
        output_var = bits[-1]
        if len(set(output_var) - set(ALLOWED_VARIABLE_CHARS)) > 0:
            raise TemplateSyntaxError("%s got output var name with forbidden chars: '%s'" % (name, output_var))
        bits = bits[:-2]
    else:
        output_var = None
    for bit in bits:
        kwarg_match = kwarg_re.match(bit)
        if kwarg_match:
            kw, var = kwarg_match.groups()
            if kw not in params and not takes_var_kwargs:
                raise TemplateSyntaxError("%s got unknown keyword argument '%s'" % (name, kw))
            elif kw in handled_params:
                raise TemplateSyntaxError("%s got multiple values for keyword argument '%s'" % (name, kw))
            else:
                kwargs[str(kw)] = var
                kwarg_found = True
                handled_params.append(kw)
        else:
            if kwarg_found:
                raise TemplateSyntaxError("%s got non-keyword arg after keyword arg" % name)
            else:
                args.append(bit)
                try:
                    handled_params.append(unhandled_params.pop(0))
                except IndexError:
                    if not takes_var_args:
                        raise TemplateSyntaxError("%s got too many arguments" % name)
    # Consider the last n params handled, where n is the number of defaults.
    if defaults is not None:
        unhandled_params = unhandled_params[:-len(defaults)]
    if len(unhandled_params) == 1:
        raise TemplateSyntaxError("%s didn't get a value for argument '%s'" % (name, unhandled_params[0]))
    elif len(unhandled_params) > 1:
        raise TemplateSyntaxError("%s didn't get values for arguments: %s" % (
                name, ', '.join(["'%s'" % p for p in unhandled_params])))

    return node_class(args, kwargs, output_var, takes_context)


def findCaller(self, stack_info=False):
        """
        Find the stack frame of the caller so that we can note the source
        file name, line number and function name.
        """
        f = logging.currentframe()
        # On some versions of IronPython, currentframe() returns None if
        # IronPython isn't run with -X:Frames.
        if f is not None:
            f = f.f_back
        rv = "(unknown file)", 0, "(unknown function)"
        while hasattr(f, "f_code"):
            co = f.f_code
            filename = os.path.normcase(co.co_filename)
            if filename == logging._srcfile or filename == self._srcfile:
                f = f.f_back
                continue
            rv = (co.co_filename, f.f_lineno, co.co_name)
            if stack_info:
                sio = io.StringIO()
                sio.write('Stack (most recent call last):\n')
                traceback.print_stack(f, file=sio)
                sinfo = sio.getvalue()
                if sinfo[-1] == '\n':
                    sinfo = sinfo[:-1]
                sio.close()
            break
        return rv

Find the stack frame of the caller so that we can note the source file name line number and function name .


def get_defining_component(pe_pe):
    '''
    get the C_C in which pe_pe is defined
    '''
    if pe_pe is None:
        return None

    if pe_pe.__class__.__name__ != 'PE_PE':
        pe_pe = xtuml.navigate_one(pe_pe).PE_PE[8001]()

    ep_pkg = xtuml.navigate_one(pe_pe).EP_PKG[8000]()
    if ep_pkg:
        return get_defining_component(ep_pkg)

    return xtuml.navigate_one(pe_pe).C_C[8003]()

def get_defining_component ( pe_pe ) : if pe_pe is None : return None if pe_pe . __class__ . __name__ != 'PE_PE' : pe_pe = xtuml . navigate_one ( pe_pe ) . PE_PE [ 8001 ] ( ) ep_pkg = xtuml . navigate_one ( pe_pe ) . EP_PKG [ 8000 ] ( ) if ep_pkg : return get_defining_component ( ep_pkg ) return xtuml . navigate_one ( pe_pe ) . C_C [ 8003 ] ( )


def main():
    '''
    Parse command line options and launch the prebuilder.
    '''
    parser = optparse.OptionParser(usage="%prog [options] <model_path> [another_model_path..]",
                                   version=xtuml.version.complete_string,
                                   formatter=optparse.TitledHelpFormatter())

    parser.add_option("-v", "--verbosity", dest='verbosity',
                                           action="count",
                                           help="increase debug logging level",
                                           default=1)

    parser.add_option("-o", "--output", dest="output", metavar="PATH",
                                        help="set output to PATH",
                                        action="store",
                                        default=None)

    (opts, args) = parser.parse_args()
    if len(args) == 0 or opts.output is None:
        parser.print_help()
        sys.exit(1)

    levels = {
              0: logging.ERROR,
              1: logging.WARNING,
              2: logging.INFO,
              3: logging.DEBUG,
    }
    logging.basicConfig(level=levels.get(opts.verbosity, logging.DEBUG))

    m = ooaofooa.load_metamodel(args)
    prebuild_model(m)

    xtuml.persist_instances(m, opts.output)

Parse command line options and launch the prebuilder .


def find_symbol(self, name=None, kind=None):
        '''
        Find a symbol in the symbol table by name, kind, or both.
        '''
        for s in reversed(self.stack):

            for symbol_name, handle in s.symbols.items():
                symbol_kind = handle.__class__.__name__

                if name == symbol_name and kind == symbol_kind:
                    return handle

                elif name is None and kind == handle.__class__.__name__:
                    return handle

                elif name == symbol_name and kind is None:
                    return handle

            if name is None and kind == s.handle.__class__.__name__:
                return s.handle

def find_symbol ( self , name = None , kind = None ) : for s in reversed ( self . stack ) : for symbol_name , handle in s . symbols . items ( ) : symbol_kind = handle . __class__ . __name__ if name == symbol_name and kind == symbol_kind : return handle elif name is None and kind == handle . __class__ . __name__ : return handle elif name == symbol_name and kind is None : return handle if name is None and kind == s . handle . __class__ . __name__ : return s . handle

Find a symbol in the symbol table by name kind or both .


def is_contained_in(pe_pe, root):
    '''
    Determine if a PE_PE is contained within a EP_PKG or a C_C.
    '''
    if not pe_pe:
        return False

    if type(pe_pe).__name__ != 'PE_PE':
        pe_pe = one(pe_pe).PE_PE[8001]()

    ep_pkg = one(pe_pe).EP_PKG[8000]()
    c_c = one(pe_pe).C_C[8003]()

    if root in [ep_pkg, c_c]:
        return True

    elif is_contained_in(ep_pkg, root):
        return True

    elif is_contained_in(c_c, root):
        return True

    else:
        return False

def is_contained_in ( pe_pe , root ) : if not pe_pe : return False if type ( pe_pe ) . __name__ != 'PE_PE' : pe_pe = one ( pe_pe ) . PE_PE [ 8001 ] ( ) ep_pkg = one ( pe_pe ) . EP_PKG [ 8000 ] ( ) c_c = one ( pe_pe ) . C_C [ 8003 ] ( ) if root in [ ep_pkg , c_c ] : return True elif is_contained_in ( ep_pkg , root ) : return True elif is_contained_in ( c_c , root ) : return True else : return False

Determine if a PE_PE is contained within a EP_PKG or a C_C .


def is_global(pe_pe):
    '''
    Check if a PE_PE is globally defined, i.e. not inside a C_C
    '''
    if type(pe_pe).__name__ != 'PE_PE':
        pe_pe = one(pe_pe).PE_PE[8001]()

    if one(pe_pe).C_C[8003]():
        return False

    pe_pe = one(pe_pe).EP_PKG[8000].PE_PE[8001]()
    if not pe_pe:
        return True

    return is_global(pe_pe)

def is_global ( pe_pe ) : if type ( pe_pe ) . __name__ != 'PE_PE' : pe_pe = one ( pe_pe ) . PE_PE [ 8001 ] ( ) if one ( pe_pe ) . C_C [ 8003 ] ( ) : return False pe_pe = one ( pe_pe ) . EP_PKG [ 8000 ] . PE_PE [ 8001 ] ( ) if not pe_pe : return True return is_global ( pe_pe )

Check if a PE_PE is globally defined i . e . not inside a C_C


def _get_data_type_name(s_dt):
    '''
    Convert a BridgePoint data type to a pyxtuml meta model type.
    '''
    s_cdt = one(s_dt).S_CDT[17]()
    if s_cdt and s_cdt.Core_Typ in range(1, 6):
        return s_dt.Name.upper()

    if one(s_dt).S_EDT[17]():
        return 'INTEGER'

    s_dt = one(s_dt).S_UDT[17].S_DT[18]()
    if s_dt:
        return _get_data_type_name(s_dt)

def _get_data_type_name ( s_dt ) : s_cdt = one ( s_dt ) . S_CDT [ 17 ] ( ) if s_cdt and s_cdt . Core_Typ in range ( 1 , 6 ) : return s_dt . Name . upper ( ) if one ( s_dt ) . S_EDT [ 17 ] ( ) : return 'INTEGER' s_dt = one ( s_dt ) . S_UDT [ 17 ] . S_DT [ 18 ] ( ) if s_dt : return _get_data_type_name ( s_dt )

Convert a BridgePoint data type to a pyxtuml meta model type .


def _get_related_attributes(r_rgo, r_rto):
    '''
    The two lists of attributes which relates two classes in an association.
    '''
    l1 = list()
    l2 = list()

    ref_filter = lambda ref: ref.OIR_ID == r_rgo.OIR_ID
    for o_ref in many(r_rto).O_RTIDA[110].O_REF[111](ref_filter):
        o_attr = one(o_ref).O_RATTR[108].O_ATTR[106]()
        l1.append(o_attr.Name)

        o_attr = one(o_ref).O_RTIDA[111].O_OIDA[110].O_ATTR[105]()
        l2.append(o_attr.Name)

    return l1, l2

def _get_related_attributes ( r_rgo , r_rto ) : l1 = list ( ) l2 = list ( ) ref_filter = lambda ref : ref . OIR_ID == r_rgo . OIR_ID for o_ref in many ( r_rto ) . O_RTIDA [ 110 ] . O_REF [ 111 ] ( ref_filter ) : o_attr = one ( o_ref ) . O_RATTR [ 108 ] . O_ATTR [ 106 ] ( ) l1 . append ( o_attr . Name ) o_attr = one ( o_ref ) . O_RTIDA [ 111 ] . O_OIDA [ 110 ] . O_ATTR [ 105 ] ( ) l2 . append ( o_attr . Name ) return l1 , l2

The two lists of attributes which relates two classes in an association .


def mk_enum(s_edt):
    '''
    Create a named tuple from a BridgePoint enumeration.
    '''
    s_dt = one(s_edt).S_DT[17]()
    enums = list()
    kwlist =['False', 'None', 'True'] + keyword.kwlist
    for enum in many(s_edt).S_ENUM[27]():
        if enum.Name in kwlist:
            enums.append(enum.Name + '_')
        else:
            enums.append(enum.Name)

    Enum = collections.namedtuple(s_dt.Name, enums)
    return Enum(*range(len(enums)))

def mk_enum ( s_edt ) : s_dt = one ( s_edt ) . S_DT [ 17 ] ( ) enums = list ( ) kwlist = [ 'False' , 'None' , 'True' ] + keyword . kwlist for enum in many ( s_edt ) . S_ENUM [ 27 ] ( ) : if enum . Name in kwlist : enums . append ( enum . Name + '_' ) else : enums . append ( enum . Name ) Enum = collections . namedtuple ( s_dt . Name , enums ) return Enum ( * range ( len ( enums ) ) )

Create a named tuple from a BridgePoint enumeration .


def mk_bridge(metamodel, s_brg):
    '''
    Create a python function from a BridgePoint bridge.
    '''
    action = s_brg.Action_Semantics_internal
    label = s_brg.Name
    return lambda **kwargs: interpret.run_function(metamodel, label,
                                                   action, kwargs)

def mk_bridge ( metamodel , s_brg ) : action = s_brg . Action_Semantics_internal label = s_brg . Name return lambda * * kwargs : interpret . run_function ( metamodel , label , action , kwargs )

Create a python function from a BridgePoint bridge .


def mk_external_entity(metamodel, s_ee):
    '''
    Create a python object from a BridgePoint external entity with bridges
    realized as python member functions.
    '''
    bridges = many(s_ee).S_BRG[19]()
    names = [brg.Name for brg in bridges]
    EE = collections.namedtuple(s_ee.Key_Lett, names)

    funcs = list()
    for s_brg in many(s_ee).S_BRG[19]():
        fn = mk_bridge(metamodel, s_brg)
        funcs.append(fn)

    return EE(*funcs)

def mk_external_entity ( metamodel , s_ee ) : bridges = many ( s_ee ) . S_BRG [ 19 ] ( ) names = [ brg . Name for brg in bridges ] EE = collections . namedtuple ( s_ee . Key_Lett , names ) funcs = list ( ) for s_brg in many ( s_ee ) . S_BRG [ 19 ] ( ) : fn = mk_bridge ( metamodel , s_brg ) funcs . append ( fn ) return EE ( * funcs )

Create a python object from a BridgePoint external entity with bridges realized as python member functions .


def mk_function(metamodel, s_sync):
    '''
    Create a python function from a BridgePoint function.
    '''
    action = s_sync.Action_Semantics_internal
    label = s_sync.Name
    return lambda **kwargs: interpret.run_function(metamodel, label,
                                                   action, kwargs)

def mk_function ( metamodel , s_sync ) : action = s_sync . Action_Semantics_internal label = s_sync . Name return lambda * * kwargs : interpret . run_function ( metamodel , label , action , kwargs )

Create a python function from a BridgePoint function .


def mk_constant(cnst_syc):
    '''
    Create a python value from a BridgePoint constant.
    '''
    s_dt = one(cnst_syc).S_DT[1500]()
    cnst_lsc = one(cnst_syc).CNST_LFSC[1502].CNST_LSC[1503]()

    if s_dt.Name == 'boolean':
        return cnst_lsc.Value.lower() == 'true'

    if s_dt.Name == 'integer':
        return int(cnst_lsc.Value)

    if s_dt.Name == 'real':
        return float(cnst_lsc.Value)

    if s_dt.Name == 'string':
        return str(cnst_lsc.Value)

def mk_constant ( cnst_syc ) : s_dt = one ( cnst_syc ) . S_DT [ 1500 ] ( ) cnst_lsc = one ( cnst_syc ) . CNST_LFSC [ 1502 ] . CNST_LSC [ 1503 ] ( ) if s_dt . Name == 'boolean' : return cnst_lsc . Value . lower ( ) == 'true' if s_dt . Name == 'integer' : return int ( cnst_lsc . Value ) if s_dt . Name == 'real' : return float ( cnst_lsc . Value ) if s_dt . Name == 'string' : return str ( cnst_lsc . Value )

Create a python value from a BridgePoint constant .


def mk_operation(metaclass, o_tfr):
    '''
    Create a python function that interprets that action of a BridgePoint class
    operation.
    '''
    o_obj = one(o_tfr).O_OBJ[115]()
    action = o_tfr.Action_Semantics_internal
    label = '%s::%s' % (o_obj.Name, o_tfr.Name)
    run = interpret.run_operation

    if o_tfr.Instance_Based:
        return lambda self, **kwargs: run(metaclass, label, action, kwargs, self)
    else:
        fn = lambda cls, **kwargs: run(metaclass, label, action, kwargs, None)
        return classmethod(fn)

def mk_operation ( metaclass , o_tfr ) : o_obj = one ( o_tfr ) . O_OBJ [ 115 ] ( ) action = o_tfr . Action_Semantics_internal label = '%s::%s' % ( o_obj . Name , o_tfr . Name ) run = interpret . run_operation if o_tfr . Instance_Based : return lambda self , * * kwargs : run ( metaclass , label , action , kwargs , self ) else : fn = lambda cls , * * kwargs : run ( metaclass , label , action , kwargs , None ) return classmethod ( fn )

Create a python function that interprets that action of a BridgePoint class operation .


def mk_derived_attribute(metaclass, o_dbattr):
    '''
    Create a python property that interprets that action of a BridgePoint derived
    attribute.
    '''
    o_attr = one(o_dbattr).O_BATTR[107].O_ATTR[106]()
    o_obj = one(o_attr).O_OBJ[102]()
    action = o_dbattr.Action_Semantics_internal
    label = '%s::%s' % (o_obj.Name, o_attr.Name)
    fget = functools.partial(interpret.run_derived_attribute, metaclass,
                             label, action, o_attr.Name)
    return property(fget)

def mk_derived_attribute ( metaclass , o_dbattr ) : o_attr = one ( o_dbattr ) . O_BATTR [ 107 ] . O_ATTR [ 106 ] ( ) o_obj = one ( o_attr ) . O_OBJ [ 102 ] ( ) action = o_dbattr . Action_Semantics_internal label = '%s::%s' % ( o_obj . Name , o_attr . Name ) fget = functools . partial ( interpret . run_derived_attribute , metaclass , label , action , o_attr . Name ) return property ( fget )

Create a python property that interprets that action of a BridgePoint derived attribute .


def mk_class(m, o_obj, derived_attributes=False):
    '''
    Create a pyxtuml class from a BridgePoint class.
    '''
    first_filter = lambda selected: not one(selected).O_ATTR[103, 'succeeds']()
    o_attr = one(o_obj).O_ATTR[102](first_filter)
    attributes = list()

    while o_attr:
        s_dt = get_attribute_type(o_attr)
        ty = _get_data_type_name(s_dt)
        if not derived_attributes and one(o_attr).O_BATTR[106].O_DBATTR[107]():
            pass
#            logger.warning('Omitting derived attribute %s.%s ' %
#                           (o_obj.Key_Lett, o_attr.Name))
        elif not ty:
            logger.warning('Omitting unsupported attribute %s.%s ' %
                           (o_obj.Key_Lett, o_attr.Name))
        else:
            attributes.append((o_attr.Name, ty))

        o_attr = one(o_attr).O_ATTR[103, 'precedes']()

    metaclass = m.define_class(o_obj.Key_Lett, list(attributes), o_obj.Descrip)

    for o_id in many(o_obj).O_ID[104]():
        o_oida = many(o_id).O_OIDA[105]()
        o_attrs = many(o_oida).O_ATTR[105]()
        if not derived_attributes and one(o_attrs).O_BATTR[106].O_DBATTR[107]():
            logger.warning('Omitting unique identifier %s.I%d' %
                           (o_obj.Key_Lett, o_id.Oid_ID + 1))
            continue

        names = [o_attr.Name for o_attr in o_attrs]
        m.define_unique_identifier(o_obj.Key_Lett, o_id.Oid_ID + 1, *names)

    for o_tfr in many(o_obj).O_TFR[115]():
        fn = mk_operation(metaclass, o_tfr)
        setattr(metaclass.clazz, o_tfr.Name, fn)

    for o_dbattr in many(o_obj).O_ATTR[102].O_BATTR[106].O_DBATTR[107]():
        o_attr = one(o_dbattr).O_BATTR[107].O_ATTR[106]()
        fn = mk_derived_attribute(metaclass, o_dbattr)
        setattr(metaclass.clazz, o_attr.Name, fn)

    return metaclass

#                           (o_obj.Key_Lett, o_attr.Name))

Create a pyxtuml class from a BridgePoint class .


def mk_simple_association(m, r_simp):
    '''
    Create a pyxtuml association from a simple association in BridgePoint.
    '''
    r_rel = one(r_simp).R_REL[206]()

    r_form = one(r_simp).R_FORM[208]()
    r_part = one(r_simp).R_PART[207]()

    r_rgo = one(r_form).R_RGO[205]()
    r_rto = one(r_part).R_RTO[204]()

    if not r_form:
        logger.info('unformalized association R%s' % (r_rel.Numb))
        r_form = one(r_simp).R_PART[207](lambda sel: sel != r_part)
        r_rgo = one(r_form).R_RTO[204]()

    source_o_obj = one(r_rgo).R_OIR[203].O_OBJ[201]()
    target_o_obj = one(r_rto).R_OIR[203].O_OBJ[201]()
    source_ids, target_ids = _get_related_attributes(r_rgo, r_rto)

    if source_o_obj.Obj_ID != target_o_obj.Obj_ID:
        source_phrase = target_phrase = ''
    else:
        source_phrase = r_part.Txt_Phrs
        target_phrase = r_form.Txt_Phrs

    m.define_association(rel_id=r_rel.Numb,
                         source_kind=source_o_obj.Key_Lett,
                         target_kind=target_o_obj.Key_Lett,
                         source_keys=source_ids,
                         target_keys=target_ids,
                         source_conditional=r_form.Cond,
                         target_conditional=r_part.Cond,
                         source_phrase=source_phrase,
                         target_phrase=target_phrase,
                         source_many=r_form.Mult,
                         target_many=r_part.Mult)

Create a pyxtuml association from a simple association in BridgePoint .


def mk_linked_association(m, r_assoc):
    '''
    Create pyxtuml associations from a linked association in BridgePoint.
    '''
    r_rel = one(r_assoc).R_REL[206]()
    r_rgo = one(r_assoc).R_ASSR[211].R_RGO[205]()
    source_o_obj = one(r_rgo).R_OIR[203].O_OBJ[201]()

    def _mk_assoc(side1, side2):
        r_rto = one(side1).R_RTO[204]()

        target_o_obj = one(r_rto).R_OIR[203].O_OBJ[201]()
        source_ids, target_ids = _get_related_attributes(r_rgo, r_rto)
        if side1.Obj_ID != side2.Obj_ID:
            source_phrase = target_phrase = ''
        else:
            source_phrase = side1.Txt_Phrs
            target_phrase = side2.Txt_Phrs

        m.define_association(rel_id=r_rel.Numb,
                             source_kind=source_o_obj.Key_Lett,
                             target_kind=target_o_obj.Key_Lett,
                             source_keys=source_ids,
                             target_keys=target_ids,
                             source_conditional=side2.Cond,
                             target_conditional=False,
                             source_phrase=source_phrase,
                             target_phrase=target_phrase,
                             source_many=side2.Mult,
                             target_many=False)

    r_aone = one(r_assoc).R_AONE[209]()
    r_aoth = one(r_assoc).R_AOTH[210]()

    _mk_assoc(r_aone, r_aoth)
    _mk_assoc(r_aoth, r_aone)

Create pyxtuml associations from a linked association in BridgePoint .


def mk_association(m, r_rel):
    '''
    Create a pyxtuml association from a R_REL in ooaofooa.
    '''
    handler = {
        'R_SIMP': mk_simple_association,
        'R_ASSOC': mk_linked_association,
        'R_SUBSUP': mk_subsuper_association,
        'R_COMP': mk_derived_association,
    }
    inst = subtype(r_rel, 206)
    fn = handler.get(type(inst).__name__)
    return fn(m, inst)

def mk_association ( m , r_rel ) : handler = { 'R_SIMP' : mk_simple_association , 'R_ASSOC' : mk_linked_association , 'R_SUBSUP' : mk_subsuper_association , 'R_COMP' : mk_derived_association , } inst = subtype ( r_rel , 206 ) fn = handler . get ( type ( inst ) . __name__ ) return fn ( m , inst )

Create a pyxtuml association from a R_REL in ooaofooa .


def mk_component(bp_model, c_c=None, derived_attributes=False):
    '''
    Create a pyxtuml meta model from a BridgePoint model.
    Optionally, restrict to classes and associations contained in the
    component c_c.
    '''
    target = Domain()

    c_c_filt = lambda sel: c_c is None or is_contained_in(sel, c_c)

    for o_obj in bp_model.select_many('O_OBJ', c_c_filt):
        mk_class(target, o_obj, derived_attributes)

    for r_rel in bp_model.select_many('R_REL', c_c_filt):
        mk_association(target, r_rel)

    for s_sync in bp_model.select_many('S_SYNC', c_c_filt):
        fn = mk_function(target, s_sync)
        target.add_symbol(s_sync.Name, fn)

    for s_dt in bp_model.select_many('S_DT', c_c_filt):
        s_edt = one(s_dt).S_EDT[17]()
        if s_edt:
            enum = mk_enum(s_edt)
            target.add_symbol(s_dt.Name, enum)

    for cnst_csp in bp_model.select_many('CNST_CSP', c_c_filt):
        for cnst_syc in many(cnst_csp).CNST_SYC[1504]():
            value = mk_constant(cnst_syc)
            target.add_symbol(cnst_syc.Name, value)

    for ass in target.associations:
        ass.formalize()

    for s_ee in bp_model.select_many('S_EE', c_c_filt):
        if s_ee.Key_Lett in ['LOG', 'ARCH', 'TIM', 'NVS', 'PERSIST']:
            target.add_symbol(s_ee.Key_Lett, getattr(builtin_ee, s_ee.Key_Lett))

        else:
            ee = mk_external_entity(target, s_ee)
            target.add_symbol(s_ee.Key_Lett, ee)

    return target

def mk_component ( bp_model , c_c = None , derived_attributes = False ) : target = Domain ( ) c_c_filt = lambda sel : c_c is None or is_contained_in ( sel , c_c ) for o_obj in bp_model . select_many ( 'O_OBJ' , c_c_filt ) : mk_class ( target , o_obj , derived_attributes ) for r_rel in bp_model . select_many ( 'R_REL' , c_c_filt ) : mk_association ( target , r_rel ) for s_sync in bp_model . select_many ( 'S_SYNC' , c_c_filt ) : fn = mk_function ( target , s_sync ) target . add_symbol ( s_sync . Name , fn ) for s_dt in bp_model . select_many ( 'S_DT' , c_c_filt ) : s_edt = one ( s_dt ) . S_EDT [ 17 ] ( ) if s_edt : enum = mk_enum ( s_edt ) target . add_symbol ( s_dt . Name , enum ) for cnst_csp in bp_model . select_many ( 'CNST_CSP' , c_c_filt ) : for cnst_syc in many ( cnst_csp ) . CNST_SYC [ 1504 ] ( ) : value = mk_constant ( cnst_syc ) target . add_symbol ( cnst_syc . Name , value ) for ass in target . associations : ass . formalize ( ) for s_ee in bp_model . select_many ( 'S_EE' , c_c_filt ) : if s_ee . Key_Lett in [ 'LOG' , 'ARCH' , 'TIM' , 'NVS' , 'PERSIST' ] : target . add_symbol ( s_ee . Key_Lett , getattr ( builtin_ee , s_ee . Key_Lett ) ) else : ee = mk_external_entity ( target , s_ee ) target . add_symbol ( s_ee . Key_Lett , ee ) return target

Create a pyxtuml meta model from a BridgePoint model . Optionally restrict to classes and associations contained in the component c_c .


def work(self, socket, call, args, kwargs, topics=()):
        """Calls a function and send results to the collector.  It supports
        all of function actions.  A function could return, yield, raise any
        packable objects.
        """
        task_id = uuid4_bytes()
        reply_socket, topics = self.replier(socket, topics, call.reply_to)
        if reply_socket:
            channel = (call.call_id, task_id, topics)
        else:
            channel = (None, None, None)
        f, rpc_spec = self.find_call_target(call)
        if rpc_spec.reject_if.__get__(self.app)(call, topics):
            reply_socket and self.reject(reply_socket, call.call_id, topics)
            return
        reply_socket and self.accept(reply_socket, channel)
        success = False
        with self.catch_exceptions():
            try:
                val = self.call(call, args, kwargs, f, rpc_spec)
            except:
                exc_info = sys.exc_info()
                self.raise_(reply_socket, channel, exc_info)
                reraise(*exc_info)
            success = True
        if not success:
            # catch_exceptions() hides exceptions.
            return
        if isinstance(val, Iterator):
            vals = val
            with self.catch_exceptions():
                try:
                    try:
                        val = next(vals)
                    except StopIteration:
                        pass
                    else:
                        self.send_reply(reply_socket, YIELD, val, *channel)
                        for val in vals:
                            self.send_reply(reply_socket, YIELD, val, *channel)
                    self.send_reply(reply_socket, BREAK, None, *channel)
                except:
                    exc_info = sys.exc_info()
                    self.raise_(reply_socket, channel, exc_info)
                    reraise(*exc_info)
        else:
            self.send_reply(reply_socket, RETURN, val, *channel)

def
work
(
self
,
socket
,
call
,
args
,
kwargs
,
topics
=
(
)
)
:
task_id
=
uuid4_bytes
(
)
reply_socket
,
topics
=
self
.
replier
(
socket
,
topics
,
call
.
reply_to
)
if
reply_socket
:
channel
=
(
call
.
call_id
,
task_id
,
topics
)
else
:
channel
=
(
,
,
)
f
,
rpc_spec
=
self
.
find_call_target
(
call
)
if
rpc_spec
.
reject_if
.
__get__
(
self
.
app
)
(
call
,
topics
)
:
reply_socket
and
self
.
reject
(
reply_socket
,
call
.
call_id
,
topics
)
return
reply_socket
and
self
.
accept
(
reply_socket
,
channel
)
success
=
False
with
self
.
catch_exceptions
(
)
:
try
:
val
=
self
.
call
(
call
,
args
,
kwargs
,
f
,
rpc_spec
)
except
:
exc_info
=
sys
.
exc_info
(
)
self
.
raise_
(
reply_socket
,
channel
,
exc_info
)
reraise
(
*
exc_info
)
success
=
True
if
not
success
:
# catch_exceptions() hides exceptions.
return
if
isinstance
(
val
,
Iterator
)
:
vals
=
val
with
self
.
catch_exceptions
(
)
:
try
:
try
:
val
=
next
(
vals
)
except
StopIteration
:
pass
else
:
self
.
send_reply
(
reply_socket
,
YIELD
,
val
,
*
channel
)
for
val
in
vals
:
self
.
send_reply
(
reply_socket
,
YIELD
,
val
,
*
channel
)
self
.
send_reply
(
reply_socket
,
BREAK
,
,
*
channel
)
except
:
exc_info
=
sys
.
exc_info
(
)
self
.
raise_
(
reply_socket
,
channel
,
exc_info
)
reraise
(
*
exc_info
)
else
:
self
.
send_reply
(
reply_socket
,
RETURN
,
val
,
*
channel
)

Calls a function and send results to the collector . It supports all of function actions . A function could return yield raise any packable objects .


def accept(self, reply_socket, channel):
        """Sends ACCEPT reply."""
        info = self.info or b''
        self.send_raw(reply_socket, ACCEPT, info, *channel)

def accept ( self , reply_socket , channel ) : info = self . info or b'' self . send_raw ( reply_socket , ACCEPT , info , * channel )


def reject(self, reply_socket, call_id, topics=()):
        """Sends REJECT reply."""
        info = self.info or b''
        self.send_raw(reply_socket, REJECT, info, call_id, b'', topics)

def reject ( self , reply_socket , call_id , topics = ( ) ) : info = self . info or b'' self . send_raw ( reply_socket , REJECT , info , call_id , b'' , topics )


def raise_(self, reply_socket, channel, exc_info=None):
        """Sends RAISE reply."""
        if not reply_socket:
            return
        if exc_info is None:
            exc_info = sys.exc_info()
        exc_type, exc, tb = exc_info
        while tb.tb_next is not None:
            tb = tb.tb_next
        if issubclass(exc_type, RemoteException):
            exc_type = exc_type.exc_type
        filename, lineno = tb.tb_frame.f_code.co_filename, tb.tb_lineno
        val = (exc_type, str(exc), filename, lineno)
        try:
            state = exc.__getstate__()
        except AttributeError:
            pass
        else:
            val += (state,)
        self.send_reply(reply_socket, RAISE, val, *channel)

def raise_ ( self , reply_socket , channel , exc_info = None ) : if not reply_socket : return if exc_info is None : exc_info = sys . exc_info ( ) exc_type , exc , tb = exc_info while tb . tb_next is not None : tb = tb . tb_next if issubclass ( exc_type , RemoteException ) : exc_type = exc_type . exc_type filename , lineno = tb . tb_frame . f_code . co_filename , tb . tb_lineno val = ( exc_type , str ( exc ) , filename , lineno ) try : state = exc . __getstate__ ( ) except AttributeError : pass else : val += ( state , ) self . send_reply ( reply_socket , RAISE , val , * channel )


def _call_wait(self, hints, name, args, kwargs, topics=(), raw=False,
                   limit=None, retry=False, max_retries=None):
        """Allocates a call id and emit."""
        col = self.collector
        if not col.is_running():
            col.start()
        call_id = uuid4_bytes()
        reply_to = (DUPLEX if self.socket is col.socket else col.topic)
        # Normal tuple is faster than namedtuple.
        header = self._make_header(name, call_id, reply_to, hints)
        payload = self._pack(args, kwargs, raw)
        # Use short names.
        def send_call():
            try:
                safe(send, self.socket, header, payload, topics, zmq.NOBLOCK)
            except zmq.Again:
                raise Undelivered('emission was not delivered')
        col.prepare(call_id, self, name, args, kwargs)
        send_call()
        return col.establish(call_id, self.timeout, limit,
                             send_call if retry else None,
                             max_retries=max_retries)


def establish(self, call_id, timeout, limit=None,
                  retry=None, max_retries=None):
        """Waits for the call is accepted by workers and starts to collect the
        results.
        """
        rejected = 0
        retried = 0
        results = []
        result_queue = self.result_queues[call_id]
        try:
            with Timeout(timeout, False):
                while True:
                    result = result_queue.get()
                    if result is None:
                        rejected += 1
                        if retry is not None:
                            if retried == max_retries:
                                break
                            retry()
                            retried += 1
                        continue
                    results.append(result)
                    if len(results) == limit:
                        break
        finally:
            del result_queue
            self.remove_result_queue(call_id)
        if not results:
            if rejected:
                raise Rejected('%d workers rejected' % rejected
                               if rejected != 1 else
                               'A worker rejected')
            else:
                raise WorkerNotFound('failed to find worker')
        return results

def
establish
(
self
,
call_id
,
timeout
,
limit
=
,
retry
=
,
max_retries
=
)
:
rejected
=
0
retried
=
0
results
=
[
]
result_queue
=
self
.
result_queues
[
call_id
]
try
:
with
Timeout
(
timeout
,
False
)
:
while
True
:
result
=
result_queue
.
get
(
)
if
result
is
:
rejected
+=
1
if
retry
is
not
:
if
retried
==
max_retries
:
break
retry
(
)
retried
+=
1
continue
results
.
append
(
result
)
if
len
(
results
)
==
limit
:
break
finally
:
del
result_queue
self
.
remove_result_queue
(
call_id
)
if
not
results
:
if
rejected
:
raise
Rejected
(
'%d workers rejected'
%
rejected
if
rejected
!=
1
else
'A worker rejected'
)
else
:
raise
WorkerNotFound
(
'failed to find worker'
)
return
results

Waits for the call is accepted by workers and starts to collect the results .


def dispatch_reply(self, reply, value):
        """Dispatches the reply to the proper queue."""
        method = reply.method
        call_id = reply.call_id
        task_id = reply.task_id
        if method & ACK:
            try:
                result_queue = self.result_queues[call_id]
            except KeyError:
                raise KeyError('already established or unprepared call')
            if method == ACCEPT:
                worker_info = value
                result = RemoteResult(self, call_id, task_id, worker_info)
                self.results[call_id][task_id] = result
                result_queue.put_nowait(result)
            elif method == REJECT:
                result_queue.put_nowait(None)
        else:
            result = self.results[call_id][task_id]
            result.set_reply(reply.method, value)

Dispatches the reply to the proper queue .


def guess_type_name(value):
    '''
    Guess the type name of a serialized value.
    '''
    value = str(value)

    if value.upper() in ['TRUE', 'FALSE']:
        return 'BOOLEAN'

    elif re.match(r'(-)?(\d+)(\.\d+)', value):
        return 'REAL'

    elif re.match(r'(-)?(\d+)', value):
        return 'INTEGER'

    elif re.match(r'\'((\'\')|[^\'])*\'', value):
        return 'STRING'

    elif re.match(r'\"([^\\\n]|(\\.))*?\"', value):
        return 'UNIQUE_ID'

def guess_type_name ( value ) : value = str ( value ) if value . upper ( ) in [ 'TRUE' , 'FALSE' ] : return 'BOOLEAN' elif re . match ( r'(-)?(\d+)(\.\d+)' , value ) : return 'REAL' elif re . match ( r'(-)?(\d+)' , value ) : return 'INTEGER' elif re . match ( r'\'((\'\')|[^\'])*\'' , value ) : return 'STRING' elif re . match ( r'\"([^\\\n]|(\\.))*?\"' , value ) : return 'UNIQUE_ID'

Guess the type name of a serialized value .


def deserialize_value(ty, value):
    '''
    Deserialize a value of some type
    '''
    uty = ty.upper()

    if uty == 'BOOLEAN':
        if value.isdigit():
            return bool(int(value))
        elif value.upper() == 'FALSE':
            return False
        elif value.upper() == 'TRUE':
            return True
        else:
            return None

    elif uty == 'INTEGER':
        if '"' in value:
            return uuid.UUID(value[1:-1]).int
        else:
            return int(value)

    elif uty == 'REAL':
        return float(value)

    elif uty == 'STRING':
        return value[1:-1].replace("''", "'")

    elif uty == 'UNIQUE_ID':
        if '"' in value:
            return uuid.UUID(value[1:-1]).int
        else:
            return int(value)

def deserialize_value ( ty , value ) : uty = ty . upper ( ) if uty == 'BOOLEAN' : if value . isdigit ( ) : return bool ( int ( value ) ) elif value . upper ( ) == 'FALSE' : return False elif value . upper ( ) == 'TRUE' : return True else : return None elif uty == 'INTEGER' : if '"' in value : return uuid . UUID ( value [ 1 : - 1 ] ) . int else : return int ( value ) elif uty == 'REAL' : return float ( value ) elif uty == 'STRING' : return value [ 1 : - 1 ] . replace ( "''" , "'" ) elif uty == 'UNIQUE_ID' : if '"' in value : return uuid . UUID ( value [ 1 : - 1 ] ) . int else : return int ( value )


def t_LPAREN(self, t):
        r'\('
        t.endlexpos = t.lexpos + len(t.value)
        return t

def t_LPAREN ( self , t ) : t . endlexpos = t . lexpos + len ( t . value ) return t


def t_RPAREN(self, t):
        r'\)'
        t.endlexpos = t.lexpos + len(t.value)
        return t

def t_RPAREN ( self , t ) : t . endlexpos = t . lexpos + len ( t . value ) return t


def get(self, content_id, feature_names=None):
        '''Retrieve a feature collection.

        If a feature collection with the given id does not
        exist, then ``None`` is returned.

        :param str content_id: Content identifier.
        :param [str] feature_names:
          A list of feature names to retrieve. When ``None``, all
          features are retrieved. Wildcards are allowed.
        :rtype: :class:`dossier.fc.FeatureCollection` or ``None``
        '''
        try:
            resp = self.conn.get(index=self.index, doc_type=self.type,
                                 id=eid(content_id),
                                 _source=self._source(feature_names))
            return self.fc_from_dict(resp['_source']['fc'])
        except NotFoundError:
            return None
        except:
            raise

def get ( self , content_id , feature_names = None ) : try : resp = self . conn . get ( index = self . index , doc_type = self . type , id = eid ( content_id ) , _source = self . _source ( feature_names ) ) return self . fc_from_dict ( resp [ '_source' ] [ 'fc' ] ) except NotFoundError : return None except : raise


def get_many(self, content_ids, feature_names=None):
        '''Returns an iterable of feature collections.

        This efficiently retrieves multiple FCs corresponding to the
        list of ids given. Tuples of identifier and feature collection
        are yielded. If the feature collection for a given id does not
        exist, then ``None`` is returned as the second element of the
        tuple.

        :param [str] content_ids: List of content ids.
        :param [str] feature_names:
          A list of feature names to retrieve. When ``None``, all
          features are retrieved. Wildcards are allowed.
        :rtype: Iterable of ``(content_id, FC)``
        '''
        try:
            resp = self.conn.mget(index=self.index, doc_type=self.type,
                                  _source=self._source(feature_names),
                                  body={'ids': map(eid, content_ids)})
        except TransportError:
            return
        for doc in resp['docs']:
            fc = None
            if doc['found']:
                fc = self.fc_from_dict(doc['_source']['fc'])
            yield did(doc['_id']), fc

def get_many ( self , content_ids , feature_names = None ) : try : resp = self . conn . mget ( index = self . index , doc_type = self . type , _source = self . _source ( feature_names ) , body = { 'ids' : map ( eid , content_ids ) } ) except TransportError : return for doc in resp [ 'docs' ] : fc = None if doc [ 'found' ] : fc = self . fc_from_dict ( doc [ '_source' ] [ 'fc' ] ) yield did ( doc [ '_id' ] ) , fc

Returns an iterable of feature collections .


def put(self, items, indexes=True):
        '''Adds feature collections to the store.

        This efficiently adds multiple FCs to the store. The iterable
        of ``items`` given should yield tuples of ``(content_id, FC)``.

        :param items: Iterable of ``(content_id, FC)``.
        :param [str] feature_names:
          A list of feature names to retrieve. When ``None``, all
          features are retrieved. Wildcards are allowed.
        '''
        actions = []
        for cid, fc in items:
            # TODO: If we store features in a columnar order, then we
            # could tell ES to index the feature values directly. ---AG
            # (But is problematic because we want to preserve the ability
            # to selectively index FCs. So we'd probably need two distinct
            # doc types.)
            idxs = defaultdict(list)
            if indexes:
                for fname in self.indexed_features:
                    if fname in fc:
                        idxs[fname_to_idx_name(fname)].extend(fc[fname])
                for fname in self.fulltext_indexed_features:
                    if fname not in fc:
                        continue
                    if isinstance(fc[fname], basestring):
                        idxs[fname_to_full_idx_name(fname)] = fc[fname]
                    else:
                        idxs[fname_to_full_idx_name(fname)].extend(fc[fname])
            actions.append({
                '_index': self.index,
                '_type': self.type,
                '_id': eid(cid),
                '_op_type': 'index',
                '_source': dict(idxs, **{
                    'fc': self.fc_to_dict(fc),
                }),
            })
        bulk(self.conn, actions, timeout=60, request_timeout=60)


def delete(self, content_id):
        '''Deletes the corresponding feature collection.

        If the FC does not exist, then this is a no-op.
        '''
        try:
            self.conn.delete(index=self.index, doc_type=self.type,
                             id=eid(content_id))
        except NotFoundError:
            pass

def delete ( self , content_id ) : try : self . conn . delete ( index = self . index , doc_type = self . type , id = eid ( content_id ) ) except NotFoundError : pass

Deletes the corresponding feature collection .


def delete_all(self):
        '''Deletes all feature collections.

        This does not destroy the ES index, but instead only
        deletes all FCs with the configured document type
        (defaults to ``fc``).
        '''
        try:
            self.conn.indices.delete_mapping(
                index=self.index, doc_type=self.type)
        except TransportError:
            logger.warn('type %r in index %r already deleted',
                        self.index, self.type, exc_info=True)

def
delete_all
(
self
)
:
try
:
self
.
conn
.
indices
.
delete_mapping
(
index
=
self
.
index
,
doc_type
=
self
.
type
)
except
TransportError
:
logger
.
warn
(
'type %r in index %r already deleted'
,
self
.
index
,
self
.
type
,
exc_info
=
True
)


def delete_index(self):
        '''Deletes the underlying ES index.

        Only use this if you know what you're doing. This destroys
        the entire underlying ES index, which could be shared by
        multiple distinct ElasticStore instances.
        '''
        if self.conn.indices.exists(index=self.index):
            self.conn.indices.delete(index=self.index)

def delete_index ( self ) : if self . conn . indices . exists ( index = self . index ) : self . conn . indices . delete ( index = self . index )


def scan(self, *key_ranges, **kwargs):
        '''Scan for FCs in the given id ranges.

        :param key_ranges:
          ``key_ranges`` should be a list of pairs of ranges. The first
          value is the lower bound id and the second value is the
          upper bound id. Use ``()`` in either position to leave it
          unbounded. If no ``key_ranges`` are given, then all FCs in
          the store are returned.
        :param [str] feature_names:
          A list of feature names to retrieve. When ``None``, all
          features are retrieved. Wildcards are allowed.
        :rtype: Iterable of ``(content_id, FC)``
        '''
        for hit in self._scan(*key_ranges, **kwargs):
            yield did(hit['_id']), self.fc_from_dict(hit['_source']['fc'])

def scan ( self , * key_ranges , * * kwargs ) : for hit in self . _scan ( * key_ranges , * * kwargs ) : yield did ( hit [ '_id' ] ) , self . fc_from_dict ( hit [ '_source' ] [ 'fc' ] )


def scan_ids(self, *key_ranges, **kwargs):
        '''Scan for ids only in the given id ranges.

        :param key_ranges:
          ``key_ranges`` should be a list of pairs of ranges. The first
          value is the lower bound id and the second value is the
          upper bound id. Use ``()`` in either position to leave it
          unbounded. If no ``key_ranges`` are given, then all FCs in
          the store are returned.
        :param [str] feature_names:
          A list of feature names to retrieve. When ``None``, all
          features are retrieved. Wildcards are allowed.
        :rtype: Iterable of ``content_id``
        '''
        kwargs['feature_names'] = False
        for hit in self._scan(*key_ranges, **kwargs):
            yield did(hit['_id'])

def scan_ids ( self , * key_ranges , * * kwargs ) : kwargs [ 'feature_names' ] = False for hit in self . _scan ( * key_ranges , * * kwargs ) : yield did ( hit [ '_id' ] )

Scan for ids only in the given id ranges .


def scan_prefix(self, prefix, feature_names=None):
        '''Scan for FCs with a given prefix.

        :param str prefix: Identifier prefix.
        :param [str] feature_names:
          A list of feature names to retrieve. When ``None``, all
          features are retrieved. Wildcards are allowed.
        :rtype: Iterable of ``(content_id, FC)``
        '''
        resp = self._scan_prefix(prefix, feature_names=feature_names)
        for hit in resp:
            yield did(hit['_id']), self.fc_from_dict(hit['_source']['fc'])

def scan_prefix ( self , prefix , feature_names = None ) : resp = self . _scan_prefix ( prefix , feature_names = feature_names ) for hit in resp : yield did ( hit [ '_id' ] ) , self . fc_from_dict ( hit [ '_source' ] [ 'fc' ] )


def scan_prefix_ids(self, prefix):
        '''Scan for ids with a given prefix.

        :param str prefix: Identifier prefix.
        :param [str] feature_names:
          A list of feature names to retrieve. When ``None``, all
          features are retrieved. Wildcards are allowed.
        :rtype: Iterable of ``content_id``
        '''
        resp = self._scan_prefix(prefix, feature_names=False)
        for hit in resp:
            yield did(hit['_id'])

def scan_prefix_ids ( self , prefix ) : resp = self . _scan_prefix ( prefix , feature_names = False ) for hit in resp : yield did ( hit [ '_id' ] )


def fulltext_scan(self, query_id=None, query_fc=None, feature_names=None,
                      preserve_order=True, indexes=None):
        '''Fulltext search.

        Yields an iterable of triples (score, identifier, FC)
        corresponding to the search results of the fulltext search
        in ``query``. This will only search text indexed under the
        given feature named ``fname``.

        Note that, unless ``preserve_order`` is set to True, the
        ``score`` will always be 0.0, and the results will be
        unordered. ``preserve_order`` set to True will cause the
        results to be scored and be ordered by score, but you should
        expect to see a decrease in performance.

        :param str fname:
          The feature to search.
        :param unicode query:
          The query.
        :param [str] feature_names:
          A list of feature names to retrieve. When ``None``, all
          features are retrieved. Wildcards are allowed.
        :rtype: Iterable of ``(score, content_id, FC)``
        '''
        it = self._fulltext_scan(query_id, query_fc,
                                 feature_names=feature_names,
                                 preserve_order=preserve_order,
                                 indexes=indexes)
        for hit in it:
            fc = self.fc_from_dict(hit['_source']['fc'])
            yield hit['_score'], did(hit['_id']), fc

def fulltext_scan ( self , query_id = None , query_fc = None , feature_names = None , preserve_order = True , indexes = None ) : it = self . _fulltext_scan ( query_id , query_fc , feature_names = feature_names , preserve_order = preserve_order , indexes = indexes ) for hit in it : fc = self . fc_from_dict ( hit [ '_source' ] [ 'fc' ] ) yield hit [ '_score' ] , did ( hit [ '_id' ] ) , fc


def fulltext_scan_ids(self, query_id=None, query_fc=None,
                          preserve_order=True, indexes=None):
        '''Fulltext search for identifiers.

        Yields an iterable of triples (score, identifier)
        corresponding to the search results of the fulltext search
        in ``query``. This will only search text indexed under the
        given feature named ``fname``.

        Note that, unless ``preserve_order`` is set to True, the
        ``score`` will always be 0.0, and the results will be
        unordered. ``preserve_order`` set to True will cause the
        results to be scored and be ordered by score, but you should
        expect to see a decrease in performance.

        :param str fname:
          The feature to search.
        :param unicode query:
          The query.
        :rtype: Iterable of ``(score, content_id)``
        '''
        it = self._fulltext_scan(query_id, query_fc, feature_names=False,
                                 preserve_order=preserve_order,
                                 indexes=indexes)
        for hit in it:
            yield hit['_score'], did(hit['_id'])

def fulltext_scan_ids ( self , query_id = None , query_fc = None , preserve_order = True , indexes = None ) : it = self . _fulltext_scan ( query_id , query_fc , feature_names = False , preserve_order = preserve_order , indexes = indexes ) for hit in it : yield hit [ '_score' ] , did ( hit [ '_id' ] )


def keyword_scan(self, query_id=None, query_fc=None, feature_names=None):
        '''Keyword scan for feature collections.

        This performs a keyword scan using the query given. A keyword
        scan searches for FCs with terms in each of the query's indexed
        fields.

        At least one of ``query_id`` or ``query_fc`` must be provided.
        If ``query_fc`` is ``None``, then the query is retrieved
        automatically corresponding to ``query_id``.

        :param str query_id: Optional query id.
        :param query_fc: Optional query feature collection.
        :type query_fc: :class:`dossier.fc.FeatureCollection`
        :param [str] feature_names:
          A list of feature names to retrieve. When ``None``, all
          features are retrieved. Wildcards are allowed.
        :rtype: Iterable of ``(content_id, FC)``
        '''
        it = self._keyword_scan(query_id, query_fc,
                                feature_names=feature_names)
        for hit in it:
            fc = self.fc_from_dict(hit['_source']['fc'])
            yield did(hit['_id']), fc

def keyword_scan ( self , query_id = None , query_fc = None , feature_names = None ) : it = self . _keyword_scan ( query_id , query_fc , feature_names = feature_names ) for hit in it : fc = self . fc_from_dict ( hit [ '_source' ] [ 'fc' ] ) yield did ( hit [ '_id' ] ) , fc


def keyword_scan_ids(self, query_id=None, query_fc=None):
        '''Keyword scan for ids.

        This performs a keyword scan using the query given. A keyword
        scan searches for FCs with terms in each of the query's indexed
        fields.

        At least one of ``query_id`` or ``query_fc`` must be provided.
        If ``query_fc`` is ``None``, then the query is retrieved
        automatically corresponding to ``query_id``.

        :param str query_id: Optional query id.
        :param query_fc: Optional query feature collection.
        :type query_fc: :class:`dossier.fc.FeatureCollection`
        :rtype: Iterable of ``content_id``
        '''
        it = self._keyword_scan(query_id, query_fc, feature_names=False)
        for hit in it:
            yield did(hit['_id'])

def keyword_scan_ids ( self , query_id = None , query_fc = None ) : it = self . _keyword_scan ( query_id , query_fc , feature_names = False ) for hit in it : yield did ( hit [ '_id' ] )


def index_scan_ids(self, fname, val):
        '''Low-level keyword index scan for ids.

        Retrieves identifiers of FCs that have a feature value
        ``val`` in the feature named ``fname``. Note that
        ``fname`` must be indexed.

        :param str fname: Feature name.
        :param str val: Feature value.
        :rtype: Iterable of ``content_id``
        '''
        disj = []
        for fname2 in self.indexes[fname]['feature_names']:
            disj.append({'term': {fname_to_idx_name(fname2): val}})
        query = {
            'constant_score': {
                'filter': {'or': disj},
            },
        }
        hits = scan(self.conn, index=self.index, doc_type=self.type, query={
            '_source': False,
            'query': query,
        })
        for hit in hits:
            yield did(hit['_id'])

def index_scan_ids ( self , fname , val ) : disj = [ ] for fname2 in self . indexes [ fname ] [ 'feature_names' ] : disj . append ( { 'term' : { fname_to_idx_name ( fname2 ) : val } } ) query = { 'constant_score' : { 'filter' : { 'or' : disj } , } , } hits = scan ( self . conn , index = self . index , doc_type = self . type , query = { '_source' : False , 'query' : query , } ) for hit in hits : yield did ( hit [ '_id' ] )

Low - level keyword index scan for ids .


def _source(self, feature_names):
        '''Maps feature names to ES's "_source" field.'''
        if feature_names is None:
            return True
        elif isinstance(feature_names, bool):
            return feature_names
        else:
            return map(lambda n: 'fc.' + n, feature_names)

def _source ( self , feature_names ) : if feature_names is None : return True elif isinstance ( feature_names , bool ) : return feature_names else : return map ( lambda n : 'fc.' + n , feature_names )

Maps feature names to ES s _source field .


def _range_filters(self, *key_ranges):
        'Creates ES filters for key ranges used in scanning.'
        filters = []
        for s, e in key_ranges:
            if isinstance(s, basestring):
                s = eid(s)
            if isinstance(e, basestring):
                # Make the range inclusive.
                # We need a valid codepoint, so use the max.
                e += u'\U0010FFFF'
                e = eid(e)

            if s == () and e == ():
                filters.append({'match_all': {}})
            elif e == ():
                filters.append({'range': {'_id': {'gte': s}}})
            elif s == ():
                filters.append({'range': {'_id': {'lte': e}}})
            else:
                filters.append({'range': {'_id': {'gte': s, 'lte': e}}})
        if len(filters) == 0:
            return [{'match_all': {}}]
        else:
            return filters

Creates ES filters for key ranges used in scanning .


def _create_index(self):
        'Create the index'
        try:
            self.conn.indices.create(
                index=self.index, timeout=60, request_timeout=60, body={
                    'settings': {
                        'number_of_shards': self.shards,
                        'number_of_replicas': self.replicas,
                    },
                })
        except TransportError:
            # Hope that this is an "index already exists" error...
            logger.warn('index already exists? OK', exc_info=True)
            pass


def _create_mappings(self):
        'Create the field type mapping.'
        self.conn.indices.put_mapping(
            index=self.index, doc_type=self.type,
            timeout=60, request_timeout=60,
            body={
                self.type: {
                    'dynamic_templates': [{
                        'default_no_analyze_fc': {
                            'match': 'fc.*',
                            'mapping': {'index': 'no'},
                        },
                    }],
                    '_all': {
                        'enabled': False,
                    },
                    '_id': {
                        'index': 'not_analyzed',  # allows range queries
                    },
                    'properties': self._get_index_mappings(),
                },
            })
        # It is possible to create an index and quickly launch a request
        # that will fail because the index hasn't been set up yet. Usually,
        # you'll get a "no active shards available" error.
        #
        # Since index creation is a very rare operation (it only happens
        # when the index doesn't already exist), we sit and wait for the
        # cluster to become healthy.
        self.conn.cluster.health(index=self.index, wait_for_status='yellow')


def _get_index_mappings(self):
        'Retrieve the field mappings. Useful for debugging.'
        maps = {}
        for fname in self.indexed_features:
            config = self.indexes.get(fname, {})
            print(fname, config)
            maps[fname_to_idx_name(fname)] = {
                'type': config.get('es_index_type', 'integer'),
                'store': False,
                'index': 'not_analyzed',
            }
        for fname in self.fulltext_indexed_features:
            maps[fname_to_full_idx_name(fname)] = {
                'type': 'string',
                'store': False,
                'index': 'analyzed',
            }
        return maps

def _get_index_mappings ( self ) : maps = { } for fname in self . indexed_features : config = self . indexes . get ( fname , { } ) print ( fname , config ) maps [ fname_to_idx_name ( fname ) ] = { 'type' : config . get ( 'es_index_type' , 'integer' ) , 'store' : False , 'index' : 'not_analyzed' , } for fname in self . fulltext_indexed_features : maps [ fname_to_full_idx_name ( fname ) ] = { 'type' : 'string' , 'store' : False , 'index' : 'analyzed' , } return maps

Retrieve the field mappings . Useful for debugging .


def _get_field_types(self):
        'Retrieve the field types. Useful for debugging.'
        mapping = self.conn.indices.get_mapping(
            index=self.index, doc_type=self.type)
        return mapping[self.index]['mappings'][self.type]['properties']

def _get_field_types ( self ) : mapping = self . conn . indices . get_mapping ( index = self . index , doc_type = self . type ) return mapping [ self . index ] [ 'mappings' ] [ self . type ] [ 'properties' ]

Retrieve the field types . Useful for debugging .


def _fc_index_disjunction_from_query(self, query_fc, fname):
        'Creates a disjunction for keyword scan queries.'
        if len(query_fc.get(fname, [])) == 0:
            return []
        terms = query_fc[fname].keys()

        disj = []
        for fname in self.indexes[fname]['feature_names']:
            disj.append({'terms': {fname_to_idx_name(fname): terms}})
        return disj

def _fc_index_disjunction_from_query ( self , query_fc , fname ) : if len ( query_fc . get ( fname , [ ] ) ) == 0 : return [ ] terms = query_fc [ fname ] . keys ( ) disj = [ ] for fname in self . indexes [ fname ] [ 'feature_names' ] : disj . append ( { 'terms' : { fname_to_idx_name ( fname ) : terms } } ) return disj

Creates a disjunction for keyword scan queries .


def fc_bytes(self, fc_dict):
        '''Take a feature collection in dict form and count its size in bytes.

        '''
        num_bytes = 0
        for _, feat in fc_dict.iteritems():
            num_bytes += len(feat)
        return num_bytes

def fc_bytes ( self , fc_dict ) : num_bytes = 0 for _ , feat in fc_dict . iteritems ( ) : num_bytes += len ( feat ) return num_bytes

Take a feature collection in dict form and count its size in bytes .


def count_bytes(self, filter_preds):
        '''Count bytes of all feature collections whose key satisfies one of
        the predicates in ``filter_preds``. The byte counts are binned
        by filter predicate.
        '''
        num_bytes = defaultdict(int)
        for hit in self._scan():
            for filter_pred in filter_preds:
                if filter_pred(did(hit['_id'])):
                    num_bytes[filter_pred] += self.fc_bytes(
                        hit['_source']['fc'])
        return num_bytes

def count_bytes ( self , filter_preds ) : num_bytes = defaultdict ( int ) for hit in self . _scan ( ) : for filter_pred in filter_preds : if filter_pred ( did ( hit [ '_id' ] ) ) : num_bytes [ filter_pred ] += self . fc_bytes ( hit [ '_source' ] [ 'fc' ] ) return num_bytes

Count bytes of all feature collections whose key satisfies one of the predicates in filter_preds . The byte counts are binned by filter predicate .


def pretty_string(fc):
    '''construct a nice looking string for an FC
    '''
    s = []
    for fname, feature in sorted(fc.items()):
        if isinstance(feature, StringCounter):
            feature = [u'%s: %d' % (k, v)
                       for (k,v) in feature.most_common()]
            feature = u'\n\t' + u'\n\t'.join(feature)
        s.append(fname + u': ' + feature)
    return u'\n'.join(s)

def
pretty_string
(
fc
)
:
s
=
[
]
for
fname
,
feature
in
sorted
(
fc
.
items
(
)
)
:
if
isinstance
(
feature
,
StringCounter
)
:
feature
=
[
u'%s: %d'
%
(
k
,
v
)
for
(
k
,
v
)
in
feature
.
most_common
(
)
]
feature
=
u'\n\t'
+
u'\n\t'
.
join
(
feature
)
s
.
append
(
fname
+
u': '
+
feature
)
return
u'\n'
.
join
(
s
)


def process_docopts():  # type: ()->None
    """
    Take care of command line options
    """

    arguments = docopt(__doc__, version="Find Known Secrets {0}".format(__version__))

    logger.debug(arguments)
    # print(arguments)
    if arguments["here"]:
        # all default
        go()
    else:
        # user config
        files = arguments["--secrets"]

        searcher = Searcher(source=arguments["--source"], files=files)
        searcher.go()

def
process_docopts
(
)
:
# type: ()->None
arguments
=
docopt
(
__doc__
,
version
=
"Find Known Secrets {0}"
.
format
(
__version__
)
)
logger
.
debug
(
arguments
)
# print(arguments)
if
arguments
[
"here"
]
:
# all default
go
(
)
else
:
# user config
files
=
arguments
[
"--secrets"
]
searcher
=
Searcher
(
source
=
arguments
[
"--source"
]
,
files
=
files
)
searcher
.
go
(
)


def default_formatter(error):
    """Escape the error, and wrap it in a span with class ``error-message``"""
    quoted = formencode.htmlfill.escape_formatter(error)
    return u'{0}'.format(quoted)

Escape the error and wrap it in a span with class error - message


def pretty_to_link(inst, link):
    '''
    Create a human-readable representation of a link on the 'TO'-side
    '''
    values = ''
    prefix = ''
    metaclass = xtuml.get_metaclass(inst)

    for name, ty in metaclass.attributes:
        if name in link.key_map:
            value = getattr(inst, name)
            value = xtuml.serialize_value(value, ty)
            name = link.key_map[name]
            values += '%s%s=%s' % (prefix, name, value)
            prefix = ', '

    return '%s(%s)' % (link.kind, values)

def
pretty_to_link
(
inst
,
link
)
:
values
=
''
prefix
=
''
metaclass
=
xtuml
.
get_metaclass
(
inst
)
for
name
,
ty
in
metaclass
.
attributes
:
if
name
in
link
.
key_map
:
value
=
getattr
(
inst
,
name
)
value
=
xtuml
.
serialize_value
(
value
,
ty
)
name
=
link
.
key_map
[
name
]
values
+=
'%s%s=%s'
%
(
prefix
,
name
,
value
)
prefix
=
', '
return
'%s(%s)'
%
(
link
.
kind
,
values
)

Create a human - readable representation of a link on the TO - side


def pretty_unique_identifier(inst, identifier):
    '''
    Create a human-readable representation a unique identifier.
    '''
    values = ''
    prefix = ''
    metaclass = xtuml.get_metaclass(inst)

    for name, ty in metaclass.attributes:
        if name in metaclass.identifying_attributes:
            value = getattr(inst, name)
            value = xtuml.serialize_value(value, ty)
            values += '%s%s=%s' % (prefix, name, value)
            prefix = ', '

    return '%s(%s)' % (identifier, values)

def
pretty_unique_identifier
(
inst
,
identifier
)
:
values
=
''
prefix
=
''
metaclass
=
xtuml
.
get_metaclass
(
inst
)
for
name
,
ty
in
metaclass
.
attributes
:
if
name
in
metaclass
.
identifying_attributes
:
value
=
getattr
(
inst
,
name
)
value
=
xtuml
.
serialize_value
(
value
,
ty
)
values
+=
'%s%s=%s'
%
(
prefix
,
name
,
value
)
prefix
=
', '
return
'%s(%s)'
%
(
identifier
,
values
)

Create a human - readable representation a unique identifier .


def check_uniqueness_constraint(m, kind=None):
    '''
    Check the model for uniqueness constraint violations.
    '''
    if kind is None:
        metaclasses = m.metaclasses.values()
    else:
        metaclasses = [m.find_metaclass(kind)]

    res = 0
    for metaclass in metaclasses:
        id_map = dict()
        for identifier in metaclass.indices:
            id_map[identifier] = dict()

        for inst in metaclass.select_many():
            # Check for null-values
            for name, ty in metaclass.attributes:
                if name not in metaclass.identifying_attributes:
                    continue

                value = getattr(inst, name)
                isnull = value is None
                isnull |= (ty == 'UNIQUE_ID' and not value)
                if isnull:
                    res += 1
                    logger.warning('%s.%s is part of an identifier and is null'
                                   % (metaclass.kind, name))

            # Check uniqueness
            for identifier in metaclass.indices:
                kwargs = dict()
                for name in metaclass.indices[identifier]:
                    kwargs[name] = getattr(inst, name)

                index_key = frozenset(kwargs.items())
                if index_key in id_map[identifier]:
                    res += 1
                    id_string = pretty_unique_identifier(inst, identifier)
                    logger.warning('uniqueness constraint violation in %s, %s'
                                   % (metaclass.kind, id_string))

                id_map[identifier][index_key] = inst

    return res

Check the model for uniqueness constraint violations .


def check_link_integrity(m, link):
    '''
    Check the model for integrity violations on an association in a particular direction.
    '''
    res = 0
    for inst in link.from_metaclass.select_many():
        q_set = list(link.navigate(inst))

        if(len(q_set) < 1 and not link.conditional) or (
          (len(q_set) > 1 and not link.many)):
            res += 1
            logger.warning('integrity violation in '
                           '%s --(%s)--> %s' % (pretty_from_link(inst, link),
                                                link.rel_id,
                                                pretty_to_link(inst, link)))

    return res

def
check_link_integrity
(
m
,
link
)
:
res
=
0
for
inst
in
link
.
from_metaclass
.
select_many
(
)
:
q_set
=
list
(
link
.
navigate
(
inst
)
)
if
(
len
(
q_set
)
<
1
and
not
link
.
conditional
)
or
(
(
len
(
q_set
)
>
1
and
not
link
.
many
)
)
:
res
+=
1
logger
.
warning
(
'integrity violation in '
'%s --(%s)--> %s'
%
(
pretty_from_link
(
inst
,
link
)
,
link
.
rel_id
,
pretty_to_link
(
inst
,
link
)
)
)
return
res

Check the model for integrity violations on an association in a particular direction .


def check_subtype_integrity(m, super_kind, rel_id):
    '''
    Check the model for integrity violations across a subtype association.
    '''
    if isinstance(rel_id, int):
        rel_id = 'R%d' % rel_id

    res = 0
    for inst in m.select_many(super_kind):
        if not xtuml.navigate_subtype(inst, rel_id):
            res += 1
            logger.warning('integrity violation across '
                           '%s[%s]' % (super_kind, rel_id))

    return res

def
check_subtype_integrity
(
m
,
super_kind
,
rel_id
)
:
if
isinstance
(
rel_id
,
int
)
:
rel_id
=
'R%d'
%
rel_id
res
=
0
for
inst
in
m
.
select_many
(
super_kind
)
:
if
not
xtuml
.
navigate_subtype
(
inst
,
rel_id
)
:
res
+=
1
logger
.
warning
(
'integrity violation across '
'%s[%s]'
%
(
super_kind
,
rel_id
)
)
return
res

Check the model for integrity violations across a subtype association .


def feature_index(*feature_names):
    '''Returns a index creation function.

    Returns a valid index ``create`` function for the feature names
    given. This can be used with the :meth:`Store.define_index`
    method to create indexes on any combination of features in a
    feature collection.

    :type feature_names: list(unicode)
    :rtype: ``(val -> index val)
              -> (content_id, FeatureCollection)
              -> generator of [index val]``
    '''
    def _(trans, (cid, fc)):
        for fname in feature_names:
            feat = fc.get(fname)
            if feat is None:
                continue
            elif isinstance(feat, unicode):
                yield trans(feat)
            else:  # string counter, sparse/dense vector
                for val in feat.iterkeys():
                    yield trans(val)
    return _

def
feature_index
(
*
feature_names
)
:
def
_
(
trans
,
(
cid
,
fc
)
)
:
for
fname
in
feature_names
:
feat
=
fc
.
get
(
fname
)
if
feat
is
:
continue
elif
isinstance
(
feat
,
unicode
)
:
yield
trans
(
feat
)
else
:
# string counter, sparse/dense vector
for
val
in
feat
.
iterkeys
(
)
:
yield
trans
(
val
)
return
_


def basic_transform(val):
    '''A basic transform for strings and integers.'''
    if isinstance(val, int):
        return struct.pack('>i', val)
    else:
        return safe_lower_utf8(val)

def basic_transform ( val ) : if isinstance ( val , int ) : return struct . pack ( '>i' , val ) else : return safe_lower_utf8 ( val )

A basic transform for strings and integers .


def put(self, items, indexes=True):
        '''Add feature collections to the store.

        Given an iterable of tuples of the form
        ``(content_id, feature collection)``, add each to the store
        and overwrite any that already exist.

        This method optionally accepts a keyword argument `indexes`,
        which by default is set to ``True``. When it is ``True``,
        it will *create* new indexes for each content object for all
        indexes defined on this store.

        Note that this will not update existing indexes. (There is
        currently no way to do this without running some sort of
        garbage collection process.)

        :param iterable items: iterable of
                               ``(content_id, FeatureCollection)``.
        :type fc: :class:`dossier.fc.FeatureCollection`
        '''
        # So why accept an iterable? Ideally, some day, `kvlayer.put` would
        # accept an iterable, so we should too.
        #
        # But we have to transform it to a list in order to update indexes
        # anyway. Well, if we don't have to update indexes, then we can avoid
        # loading everything into memory, which seems like an optimization
        # worth having even if it's only some of the time.
        #
        # N.B. If you're thinking, "Just use itertools.tee", then you should
        # heed this warning from Python docs: "This itertool may require
        # significant auxiliary storage (depending on how much temporary data
        # needs to be stored). In general, if one iterator uses most or all of
        # the data before another iterator starts, it is faster to use list()
        # instead of tee()."
        #
        # i.e., `tee` has to store everything into memory because `kvlayer`
        # will exhaust the first iterator before indexes get updated.
        items = list(items)
        self.kvl.put(self.TABLE,
                     *imap(lambda (cid, fc): ((cid,), fc_dumps(fc)), items))
        if indexes:
            for idx_name in self._indexes:
                self._index_put(idx_name, *items)


def delete_all(self):
        '''Deletes all storage.

        This includes every content object and all index data.
        '''
        self.kvl.clear_table(self.TABLE)
        self.kvl.clear_table(self.INDEX_TABLE)

def delete_all ( self ) : self . kvl . clear_table ( self . TABLE ) self . kvl . clear_table ( self . INDEX_TABLE )


def scan(self, *key_ranges):
        '''Retrieve feature collections in a range of ids.

        Returns a generator of content objects corresponding to the
        content identifier ranges given. `key_ranges` can be a possibly
        empty list of 2-tuples, where the first element of the tuple
        is the beginning of a range and the second element is the end
        of a range. To specify the beginning or end of the table, use
        an empty tuple `()`.

        If the list is empty, then this yields all content objects in
        the storage.

        :param key_ranges: as described in
                           :meth:`kvlayer._abstract_storage.AbstractStorage`
        :rtype: generator of
                (``content_id``, :class:`dossier.fc.FeatureCollection`).
        '''
        # (id, id) -> ((id,), (id,))
        key_ranges = [(tuplify(s), tuplify(e)) for s, e in key_ranges]
        return imap(lambda (cid, fc): (cid[0], fc_loads(fc)),
                    self.kvl.scan(self.TABLE, *key_ranges))

def
scan
(
self
,
*
key_ranges
)
:
# (id, id) -> ((id,), (id,))
key_ranges
=
[
(
tuplify
(
s
)
,
tuplify
(
e
)
)
for
s
,
e
in
key_ranges
]
return
imap
(
lambda
(
cid
,
fc
)
:
(
cid
[
0
]
,
fc_loads
(
fc
)
)
,
self
.
kvl
.
scan
(
self
.
TABLE
,
*
key_ranges
)
)

Retrieve feature collections in a range of ids .


def scan_ids(self, *key_ranges):
        '''Retrieve content ids in a range of ids.

        Returns a generator of ``content_id`` corresponding to the
        content identifier ranges given. `key_ranges` can be a possibly
        empty list of 2-tuples, where the first element of the tuple
        is the beginning of a range and the second element is the end
        of a range. To specify the beginning or end of the table, use
        an empty tuple `()`.

        If the list is empty, then this yields all content ids in
        the storage.

        :param key_ranges: as described in
                           :meth:`kvlayer._abstract_storage.AbstractStorage`
        :rtype: generator of ``content_id``
        '''
        # (id, id) -> ((id,), (id,))
        key_ranges = [(tuplify(s), tuplify(e)) for s, e in key_ranges]
        scanner = self.kvl.scan_keys(self.TABLE, *key_ranges)
        return imap(itemgetter(0), scanner)

def
scan_ids
(
self
,
*
key_ranges
)
:
# (id, id) -> ((id,), (id,))
key_ranges
=
[
(
tuplify
(
s
)
,
tuplify
(
e
)
)
for
s
,
e
in
key_ranges
]
scanner
=
self
.
kvl
.
scan_keys
(
self
.
TABLE
,
*
key_ranges
)
return
imap
(
itemgetter
(
0
)
,
scanner
)

Retrieve content ids in a range of ids .


def index_scan(self, idx_name, val):
        '''Returns ids that match an indexed value.

        Returns a generator of content identifiers that have an entry
        in the index ``idx_name`` with value ``val`` (after index
        transforms are applied).

        If the index named by ``idx_name`` is not registered, then a
        :exc:`~exceptions.KeyError` is raised.

        :param unicode idx_name: name of index
        :param val: the value to use to search the index
        :type val: unspecified (depends on the index, usually ``unicode``)
        :rtype: generator of ``content_id``
        :raises: :exc:`~exceptions.KeyError`
        '''
        idx = self._index(idx_name)['transform']
        key = (idx(val), idx_name.encode('utf-8'))
        keys = self.kvl.scan_keys(self.INDEX_TABLE, (key, key))
        return imap(lambda k: k[2], keys)

def index_scan ( self , idx_name , val ) : idx = self . _index ( idx_name ) [ 'transform' ] key = ( idx ( val ) , idx_name . encode ( 'utf-8' ) ) keys = self . kvl . scan_keys ( self . INDEX_TABLE , ( key , key ) ) return imap ( lambda k : k [ 2 ] , keys )

Returns ids that match an indexed value .


def index_scan_prefix(self, idx_name, val_prefix):
        '''Returns ids that match a prefix of an indexed value.

        Returns a generator of content identifiers that have an entry
        in the index ``idx_name`` with prefix ``val_prefix`` (after
        index transforms are applied).

        If the index named by ``idx_name`` is not registered, then a
        :exc:`~exceptions.KeyError` is raised.

        :param unicode idx_name: name of index
        :param val_prefix: the value to use to search the index
        :type val: unspecified (depends on the index, usually ``unicode``)
        :rtype: generator of ``content_id``
        :raises: :exc:`~exceptions.KeyError`
        '''
        return self._index_scan_prefix_impl(
            idx_name, val_prefix, lambda k: k[2])

def index_scan_prefix ( self , idx_name , val_prefix ) : return self . _index_scan_prefix_impl ( idx_name , val_prefix , lambda k : k [ 2 ] )

Returns ids that match a prefix of an indexed value .


def index_scan_prefix_and_return_key(self, idx_name, val_prefix):
        '''Returns ids that match a prefix of an indexed value, and the
        specific key that matched the search prefix.

        Returns a generator of (index key, content identifier) that
        have an entry in the index ``idx_name`` with prefix
        ``val_prefix`` (after index transforms are applied).

        If the index named by ``idx_name`` is not registered, then a
        :exc:`~exceptions.KeyError` is raised.

        :param unicode idx_name: name of index
        :param val_prefix: the value to use to search the index
        :type val: unspecified (depends on the index, usually ``unicode``)
        :rtype: generator of (``index key``, ``content_id``)
        :raises: :exc:`~exceptions.KeyError`

        '''
        return self._index_scan_prefix_impl(
            idx_name, val_prefix, lambda k: (k[0], k[2]))

def index_scan_prefix_and_return_key ( self , idx_name , val_prefix ) : return self . _index_scan_prefix_impl ( idx_name , val_prefix , lambda k : ( k [ 0 ] , k [ 2 ] ) )

Returns ids that match a prefix of an indexed value and the specific key that matched the search prefix .


def _index_scan_prefix_impl(self, idx_name, val_prefix, retfunc):
        '''Implementation for index_scan_prefix and
        index_scan_prefix_and_return_key, parameterized on return
        value function.

        retfunc gets passed a key tuple from the index:
        (index name, index value, content_id)
        '''
        idx = self._index(idx_name)['transform']
        val_prefix = idx(val_prefix)

        idx_name = idx_name.encode('utf-8')
        s = (val_prefix, idx_name)
        e = (val_prefix + '\xff', idx_name)
        keys = self.kvl.scan_keys(self.INDEX_TABLE, (s, e))
        return imap(retfunc, keys)

def _index_scan_prefix_impl ( self , idx_name , val_prefix , retfunc ) : idx = self . _index ( idx_name ) [ 'transform' ] val_prefix = idx ( val_prefix ) idx_name = idx_name . encode ( 'utf-8' ) s = ( val_prefix , idx_name ) e = ( val_prefix + '\xff' , idx_name ) keys = self . kvl . scan_keys ( self . INDEX_TABLE , ( s , e ) ) return imap ( retfunc , keys )

Implementation for index_scan_prefix and index_scan_prefix_and_return_key parameterized on return value function .


def define_index(self, idx_name, create, transform):
        '''Add an index to this store instance.

        Adds an index transform to the current FC store. Once an index
        with name ``idx_name`` is added, it will be available in all
        ``index_*`` methods. Additionally, the index will be automatically
        updated on calls to :meth:`~dossier.fc.store.Store.put`.

        If an index with name ``idx_name`` already exists, then it is
        overwritten.

        Note that indexes do *not* persist. They must be re-defined for
        each instance of :class:`Store`.

        For example, to add an index on the ``boNAME`` feature, you can
        use the ``feature_index`` helper function:

        .. code-block:: python

            store.define_index('boNAME',
                               feature_index('boNAME'),
                               lambda s: s.encode('utf-8'))

        Another example for creating an index on names:

        .. code-block:: python

            store.define_index('NAME',
                               feature_index('canonical_name', 'NAME'),
                               lambda s: s.lower().encode('utf-8'))

        :param idx_name: The name of the index. Must be UTF-8 encodable.
        :type idx_name: unicode
        :param create: A function that accepts the ``transform`` function and
                       a pair of ``(content_id, fc)`` and produces a generator
                       of index values from the pair given using ``transform``.
        :param transform: A function that accepts an arbitrary value and
                          applies a transform to it. This transforms the
                          *stored* value to the *index* value. This *must*
                          produce a value with type `str` (or `bytes`).
        '''
        assert isinstance(idx_name, (str, unicode))
        idx_name = idx_name.decode('utf-8')
        self._indexes[idx_name] = {'create': create, 'transform': transform}

def define_index ( self , idx_name , create , transform ) : assert isinstance ( idx_name , ( str , unicode ) ) idx_name = idx_name . decode ( 'utf-8' ) self . _indexes [ idx_name ] = { 'create' : create , 'transform' : transform }


def _index_put(self, idx_name, *ids_and_fcs):
        '''Add new index values.

        Adds new index values for index ``idx_name`` for the pairs
        given. Each pair should be a content identifier and a
        :class:`dossier.fc.FeatureCollection`.

        :type idx_name: unicode
        :type ids_and_fcs: ``[(content_id, FeatureCollection)]``
        '''
        keys = self._index_keys_for(idx_name, *ids_and_fcs)
        with_vals = map(lambda k: (k, '0'), keys)
        # TODO: use imap when kvl.put takes an iterable
        self.kvl.put(self.INDEX_TABLE, *with_vals)


def _index_put_raw(self, idx_name, content_id, val):
        '''Add new raw index values.

        Adds a new index key corresponding to
        ``(idx_name, transform(val), content_id)``.

        This method bypasses the *creation* of indexes from content
        objects, but values are still transformed.

        :type idx_name: unicode
        :type content_id: str
        :type val: unspecified (depends on the index, usually ``unicode``)
        '''
        idx = self._index(idx_name)['transform']
        key = (idx(val), idx_name.encode('utf-8'), content_id)
        self.kvl.put(self.INDEX_TABLE, (key, '0'))

def _index_put_raw ( self , idx_name , content_id , val ) : idx = self . _index ( idx_name ) [ 'transform' ] key = ( idx ( val ) , idx_name . encode ( 'utf-8' ) , content_id ) self . kvl . put ( self . INDEX_TABLE , ( key , '0' ) )


def _index_keys_for(self, idx_name, *ids_and_fcs):
        '''Returns a generator of index triples.

        Returns a generator of index keys for the ``ids_and_fcs`` pairs
        given. The index keys have the form ``(idx_name, idx_val,
        content_id)``.

        :type idx_name: unicode
        :type ids_and_fcs: ``[(content_id, FeatureCollection)]``
        :rtype: generator of ``(str, str, str)``
        '''
        idx = self._index(idx_name)
        icreate, itrans = idx['create'], idx['transform']
        if isinstance(idx_name, unicode):
            idx_name = idx_name.encode('utf-8')

        for cid_fc in ids_and_fcs:
            content_id = cid_fc[0]

            # Be sure to dedup index_values or else we may
            # suffer duplicate_pkey errors down the line.
            seen_values = set()
            for index_value in icreate(itrans, cid_fc):
                if index_value and index_value not in seen_values:
                    yield (index_value, idx_name, content_id)
                    seen_values.add(index_value)


def _index(self, name):
        '''Returns index transforms for ``name``.

        :type name: unicode
        :rtype: ``{ create |--> function, transform |--> function }``
        '''
        name = name.decode('utf-8')
        try:
            return self._indexes[name]
        except KeyError:
            raise KeyError('Index "%s" has not been registered with '
                           'this FC store.' % name)


def check_pypi_name(pypi_package_name, pypi_registry_host=None):
    """
    Check if a package name exists on pypi.

    TODO: Document the Registry URL construction.
        It may not be obvious how pypi_package_name and pypi_registry_host are used
        I'm appending the simple HTTP API parts of the registry standard specification.

    It will return True if the package name, or any equivalent variation as defined by PEP 503 normalisation
     rules (https://www.python.org/dev/peps/pep-0503/#normalized-names) is registered in the PyPI registry.

    >>> check_pypi_name('pip')
    True
    >>> check_pypi_name('Pip')
    True

    It will return False if the package name, or any equivalent variation as defined by PEP 503 normalisation
     rules (https://www.python.org/dev/peps/pep-0503/#normalized-names) is not registered in the PyPI registry.

    >>> check_pypi_name('testy_mc-test_case-has.a.cousin_who_should_never_write_a_package')
    False

    :param pypi_package_name:
    :param pypi_registry_host:
    :return:
    """
    if pypi_registry_host is None:
        pypi_registry_host = 'pypi.python.org'

    # Just a helpful reminder why this bytearray size was chosen.
    #                            HTTP/1.1 200 OK
    #                            HTTP/1.1 404 Not Found
    receive_buffer = bytearray(b'------------')
    context = ssl.create_default_context()
    ssl_http_socket = context.wrap_socket(socket.socket(socket.AF_INET), server_hostname=pypi_registry_host)
    ssl_http_socket.connect((pypi_registry_host, 443))
    ssl_http_socket.send(b''.join([
        b"HEAD /simple/", pypi_package_name.encode('ascii'), b"/ HTTP/1.0", b"\r\n",
        b"Host: ", pypi_registry_host.encode('ascii'), b"\r\n",
        b"\r\n\r\n"
    ]))
    ssl_http_socket.recv_into(receive_buffer)

    # Early return when possible.
    if b'HTTP/1.1 200' in receive_buffer:
        ssl_http_socket.shutdown(1)
        ssl_http_socket.close()
        return True
    elif b'HTTP/1.1 404' in receive_buffer:
        ssl_http_socket.shutdown(1)
        ssl_http_socket.close()
        return False

    remaining_bytes = ssl_http_socket.recv(2048)
    redirect_path_location_start = remaining_bytes.find(b'Location:') + 10
    redirect_path_location_end = remaining_bytes.find(b'\r\n', redirect_path_location_start)
    # Append the trailing slash to avoid a needless extra redirect.
    redirect_path = remaining_bytes[redirect_path_location_start:redirect_path_location_end] + b'/'

    ssl_http_socket.shutdown(1)
    ssl_http_socket.close()

    # Reset the bytearray to empty
    # receive_buffer = bytearray(b'------------')

    ssl_http_socket = context.wrap_socket(socket.socket(socket.AF_INET), server_hostname=pypi_registry_host)
    ssl_http_socket.connect((pypi_registry_host, 443))

    ssl_http_socket.send(b''.join([
        b"HEAD ", redirect_path, b" HTTP/1.0", b"\r\n",
        b"Host: ", pypi_registry_host.encode('ascii'), b"\r\n",
        b"\r\n\r\n"]))
    ssl_http_socket.recv_into(receive_buffer)

    if b'HTTP/1.1 200' in receive_buffer:
        return True
    elif b'HTTP/1.1 404' in receive_buffer:
        return False
    else:
        NotImplementedError('A definitive answer was not found by primary or secondary lookups.')

#                            HTTP/1.1 404 Not Found

Check if a package name exists on pypi .


def add_direction(value, arg=u"rtl_only"):
    """Adds direction to the element

    :arguments:
        arg
            * rtl_only: Add the direction only in case of a
              right-to-left language (default)
            * both: add the direction in both case
            * ltr_only: Add the direction only in case of a
              left-to-right language

    {{image_name|add_direction}} when image_name is 'start_arrow.png'
    results in 'start_arrow_rtl.png' in case of RTL language, and
    'start_arrow.png' or 'start_arrow_ltr.png' depends on `arg` value.

    """

    if arg == u'rtl_only':
        directions = (u'', u'_rtl')
    elif arg == u'both':
        directions = (u'_ltr', u'_rtl')
    elif arg == u'ltr_only':
        directions = (u'_ltr', u'')
    else:
        raise template.TemplateSyntaxError('add_direction can use arg with one of ["rtl_only", "both", "ltr_only"]')

    parts = value.rsplit('.', 1)
    if not len(parts):
        return value
    elif len(parts) == 1:
        return value + directions[translation.get_language_bidi()]
    else:
        return '.'.join((parts[0]+directions[translation.get_language_bidi()],parts[1]))


def get_type_name(s_dt):
    '''
    get the xsd name of a S_DT
    '''
    s_cdt = nav_one(s_dt).S_CDT[17]()
    if s_cdt and s_cdt.Core_Typ in range(1, 6):
        return s_dt.Name

    s_edt = nav_one(s_dt).S_EDT[17]()
    if s_edt:
        return s_dt.Name

    s_udt = nav_one(s_dt).S_UDT[17]()
    if s_udt:
        return s_dt.Name


def get_refered_attribute(o_attr):
    '''
    Get the the referred attribute.
    '''
    o_attr_ref = nav_one(o_attr).O_RATTR[106].O_BATTR[113].O_ATTR[106]()
    if o_attr_ref:
        return get_refered_attribute(o_attr_ref)
    else:
        return o_attr


def build_core_type(s_cdt):
    '''
    Build an xsd simpleType out of a S_CDT.
    '''
    s_dt = nav_one(s_cdt).S_DT[17]()

    if s_dt.name == 'void':
        type_name = None

    elif s_dt.name == 'boolean':
        type_name = 'xs:boolean'

    elif s_dt.name == 'integer':
        type_name = 'xs:integer'

    elif s_dt.name == 'real':
        type_name = 'xs:decimal'

    elif s_dt.name == 'string':
        type_name = 'xs:string'

    elif s_dt.name == 'unique_id':
        type_name = 'xs:integer'

    else:
        type_name = None

    if type_name:
        mapped_type = ET.Element('xs:simpleType', name=s_dt.name)
        ET.SubElement(mapped_type, 'xs:restriction', base=type_name)
        return mapped_type

def build_core_type ( s_cdt ) : s_dt = nav_one ( s_cdt ) . S_DT [ 17 ] ( ) if s_dt . name == 'void' : type_name = None elif s_dt . name == 'boolean' : type_name = 'xs:boolean' elif s_dt . name == 'integer' : type_name = 'xs:integer' elif s_dt . name == 'real' : type_name = 'xs:decimal' elif s_dt . name == 'string' : type_name = 'xs:string' elif s_dt . name == 'unique_id' : type_name = 'xs:integer' else : type_name = None if type_name : mapped_type = ET . Element ( 'xs:simpleType' , name = s_dt . name ) ET . SubElement ( mapped_type , 'xs:restriction' , base = type_name ) return mapped_type

Build an xsd simpleType out of a S_CDT .


def build_enum_type(s_edt):
    '''
    Build an xsd simpleType out of a S_EDT.
    '''
    s_dt = nav_one(s_edt).S_DT[17]()
    enum = ET.Element('xs:simpleType', name=s_dt.name)
    enum_list = ET.SubElement(enum, 'xs:restriction', base='xs:string')

    first_filter = lambda selected: not nav_one(selected).S_ENUM[56, 'succeeds']()

    s_enum = nav_any(s_edt).S_ENUM[27](first_filter)
    while s_enum:
        ET.SubElement(enum_list, 'xs:enumeration', value=s_enum.name)
        s_enum = nav_one(s_enum).S_ENUM[56, 'precedes']()

    return enum

def build_enum_type ( s_edt ) : s_dt = nav_one ( s_edt ) . S_DT [ 17 ] ( ) enum = ET . Element ( 'xs:simpleType' , name = s_dt . name ) enum_list = ET . SubElement ( enum , 'xs:restriction' , base = 'xs:string' ) first_filter = lambda selected : not nav_one ( selected ) . S_ENUM [ 56 , 'succeeds' ] ( ) s_enum = nav_any ( s_edt ) . S_ENUM [ 27 ] ( first_filter ) while s_enum : ET . SubElement ( enum_list , 'xs:enumeration' , value = s_enum . name ) s_enum = nav_one ( s_enum ) . S_ENUM [ 56 , 'precedes' ] ( ) return enum

Build an xsd simpleType out of a S_EDT .


def build_struct_type(s_sdt):
    '''
    Build an xsd complexType out of a S_SDT.
    '''
    s_dt = nav_one(s_sdt).S_DT[17]()
    struct = ET.Element('xs:complexType', name=s_dt.name)

    first_filter = lambda selected: not nav_one(selected).S_MBR[46, 'succeeds']()

    s_mbr = nav_any(s_sdt).S_MBR[44](first_filter)
    while s_mbr:
        s_dt = nav_one(s_mbr).S_DT[45]()
        type_name = get_type_name(s_dt)
        ET.SubElement(struct, 'xs:attribute', name=s_mbr.name, type=type_name)
        s_mbr = nav_one(s_mbr).S_MBR[46, 'precedes']()

    return struct

def build_struct_type ( s_sdt ) : s_dt = nav_one ( s_sdt ) . S_DT [ 17 ] ( ) struct = ET . Element ( 'xs:complexType' , name = s_dt . name ) first_filter = lambda selected : not nav_one ( selected ) . S_MBR [ 46 , 'succeeds' ] ( ) s_mbr = nav_any ( s_sdt ) . S_MBR [ 44 ] ( first_filter ) while s_mbr : s_dt = nav_one ( s_mbr ) . S_DT [ 45 ] ( ) type_name = get_type_name ( s_dt ) ET . SubElement ( struct , 'xs:attribute' , name = s_mbr . name , type = type_name ) s_mbr = nav_one ( s_mbr ) . S_MBR [ 46 , 'precedes' ] ( ) return struct

Build an xsd complexType out of a S_SDT .


def build_user_type(s_udt):
    '''
    Build an xsd simpleType out of a S_UDT.
    '''
    s_dt_user = nav_one(s_udt).S_DT[17]()
    s_dt_base = nav_one(s_udt).S_DT[18]()

    base_name = get_type_name(s_dt_base)
    if base_name:
        user = ET.Element('xs:simpleType', name=s_dt_user.name)
        ET.SubElement(user, 'xs:restriction', base=base_name)

        return user

def build_user_type ( s_udt ) : s_dt_user = nav_one ( s_udt ) . S_DT [ 17 ] ( ) s_dt_base = nav_one ( s_udt ) . S_DT [ 18 ] ( ) base_name = get_type_name ( s_dt_base ) if base_name : user = ET . Element ( 'xs:simpleType' , name = s_dt_user . name ) ET . SubElement ( user , 'xs:restriction' , base = base_name ) return user

Build an xsd simpleType out of a S_UDT .


def build_type(s_dt):
    '''
    Build a partial xsd tree out of a S_DT and its sub types S_CDT, S_EDT, S_SDT and S_UDT.
    '''
    s_cdt = nav_one(s_dt).S_CDT[17]()
    if s_cdt:
        return build_core_type(s_cdt)

    s_edt = nav_one(s_dt).S_EDT[17]()
    if s_edt:
        return build_enum_type(s_edt)

    s_udt = nav_one(s_dt).S_UDT[17]()
    if s_udt:
        return build_user_type(s_udt)

Build a partial xsd tree out of a S_DT and its sub types S_CDT S_EDT S_SDT and S_UDT .


def build_class(o_obj):
    '''
    Build an xsd complex element out of a O_OBJ, including its O_ATTR.
    '''
    cls = ET.Element('xs:element', name=o_obj.key_lett, minOccurs='0', maxOccurs='unbounded')
    attributes = ET.SubElement(cls, 'xs:complexType')
    for o_attr in nav_many(o_obj).O_ATTR[102]():
        o_attr_ref = get_refered_attribute(o_attr)
        s_dt = nav_one(o_attr_ref).S_DT[114]()
        while nav_one(s_dt).S_UDT[17]():
            s_dt = nav_one(s_dt).S_UDT[17].S_DT[18]()

        type_name = get_type_name(s_dt)
        if type_name and not nav_one(o_attr).O_BATTR[106].O_DBATTR[107]():
            ET.SubElement(attributes, 'xs:attribute', name=o_attr.name, type=type_name)
        else:
            logger.warning('Omitting %s.%s' % (o_obj.key_lett, o_attr.Name))
    return cls

def
build_class
(
o_obj
)
:
cls
=
ET
.
Element
(
'xs:element'
,
name
=
o_obj
.
key_lett
,
minOccurs
=
'0'
,
maxOccurs
=
'unbounded'
)
attributes
=
ET
.
SubElement
(
cls
,
'xs:complexType'
)
for
o_attr
in
nav_many
(
o_obj
)
.
O_ATTR
[
102
]
(
)
:
o_attr_ref
=
get_refered_attribute
(
o_attr
)
s_dt
=
nav_one
(
o_attr_ref
)
.
S_DT
[
114
]
(
)
while
nav_one
(
s_dt
)
.
S_UDT
[
17
]
(
)
:
s_dt
=
nav_one
(
s_dt
)
.
S_UDT
[
17
]
.
S_DT
[
18
]
(
)
type_name
=
get_type_name
(
s_dt
)
if
type_name
and
not
nav_one
(
o_attr
)
.
O_BATTR
[
106
]
.
O_DBATTR
[
107
]
(
)
:
ET
.
SubElement
(
attributes
,
'xs:attribute'
,
name
=
o_attr
.
name
,
type
=
type_name
)
else
:
logger
.
warning
(
'Omitting %s.%s'
%
(
o_obj
.
key_lett
,
o_attr
.
Name
)
)
return
cls

Build an xsd complex element out of a O_OBJ including its O_ATTR .