streamcorpus_pipeline/_pipeline.py
def create(self, stage, scp_config, config=None):
        '''Create a pipeline stage.

        Instantiates `stage` with `config`.  This essentially
        translates to ``stage(config)``, except that two keys from
        `scp_config` are injected into the configuration:
        ``tmp_dir_path`` is an execution-specific directory from
        combining the top-level ``tmp_dir_path`` configuration with
        :attr:`tmp_dir_suffix`; and ``third_dir_path`` is the same
        path from the top-level configuration.  `stage` may be either
        a callable returning the stage (e.g. its class), or its name
        in the configuration.

        `scp_config` is the configuration for the pipeline as a
        whole, and is required.  `config` is the configuration for
        the stage; if it is :const:`None` then it is extracted
        from `scp_config`.

        If you already have a fully formed configuration block
        and want to create a stage, you can call

        .. code-block:: python

            factory.registry[stage](stage_config)

        In most cases if you have a stage class object and want to
        instantiate it with its defaults you can call

        .. code-block:: python

            stage = stage_cls(stage_cls.default_config)

        .. note:: This mirrors
                  :meth:`yakonfig.factory.AutoFactory.create`, with
                  some thought that this factory class might migrate
                  to using that as a base in the future.

        :param stage: pipeline stage class, or its name in the registry
        :param dict scp_config: configuration block for the pipeline
        :param dict config: configuration block for the stage, or
          :const:`None` to get it from `scp_config`

        '''
        # Figure out what we have for a stage and its name
        if isinstance(stage, basestring):
            stage_name = stage
            stage_obj = self.registry[stage_name]
        else:
            stage_name = getattr(stage, 'config_name', stage.__name__)
            stage_obj = stage

        # Find the configuration; get a copy we can mutate
        if config is None:
            config = scp_config.get(stage_name, None)
        if config is None:
            config = getattr(stage_obj, 'default_config', {})
        config = dict(config)

        # Fill in more values
        if self.tmp_dir_suffix is None:
            config['tmp_dir_path'] = scp_config['tmp_dir_path']
        else:
            config['tmp_dir_path'] = os.path.join(scp_config['tmp_dir_path'],
                                                  self.tmp_dir_suffix)
        config['third_dir_path'] = scp_config['third_dir_path']

        return stage_obj(config)
def create(self, stage, scp_config, config=None):
        '''Create a pipeline stage.

        Instantiates `stage` with `config`.  This essentially
        translates to ``stage(config)``, except that two keys from
        `scp_config` are injected into the configuration:
        ``tmp_dir_path`` is an execution-specific directory from
        combining the top-level ``tmp_dir_path`` configuration with
        :attr:`tmp_dir_suffix`; and ``third_dir_path`` is the same
        path from the top-level configuration.  `stage` may be either
        a callable returning the stage (e.g. its class), or its name
        in the configuration.

        `scp_config` is the configuration for the pipeline as a
        whole, and is required.  `config` is the configuration for
        the stage; if it is :const:`None` then it is extracted
        from `scp_config`.

        If you already have a fully formed configuration block
        and want to create a stage, you can call

        .. code-block:: python

            factory.registry[stage](stage_config)

        In most cases if you have a stage class object and want to
        instantiate it with its defaults you can call

        .. code-block:: python

            stage = stage_cls(stage_cls.default_config)

        .. note:: This mirrors
                  :meth:`yakonfig.factory.AutoFactory.create`, with
                  some thought that this factory class might migrate
                  to using that as a base in the future.

        :param stage: pipeline stage class, or its name in the registry
        :param dict scp_config: configuration block for the pipeline
        :param dict config: configuration block for the stage, or
          :const:`None` to get it from `scp_config`

        '''
        # Figure out what we have for a stage and its name
        if isinstance(stage, basestring):
            stage_name = stage
            stage_obj = self.registry[stage_name]
        else:
            stage_name = getattr(stage, 'config_name', stage.__name__)
            stage_obj = stage

        # Find the configuration; get a copy we can mutate
        if config is None:
            config = scp_config.get(stage_name, None)
        if config is None:
            config = getattr(stage_obj, 'default_config', {})
        config = dict(config)

        # Fill in more values
        if self.tmp_dir_suffix is None:
            config['tmp_dir_path'] = scp_config['tmp_dir_path']
        else:
            config['tmp_dir_path'] = os.path.join(scp_config['tmp_dir_path'],
                                                  self.tmp_dir_suffix)
        config['third_dir_path'] = scp_config['third_dir_path']

        return stage_obj(config)
def create ( self , stage , scp_config , config = None ) : # Figure out what we have for a stage and its name if isinstance ( stage , basestring ) : stage_name = stage stage_obj = self . registry [ stage_name ] else : stage_name = getattr ( stage , 'config_name' , stage . __name__ ) stage_obj = stage # Find the configuration; get a copy we can mutate if config is None : config = scp_config . get ( stage_name , None ) if config is None : config = getattr ( stage_obj , 'default_config' , { } ) config = dict ( config ) # Fill in more values if self . tmp_dir_suffix is None : config [ 'tmp_dir_path' ] = scp_config [ 'tmp_dir_path' ] else : config [ 'tmp_dir_path' ] = os . path . join ( scp_config [ 'tmp_dir_path' ] , self . tmp_dir_suffix ) config [ 'third_dir_path' ] = scp_config [ 'third_dir_path' ] return stage_obj ( config )
Create a pipeline stage.

        Instantiates `stage` with `config`.  This essentially
        translates to ``stage(config)``, except that two keys from
        `scp_config` are injected into the configuration:
        ``tmp_dir_path`` is an execution-specific directory from
        combining the top-level ``tmp_dir_path`` configuration with
        :attr:`tmp_dir_suffix`; and ``third_dir_path`` is the same
        path from the top-level configuration.  `stage` may be either
        a callable returning the stage (e.g. its class), or its name
        in the configuration.

        `scp_config` is the configuration for the pipeline as a
        whole, and is required.  `config` is the configuration for
        the stage; if it is :const:`None` then it is extracted
        from `scp_config`.

        If you already have a fully formed configuration block
        and want to create a stage, you can call

        .. code-block:: python

            factory.registry[stage](stage_config)

        In most cases if you have a stage class object and want to
        instantiate it with its defaults you can call

        .. code-block:: python

            stage = stage_cls(stage_cls.default_config)

        .. note:: This mirrors
                  :meth:`yakonfig.factory.AutoFactory.create`, with
                  some thought that this factory class might migrate
                  to using that as a base in the future.

        :param stage: pipeline stage class, or its name in the registry
        :param dict scp_config: configuration block for the pipeline
        :param dict config: configuration block for the stage, or
          :const:`None` to get it from `scp_config`
Create a pipeline stage .