class PipelineFactory(object):
    def __init__(self, context, resolver=None):
        super(PipelineFactory, self).__init__()
        self.__context = context
        self.__resolver = resolver
        
    def __make_lang_pipeline(self, lang, ispiped, resolve, cmdtext):
        if ispiped:
            args = ['current', hotwire.script.PIPE]
        else:
            args = []
        if lang.builtin_eval:
            args.extend([lang.builtin_eval, cmdtext])
        else:
            args.append(lang.interpreter_exec)
            args.extend(lang.exec_args)
            args.append(cmdtext)
        pipeline = Pipeline.create(self.__context, resolve and self.__resolver or None, *args)
        return pipeline      
        
    def parse(self, text, curlang=None, resolve=True, **kwargs):
        # If input is not HotwirePipe, pass it through
        if curlang.uuid != '62270c40-a94a-44dd-aaa0-689f882acf34':
            return self.__make_lang_pipeline(curlang, False, resolve, text)
        ispiped = text.startswith('|')
        if ispiped:
            text = text[1:]        
        tokens = None
        for lang in PipelineLanguageRegistry.getInstance():
            # Skip languages which don't have a specified prefix - but we allow
            # setting them manually.
            if lang.prefix is None:
                continue
            elif not text.startswith(lang.prefix):
                continue
            _logger.debug("matched lang %r", lang)
            rest = text[len(lang.prefix):]
            if rest.strip() == '':
                scriptargs = ['hotscript', '--new', lang.uuid]
                if ispiped:
                    scriptargs.insert(0, hotwire.script.PIPE)
                    scriptargs.insert(0, 'current')
                return Pipeline.create(self.__context, resolve and self.__resolver or None, *scriptargs)
            else:
                # Require a space - should probably handle this better
                if not text.startswith(lang.prefix + " "):
                    continue
                return self.__make_lang_pipeline(lang, ispiped, resolve, rest[1:])
        # Try parsing as HotwirePipe
        if ispiped:
            text = 'current | ' + text
        return Pipeline.parse(text, context=self.__context, resolver=(resolve and self.__resolver or None), **kwargs) class Pipeline(object):
    """A sequence of Commands."""
    
    is_singlevalue = property(lambda self: self._is_singlevalue)

    __ws_re = re.compile(r'\s+')

    def __init__(self, components, input_type='unknown', input_optional=False,
                 output_type='unknown', locality=None,
                 idempotent=False,
                 undoable=False,
                 singlevalue=False):
        super(Pipeline, self).__init__()
        self.__executing_sync = False
        self.__components = components
        for component in self.__components:
            component.set_pipeline(self)
        self.__locality = locality
        self.__input_type = input_type
        self.__input_optional = input_optional
        self.__idempotent = idempotent
        self.__undoable = undoable
        self._is_singlevalue = singlevalue
        self.__output_type = output_type
        self.__undo = []
        self.__cmd_metadata_lock = threading.Lock()
        self.__idle_emit_cmd_metadata_id = 0
        self.__cmd_metaidx = {} # cmd -> int
        self.__cmd_metadata = {}
        self.__cmd_complete_count = 0
        self.__state = 'waiting'
        self.__completion_time = None
        
    def get_state(self):
        return self.__state     

    def disconnect(self):
        for cmd in self.__components:
            cmd.disconnect()
    
    def __execute_internal(self, force_sync, opt_formats=[], assert_all_threaded=False):
        _logger.debug("Executing %s", self)
        self.__set_state('executing')
        meta_idx = 0          
        for i,cmd in enumerate(self.__components):
            if assert_all_threaded and not cmd.builtin.threaded:
                raise ValueError("assert_all_threaded is enabled but trying to execute non-threaded builtin %s" % (cmd.builtin,))
            dispatcher.connect(self.__on_cmd_complete, 'complete', cmd)
            dispatcher.connect(self.__on_cmd_exception, "exception", cmd)
            # Here we record which commands include metadata, and
            # pass in the index in the pipeline for them.
            if cmd.builtin.hasmeta:
                _logger.debug("connecting to metadata on cmd %s, idx=%s", cmd, meta_idx)
                dispatcher.connect(self.__on_cmd_metadata, 'metadata', cmd)
                self.__cmd_metaidx[cmd] = meta_idx
                meta_idx += 1                
        prev_opt_formats = []
        for cmd in self.__components:
            if cmd.input:
                cmd.input.negotiate(prev_opt_formats, cmd.get_input_opt_formats())
            prev_opt_formats = cmd.get_output_opt_formats()
            if cmd.out_redir:
                prev_opt_formats = []
        last = self.__components[-1]
        if not last.out_redir:
            last_opt_fmts = last.get_output_opt_formats()
        else:
            last_opt_fmts = []
        last.output.negotiate(last_opt_fmts, opt_formats)
        for i,cmd in enumerate(self.__components[:-1]):
            cmd.execute(force_sync)
        last.execute(force_sync)
        
    def validate_state_transition(self, state):
        if self.__state == 'waiting':
            return state in ('executing', 'exception')
        elif self.__state == 'executing':
            return state in ('complete', 'cancelled', 'exception')
        elif self.__state in ('cancelled', 'exception', 'undone'):
            return None
        elif self.__state == 'complete':
            return state in ('undone',) and self.get_undoable()
        assert(False)
        
    def is_complete(self):
        return self.__state in ('complete', 'cancelled', 'exception', 'undone')
        
    def __set_state(self, state):
        trans = self.validate_state_transition(state)
        if trans is None:
            _logger.debug("ignoring transition from state %s to %s", self.__state, state)
            return
        elif not trans:
            raise ValueError("Invalid state transition %s to %s", self.__state, state)
        
        self.__state = state
        if self.is_complete():
            self.__completion_time = time.time()         
        dispatcher.send('state-changed', self)

    def execute(self, **kwargs):
        self.__execute_internal(False, **kwargs)

    def execute_sync(self, **kwargs):
        self.__executing_sync = True
        self.__execute_internal(True, **kwargs)

    def push_undo(self, fn):
        self.__undo.append(fn)

    def get_undoable(self):
        return self.__undoable

    def undo(self):
        for fn in self.__undo:
            fn()
        self.__set_state('undone')

    def get_completion_time(self):
        return self.__completion_time

    def get_idempotent(self):
        return self.__idempotent

    def get_status_commands(self):
        for cmd in self.__components:
            if cmd.builtin.hasstatus:
                yield cmd.builtin.name

    def __on_cmd_metadata(self, key, flags, meta, sender=None):
        cmd = sender
        cmdidx = self.__cmd_metaidx[cmd]
        self.__cmd_metadata_lock.acquire()
        if self.__idle_emit_cmd_metadata_id == 0:
            self.__idle_emit_cmd_metadata_id = call_timeout(200, self.__idle_emit_cmd_metadata)
        self.__cmd_metadata[(cmd, cmdidx, key)] = (flags, meta)
        self.__cmd_metadata_lock.release()

    def __idle_emit_cmd_metadata(self):
        _logger.debug("signalling command metadata")      
        self.__cmd_metadata_lock.acquire()
        self.__idle_emit_cmd_metadata_id = 0
        meta_ref = self.__cmd_metadata
        self.__cmd_metadata = {}
        self.__cmd_metadata_lock.release()
        for (cmd,cmdidx,key),(flags,meta) in meta_ref.iteritems():
            dispatcher.send('metadata', self, cmdidx, cmd, key, flags, meta)

    def __on_cmd_complete(self, sender=None):
        cmd = sender
        _logger.debug("command complete: %s", cmd)
        if cmd.get_executing_sync():
            self.__idle_handle_cmd_complete(cmd)
        else:  
            call_idle(lambda: self.__idle_handle_cmd_complete(cmd))
        
    def __idle_handle_cmd_complete(self, cmd):
        self.__cmd_complete_count += 1
        if self.__cmd_complete_count == len(self.__components):
            self.__set_state('complete')

    def __on_cmd_exception(self, e, sender=None):
        cmd = sender
        if not self.__state == 'executing':
            return        
        try:
            self.cancel(changestate=False)
        except:
            _logger.exception("Nested exception while cancelling")
            pass
        dispatcher.send('exception', self, e, cmd)
        self.__exception_info = (e.__class__, str(e), cmd, traceback.format_exc())
        self.__set_state('exception')
        
    def get_exception_info(self):
        return self.__exception_info

    def get_output(self):
        return self.__components[-1].output

    def get_input_type(self):
        return self.__input_type

    def get_input_optional(self):
        return self.__input_optional

    def get_output_type(self):
        return self.__output_type
    
    def get_output_metadata(self):
        return PipelineTypeData(self)

    def get_auxstreams(self):
        for cmd in self.__components:
            for obj in cmd.get_auxstreams():
                yield obj

    def cancel(self, changestate=True):
        if not self.__state == 'executing':
            return
        if changestate:
            self.__set_state('cancelled')        
        for component in self.__components:
            component.cancel()        

    def is_nodisplay(self):
        return self.__components[0].builtin.nodisplay

    def set_output_queue(self, queue, map_fn):
        self.__components[-1].set_output_queue(queue, map_fn)
        
    def set_input_queue(self, queue):
        # FIXME - remove this is_first bit
        self.__components[0].set_input(queue, is_first=True)

    def get_locality(self):
        return self.__locality

    @staticmethod
    def streamtype_is_assignable(out_spec, in_spec, in_optional):
        if out_spec is None:
            return in_optional
        if in_spec in ('any', 'identity'):
            return True
        if out_spec == 'any':
            # Allow anys to connect to anything...even though it may TypeError
            return True
        if out_spec is in_spec:
            return True
        return class_is_assignable(in_spec, out_spec)

    @staticmethod
    def __parse_option_or_arg(opts, arg, raise_on_invalid=True):
        """If argument string is an option, parse it into components and canonicalize it.
Otherwise, return arg."""
        if arg.startswith('--'):
            args = [arg[1:]] # we re-add the '-' below
        elif arg.startswith('-') and len(arg) >= 2:
            args = list(arg[1:])
        else:
            return False
        results = []
        for arg in args:
            found = False
            if opts is not None:
                for aliases in opts:
                    if '-'+arg in aliases:
                        results.append(aliases[0])
                        found = True
                        break
            if (not found) and raise_on_invalid:                
                    raise PipelineParseException("Invalid option %s" % (arg,))
        return results

    @staticmethod
    def mkparser(text):
        if isinstance(text, unicode):
            utext = text
        else:
            utext = unicode(text, 'utf-8')
        countstream = CountingStream(StringIO(utext))
        parser = shlex.shlex(countstream, posix=True)
        parser.wordchars += ',./[]{}~!@$%^&*()-_=+:;'
        return (countstream, parser)

    @staticmethod
    def tokenize(text, context=None, assertfn=None, accept_partial=False, internal=False):
        result = []
        _logger.debug("parsing '%s'", text)
        
        (countstream, parser) = Pipeline.mkparser(text)
        
        def mktoken(text, *args, **kwargs):
            if internal:
                return ParsedToken(text, -1, **kwargs)
            return ParsedToken(text, *args, **kwargs)
        
        is_initial = True
        first_token = None
        curpos = 0
        while True:
            try:
                (token, quoted) = parser.get_token_info()
            except ValueError, e:
                # FIXME gross, but...any way to fix?
                msg = hasattr(e, 'message') and e.message or (e.args[0])
                was_quotation_error = (e.message == 'No closing quotation' and parser.token[0:1] == "'")
                if (not accept_partial) or (not was_quotation_error):
                    _logger.debug("caught lexing exception", exc_info=True)
                    raise PipelineParseException(e)
                arg = parser.token[1:]
                if arg:
                    token = mktoken(arg, curpos+1, was_unquoted=True)
                    _logger.debug("handling unclosed quote, returning %s", token)
                    yield token
                    return
                else:
                    _logger.debug("handling unclosed quote, but token was empty")
                    return
            _logger.debug("tokenize: %r", token)
            # empty input
            if token is None:
                break 
            is_initial = False
            end = countstream.get_count()
            if not quoted and token in ('|', '<', '>', '>>'):
                if token == '|':
                    yield hotwire.script.PIPE
                elif token == '>':
                    yield hotwire.script.REDIR_OUT
                elif token == '>>':
                    yield hotwire.script.REDIR_OUT_APPEND
                elif token == '<':
                    yield hotwire.script.REDIR_IN           
            else:
                if end-curpos > len(token):
                    end = curpos+len(token)
                yield mktoken(token, curpos, end=end, quoted=quoted)
            curpos = end
        
    @staticmethod
    def create(context, resolver, *tokens, **kwargs):
        if context is None:
            context = HotwireContext()
        accept_partial = (('accept_partial' in kwargs) and kwargs['accept_partial'])
        components = []
        undoable = None
        idempotent = True
        prev = None
        pipeline_input_type = 'unknown'
        pipeline_input_optional = 'unknown'
        pipeline_output_type = None
        prev_locality = None
        pipeline_singlevalue = None
        pipeline_type_validates = True
        pushback = []
        tokens = list(tokens)
        def pull_token():
            if pushback:
                return pushback.pop(0)
            else:
                return tokens.pop(0)            
        while tokens or pushback:
            builtin_token = pull_token()
            _logger.debug("token = %r", builtin_token)    
                
            def forcetoken(t):
                # Allow passing plain strings for convenience from Python.
                # Treat them as quoted.
                if isinstance(t, basestring):
                    return ParsedToken(t, -1, quoted=True)
                return t
            
            builtin_token = forcetoken(builtin_token)

            # Attempt to determine which builtin we're using
            if isinstance(builtin_token, Builtin):
                b = builtin_token
                cmdargs = []
            # If we're parsing without a resolver, assume we're using sys
            elif isinstance(builtin_token, ParsedToken):
                try:
                    b = BuiltinRegistry.getInstance()[builtin_token.text]
                    cmdargs = []
                except KeyError, e:
                    if resolver:
                        (b, cmdargs) = resolver.resolve(builtin_token.text, context)
                        _logger.debug("resolved: %r to %r %r", builtin_token.text, b, cmdargs)
                        if not b:
                            raise PipelineParseException(_('No matches for %s') % (builtin_token.text,))
                    else:
                        b = BuiltinRegistry.getInstance()['sys']
                        cmdargs = [ParsedToken(builtin_token.text, builtin_token.start, end=builtin_token.end)]
            elif builtin_token in (hotwire.script.PIPE, hotwire.script.REDIR_IN, hotwire.script.REDIR_OUT, hotwire.script.REDIR_OUT_APPEND):
                raise PipelineParseException(_("Unexpected token %d") % (builtin_token,))
            else:
                _logger.error("unknown in parse stream: %r", builtin_token)
                assert False
                
            _logger.debug("target builtin is %r", b)
                
            # We maintain the set of all tokens we processed in the command so that the completion system can use them.
            alltokens = [builtin_token]
            cmdargs = map(forcetoken, cmdargs)
            alltokens.extend(cmdargs)
                
            in_redir = None
            out_redir = None

            # Pull from the stream to get all the arguments
            while tokens:
                cmdarg = forcetoken(pull_token())
                if cmdarg == hotwire.script.PIPE:
                    break
                elif cmdarg == hotwire.script.REDIR_IN:
                    if not tokens:
                        raise PipelineParseException(_('Must specify target for input redirection'))
                    in_redir_token = pull_token()
                    in_redir = in_redir_token.text
                    alltokens.append(in_redir_token)
                elif cmdarg == hotwire.script.REDIR_OUT:
                    if not tokens:
                        raise PipelineParseException(_('Must specify target for output redirection'))                   
                    out_redir_token = pull_token()
                    out_redir = out_redir_token.text                     
                    alltokens.append(out_redir_token)               
                else:
                    alltokens.append(cmdarg)
                    cmdargs.append(cmdarg)         

            builtin_opts = b.options

            options = []
            expanded_cmdargs = []
            options_ended = False
            raise_on_invalid_options = not (b.options_passthrough or accept_partial)
            _logger.debug("raise: %r valid options %r, argument/option pool: %r", raise_on_invalid_options,
                          builtin_opts, cmdargs)
            for token in cmdargs:
                arg = CommandArgument(token.text, quoted=token.quoted)
                if token.text == u'--':
                    options_ended = True
                elif options_ended:
                    expanded_cmdargs.append(arg)
                else:      
                    argopts = Pipeline.__parse_option_or_arg(builtin_opts, token.text, 
                                                             raise_on_invalid=raise_on_invalid_options)
                    if argopts:
                        options.extend(argopts)
                    else:
                        expanded_cmdargs.append(arg)
                        
            argspec = b.argspec
            if argspec is False or accept_partial:
                # If we don't have an argspec, don't do any checking
                pass
            elif argspec is None:
                if len(expanded_cmdargs) > 0:
                    raise PipelineParseException(_('Command %s takes no arguments, %d given') % (b.name, len(expanded_cmdargs)))
            elif isinstance(argspec, MultiArgSpec):
                if len(expanded_cmdargs) < argspec.min:
                    raise PipelineParseException(_("Command %s requires %d args, %d given") % (b.name,
                                                                                               argspec.min,
                                                                                               len(expanded_cmdargs)))  
            elif isinstance(argspec, tuple):
                mincount = 0
                for o in argspec:
                    if not o.opt: 
                        mincount += 1
                if len(expanded_cmdargs) > len(argspec):
                    raise PipelineParseException(_("Command %s takes at most %d args, %d given") % (b.name,
                                                                                                    len(argspec),
                                                                                                    len(expanded_cmdargs)))
                if len(expanded_cmdargs) < mincount:
                    raise PipelineParseException(_("Command %s requires %d args, %d given") % (b.name,
                                                                                               mincount,
                                                                                               len(expanded_cmdargs)))                
            cmdtokens = [builtin_token]
            cmdtokens.extend(cmdargs)
            cmd = Command(b, expanded_cmdargs, options, context, tokens=alltokens, in_redir=in_redir, out_redir=out_redir)
            components.append(cmd)
            if (not in_redir) and prev:
                cmd.set_input(prev.output)
            if pipeline_output_type:
                cmd.set_input_type(pipeline_output_type)
            input_accepts_type = cmd.builtin.input_type
            input_optional = cmd.builtin.input_is_optional
            if pipeline_input_optional == 'unknown':
                pipeline_input_optional = input_optional
            _logger.debug("Validating input %s vs prev %s", input_accepts_type, pipeline_output_type)

            if prev and not pipeline_output_type:
                raise PipelineParseException(_("Command %s yields no output for pipe") % \
                                             (prev.builtin.name))
            if (not prev) and input_accepts_type and not (input_optional): 
                raise PipelineParseException(_("Command %s requires input of type %s") % \
                                             (cmd.builtin.name, input_accepts_type))
            if input_accepts_type and prev \
                   and not Pipeline.streamtype_is_assignable(pipeline_output_type, input_accepts_type, input_optional):
                raise PipelineParseException(_("Command %s yields '%s' but %s accepts '%s'") % \
                                             (prev.builtin.name, pipeline_output_type, cmd.builtin.name, input_accepts_type))
            if (not input_optional) and (not input_accepts_type) and pipeline_output_type:
                raise PipelineParseException(_("Command %s takes no input but type '%s' given") % \
                                             (cmd.builtin.name, pipeline_output_type))
            locality = cmd.builtin.locality
            if prev_locality and locality and (locality != prev_locality):
                raise PipelineParseException(_("Command %s locality conflict with '%s'") % \
                                             (cmd.builtin.name, prev.builtin.name))
            prev_locality = locality
                
            if out_redir:
                prev = None
            else:
                prev = cmd
            if pipeline_input_type == 'unknown':
                pipeline_input_type = input_accepts_type
                
            if pipeline_singlevalue is None or (pipeline_singlevalue):
                pipeline_singlevalue = cmd.builtin.singlevalue

            if cmd.builtin.output_type != 'identity':
                if context and cmd.builtin.output_typefunc:
                    pipeline_output_type = cmd.builtin.output_typefunc(context)
                    _logger.debug("retrieved type %r from typefunc", pipeline_output_type)
                else:
                    pipeline_output_type = cmd.builtin.output_type

            if undoable is None:
                undoable = cmd.builtin.undoable
            elif not cmd.builtin.undoable:
                undoable = False

            if not cmd.builtin.idempotent:
                idempotent = False
                
        if len(components) == 0:
            raise PipelineParseException(_("Empty pipeline"))

        if undoable is None:
            undoable = False
        pipeline = Pipeline(components,
                            input_type=pipeline_input_type,
                            input_optional=pipeline_input_optional,
                            output_type=pipeline_output_type,
                            locality=prev_locality,
                            undoable=undoable,
                            idempotent=idempotent,
                            singlevalue=pipeline_singlevalue)
        _logger.debug("Parsed pipeline %s (%d components, input %s, output %s)",
                      pipeline, len(components),
                      pipeline.get_input_type(),
                      pipeline.get_output_type())
        return pipeline 

    @staticmethod
    def parse(text, context=None, resolver=None, accept_partial=False):
        tokens = list(Pipeline.tokenize(text, context, accept_partial=accept_partial))
        return Pipeline.create(context, resolver, accept_partial=accept_partial, *tokens)
    
    def __iter__(self):
        for component in self.__components:
            yield component
        
    def __getitem__(self, i):
        return self.__components[i]

    def __str__(self):
        return string.join(map(lambda x: x.__str__(), self.__components), ' | ')         class Command(object):
    """Represents a complete executable object in a pipeline."""

    def __init__(self, builtin, args, options, hotwire, tokens=None, in_redir=None, out_redir=None, 
                  out_append=False):
        super(Command, self).__init__()
        self.builtin = builtin
        self.context = CommandContext(hotwire)
        # The concept of multiple object streams is dead. 
        #for schema in self.builtin.get_aux_outputs():
        #    self.context.attach_auxstream(CommandAuxStream(self, schema))
        if self.builtin.hasmeta:
            def on_meta(*args):
                dispatcher.send('metadata', self, *args)
            self.context.set_metadata_handler(on_meta)
        self.input = None
        self.output = CommandQueue()
        self.map_fn = lambda x: x
        self.args = args
        self.context.options = options
        self.in_redir = in_redir and FilePath(os.path.expanduser(in_redir), self.context.cwd)
        self.out_redir = out_redir and FilePath(os.path.expanduser(out_redir), self.context.cwd)
        self.out_append = out_append
        
        self.__thread = None
        self.__executing_sync = None
        self._cancelled = False
        self.__tokens = tokens

    def set_pipeline(self, pipeline):
        self.context.set_pipeline(pipeline)

    def set_input(self, input, is_first=False):
        self.input = input       
        self.context.input = self.input
        self.context.input_is_first = is_first
        
    def set_input_type(self, in_type):
        """Note the pipeline object type used for input."""
        self.context.input_type = in_type
        
    def disconnect(self):
        self.context = None
        
    def cancel(self):
        if self._cancelled:
            return
        self._cancelled = True
        self.context.cancelled = True
        if self.context.input:
            self.context.input.cancel()
        self.builtin.cancel(self.context)

    def get_input_opt_formats(self):
        return self.builtin.input_opt_formats

    def get_output_opt_formats(self):
        return self.builtin.output_opt_formats

    def execute(self, force_sync, **kwargs):
        if force_sync or not self.builtin.threaded:
            _logger.debug("executing sync: %s", self)
            self.__executing_sync = True
            self.__run(**kwargs)
        else:         
            _logger.debug("executing async: %s", self)              
            self.__executing_sync = False             
            self.__thread = threading.Thread(target=self.__run)
            self.__thread.setDaemon(True)            
            self.__thread.start()

    def set_output_queue(self, queue, map_fn):
        self.output = queue
        self.map_fn = map_fn

    def get_auxstreams(self):
        for obj in self.context.get_auxstreams():
            yield obj
    
    def get_tokens(self):
        return self.__tokens

    def __run(self, *args, **kwargs):
        if self._cancelled:
            _logger.debug("%s cancelled, returning", self)
            self.output.put(self.map_fn(None))
            return
        try:
            matched_files = []
            oldlen = 0
            for globarg_in in self.args:
                if isinstance(globarg_in, CommandArgument) and globarg_in.isquoted:
                    globarg = globarg_in
                    newlen = oldlen                    
                else:
                    globarg = os.path.expanduser(globarg_in)
                    matched_files.extend(hotwire.fs.dirglob(self.context.cwd, globarg))
                    _logger.debug("glob on %s matched is: %s", globarg_in, matched_files) 
                    newlen = len(matched_files)
                if oldlen == newlen:
                    matched_files.append(globarg)
                    newlen += 1
                oldlen = newlen
            target_args = [matched_files]
            _logger.info("Execute '%s' args: %s options: %s", self.builtin, target_args, self.context.options)
            kwargs = {}
            if self.context.options and not self.builtin.flattened_args:
                kwargs['options'] = self.context.options
            if self.input is not None and self.input.opt_type and not self.in_redir:
                kwargs['in_opt_format'] = self.input.opt_type                
            if self.output.opt_type and not self.out_redir:
                kwargs['out_opt_format'] = self.output.opt_type
            if self.in_redir:
                _logger.debug("input redirected, opening %s", self.in_redir)
                self.context.input = CommandFileQueue(open_text_file(self.in_redir, 'r'))
            if self.out_redir:
                _logger.debug("output redirected, opening %s", self.out_redir)
                outfile = open_text_file(self.out_redir, self.out_append and 'a+' or 'w')
            else:
                outfile = None
            try:
                exectarget = self.builtin.execfunc
                if self.builtin.flattened_args:
                    target_args = target_args[0]
                execresult = exectarget(self.context, *target_args, **kwargs)                
                if self.builtin.singlevalue:
                    if outfile:
                        outfile.write(unicode(execresult))
                    else:
                        self.output.put(execresult)
                else:
                    for result in execresult:
                        # if it has status, let it do its own cleanup
                        if self._cancelled and not self.builtin.hasstatus:
                            _logger.debug("%s cancelled, returning", self)
                            self.output.put(self.map_fn(None))
                            dispatcher.send('complete', self)
                            return
                        if outfile and (result is not None):
                            result = unicode(result)
                            outfile.write(result)
                        else:                        
                            self.output.put(self.map_fn(result))
            finally:
                if outfile:
                    outfile.close()
                self.builtin.cleanup(self.context)
        except Exception, e:
            _logger.debug("Caught exception from command: %s", e, exc_info=True)
            if self.__executing_sync:
                raise
            else:
                dispatcher.send('exception', self, e)
        self.output.put(self.map_fn(None))
        dispatcher.send('complete', self)
        
    def get_executing_sync(self):
        return self.__executing_sync      

    def __str__(self):
        def unijoin(args):
            return ' '.join(map(unicode, args))
        args = [self.builtin.name]
        args.extend(self.context.options)
        for cmdarg in self.args:
            if isinstance(cmdarg, CommandArgument) and cmdarg.isquoted:
                args.append(quote_arg(cmdarg))                
            else:
                args.append(cmdarg)
        if self.in_redir:
            args.extend(['<', self.in_redir])
        if self.out_redir:
            args.extend(['>', self.out_redir])            
        return unijoin(args) class HotwireContext(object):
    cwd = property(lambda self: self.get_cwd(), doc="""Current working directory for this context.""")
    
    """The interface to manipulating a Hotwire execution shell.  Item
    such as the current working diretory may be changed via this class,
    and subclasses define further extended commands."""
    def __init__(self, initcwd=None):
        super(HotwireContext, self).__init__()
        self.chdir(initcwd or os.path.expanduser('~'))
        _logger.debug("Context created, dir=%s" % (self.get_cwd(),))

    def chdir(self, dpath):
        if not isinstance(dpath, unicode):
            dpath = unicode(dpath, 'utf-8')
        dpath = os.path.expanduser(dpath)
        newcwd = os.path.isabs(dpath) and dpath or posixpath.join(self.__cwd, dpath)
        newcwd = path_normalize(newcwd)
        _logger.debug("chdir: %s    post-normalize: %s", dpath, newcwd)
        os.stat(newcwd) # lose on nonexistent
        self.__cwd = newcwd
        dispatcher.send('cwd', self, newcwd)
        return self.__cwd

    def get_cwd(self):
        return self.__cwd
    
    def get_gtk_event_time(self):
        return 0

    def info_msg(self, msg):
        _logger.info("msg: %s", msg)
    
    def get_current_output_metadata(self):
        raise NotImplementedError()
    
    def get_current_output_ref(self):
        raise NotImplementedError()
    
    def snapshot_output(self, ref):
        raise NotImplementedError()
    
    def snapshot_selected_output(self, ref):
        raise NotImplementedError() class PipelineLanguageRegistry(Singleton):
    """Registry for supported pipeline languages."""
    def __init__(self):
        self.__langs = {} # uuid->lang
        
    def __getitem__(self, uuid):
        return self.__langs[uuid]
        
    def get_by_fileext(self, ext):
        for lang in self:
            if lang.fileext == ext:
                return lang
        
    def __iter__(self):
        for x in self.__langs.itervalues():
            yield x
            
    def iter_sorted(self):
        langs = list(self)
        # Append Hotwire and Python in order
        result = [self['62270c40-a94a-44dd-aaa0-689f882acf34'], self['da3343a0-8bce-46ed-a463-2d17ab09d9b4']]
        for lang in sorted(langs, lambda a,b: locale.strcoll(a.langname, b.langname)):
            if lang.builtin_eval is not None:
                continue
            result.append(lang)
        for lang in result:
            yield lang               

    def register(self, lang):
        if lang.uuid in self.__langs:
            raise ValueError("Language uuid %s already registered", lang.uuid)
        self.__langs[lang.uuid] = lang
        dispatcher.send(sender=self) class BaseCommandResolver(object):
    """Expands command names.  Example: ifconfig => sys ifconfig"""
    def __init__(self):
        super(BaseCommandResolver, self).__init__()
        # TODO move this logic elsewhere somehow better; maybe stick in hotwire_ui/pipeline.py.
        from hotwire.completion import VerbCompleter
        self.__verb_completer = VerbCompleter()
        
    def resolve(self, text, context):
        resolutions = []
        vc = self.__verb_completer
            
        resolution_match = None
        for completion in vc.completions(text, context.get_cwd()):
            target = completion.target
            if self._resolve_verb_completion(text, completion):
                resolution_match = completion
                break
        _logger.debug("resolution match is %s", resolution_match)                
        if resolution_match:            
            return self._expand_verb_completion(resolution_match)
        return (None, None)
    
    def _resolve_verb_completion(self, text, completion):
        fs = Filesystem.getInstance()
        target = completion.target
        if not (isinstance(target, File) and not target.is_directory):
            return False
        # Determine whether this input matches an executable file
        return fs.path_executable_match(text, target.path)
    
    def _expand_verb_completion(self, completion):
        target = completion.target
        if isinstance(target, File):
            return (BuiltinRegistry.getInstance()['sys'], [target.path])        class Completion(object):
    """Represents a match of a string by some text input."""
    __slots__ = ['suffix', 'target', 'matchbase', 'icon']
    def __init__(self, 
                  suffix,
                  target,
                  matchbase,
                  icon=None):
        self.suffix = suffix
        self.matchbase = matchbase
        self.target = target
        self.icon = icon

    def __cmp__(self, other):
        return cmp(self.suffix,other.suffix)
    
    def __repr__(self):
        return "<Completion s:%r>" % (self.suffix,) class VerbCompleter(Completer):
    def __init__(self):
        super(VerbCompleter, self).__init__()

    def completions(self, text, cwd, context=None):
        bc = BuiltinCompleter()
        for completion in bc.completions(text, cwd, context=context):
            yield completion
        aliases = AliasRegistry.getInstance()
        for alias in aliases:
            compl = self._match(alias.name, text, alias)
            if compl: yield compl
        textpath = FilePath(text, cwd)
        expanded_textpath = path_expanduser(textpath)
        (text_dpath, text_prefix) = os.path.split(expanded_textpath)             
        if text.find('/') >= 0 or text.startswith('.' + os.sep):
            pc = PathCompleter()
            for completion in pc.completions(text, cwd):
                fobj = completion.target
                if fobj.is_directory or fobj.is_executable:
                    yield completion
        else:
            fs = Filesystem.getInstance()           
            for dpath in fs.get_path_generator():
                if not os.access(dpath, os.X_OK):
                    continue
                for fpath in iterd_sorted(dpath):
                    fname = unix_basename(fpath)
                    if not fname.startswith(text_prefix):
                        continue
                    fobj = fs.get_file_sync(fpath)
                    if fobj.is_executable:
                        yield _mkfile_completion(text, fpath, fobj) class TokenCompleter(Completer):
    def __init__(self):
        super(TokenCompleter, self).__init__()

    def completions(self, text, cwd):
        pathcompleter = PathCompleter()
        for completion in pathcompleter.completions(text, cwd):
            yield completion class PixbufCache(Singleton):
    def __init__(self):
        super(PixbufCache, self).__init__()
        self.__cache = {}

    def get(self, path, size=24, animation=False, trystock=False, stocksize=None):
        if trystock:
            pixbuf = self.get_stock(path, size)
            if pixbuf:
                return pixbuf
        if not os.path.isabs(path):
            path = _find_in_datadir(path)
        if not path:
            if path == gtk.STOCK_MISSING_IMAGE:
                return None
            return self.get_stock(gtk.STOCK_MISSING_IMAGE)        
        if not self.__cache.has_key((path, size)):
            pixbuf = self.__do_load(path, size, animation)
            self.__cache[(path, size)] = pixbuf
        return self.__cache[(path, size)]
    
    def get_stock(self, name, size=24):
        if name.find(os.sep) >= 0:
            name = os.path.basename(name)
        (root, ext) = os.path.splitext(name)
        theme = gtk.icon_theme_get_default()
        try:
            return theme.load_icon(name, size, 0)
        except gobject.GError, e:
            return None
        
    def __do_load(self, path, size, animation):
        f = open(path, 'rb')
        data = f.read()
        f.close()
        loader = gtk.gdk.PixbufLoader()
        if size:
            loader.set_size(size, size)
        loader.write(data)
        loader.close()
        if animation:
            return loader.get_animation()
        return loader.get_pixbuf() class Singleton(object):
    __metaclass__ = MetaSingleton
    
    def getInstance(cls, *lstArgs):
        """
        Call this to instantiate an instance or retrieve the existing instance.
        If the singleton requires args to be instantiated, include them the first
        time you call getInstance.        
        """
        if cls._isInstantiated():
            if len(lstArgs) != 0:
                raise SingletonException, 'If no supplied args, singleton must already be instantiated, or __init__ must require no args'
        else:
            if cls._getConstructionArgCountNotCountingSelf() > 0 and len(lstArgs) <= 0:
                raise SingletonException, 'If the singleton requires __init__ args, supply them on first instantiation'
            instance = cls.__new__(cls)
            instance.__init__(*lstArgs)
            cls.cInstance = instance
        return cls.cInstance
    getInstance = classmethod(getInstance)
    
    def _isInstantiated(cls):
        return hasattr(cls, 'cInstance')
    _isInstantiated = classmethod(_isInstantiated)  

    def _getConstructionArgCountNotCountingSelf(cls):
        return cls.__init__.im_func.func_code.co_argcount - 1
    _getConstructionArgCountNotCountingSelf = classmethod(_getConstructionArgCountNotCountingSelf)

    def _forgetClassInstanceReferenceForTesting(cls):
        """
        This is designed for convenience in testing -- sometimes you 
        want to get rid of a singleton during test code to see what
        happens when you call getInstance() under a new situation.
        
        To really delete the object, all external references to it
        also need to be deleted.
        """
        try:
            delattr(cls,'cInstance')
        except AttributeError:
            # run up the chain of base classes until we find the one that has the instance
            # and then delete it there
            for baseClass in cls.__bases__: 
                if issubclass(baseClass, Singleton):
                    baseClass._forgetClassInstanceReferenceForTesting()
    _forgetClassInstanceReferenceForTesting = classmethod(_forgetClassInstanceReferenceForTesting) class BuiltinRegistry(Singleton):
    """Manages the set of registered builtins.
Currently there are 3 possible categories of builtin:
 "system" - These builtins are part of extensions shipped with the operating system.
 "hotwire" - Builtins included with the Hotwire source distribution
 "user" - Custom builtins loaded from per-user configuration.
"""    
    
    system_set = property(lambda self: self.__system_builtins, doc="""Set of system builtins""")
    hotwire_set = property(lambda self: self.__hotwire_builtins, doc="""Set of system builtins""")
    user_set = property(lambda self: self.__user_builtins, doc="""Set of user builtins""")

    def __init__(self):
        self.__system_builtins = set()        
        self.__hotwire_builtins = set()
        self.__user_builtins = set()
        self.__sets = [self.__user_builtins, self.__hotwire_builtins, self.__system_builtins]

    def __lookup_builtin(self, bset, name):
        for b in bset:
            if b.name == name or name in b.aliases:
                return b
        return None
    
    def __getitem__(self, name):
        for bset in self.__sets:
            b = self.__lookup_builtin(bset, name)
            if b: 
                return b
        raise KeyError(name)
        
    def __iter__(self):
        for bset in self.__sets:
            for b in bset:
                yield b   

    def __register(self, bset, builtin):
        existing = None
        for b in bset:
            if b.name == builtin.name:
                existing = b
                break
            b_aliases = set(b.aliases)
            builtin_aliases = set(builtin.aliases)
            inter = b_aliases.intersection(builtin_aliases)
            if len(inter) > 0:
                existing = b
                break
        if existing is not None:
            _logger.debug("deregistering existing instance %r", existing)
            bset.remove(existing)
        _logger.debug("registering %r (set: %r)", builtin, bset)
        bset.add(builtin)

    def register_system(self, builtin):
        self.__register(self.__system_builtins, builtin)
        
    def register_hotwire(self, builtin):
        self.__register(self.__hotwire_builtins, builtin)
        
    def register_user(self, builtin):
        self.__register(self.__user_builtins, builtin)                      def markup_for_match(text, start, end, matchtarget=None):
    source = matchtarget or text
    return  '%s<b>%s</b>%s%s' % (escape_xml(source[0:start]),
                                 escape_xml(source[start:end]),
                                 escape_xml(source[end:]),
                                 matchtarget and (' - <i>' + text + '</i>') or '') def quote_arg(arg):
    """Quote arg for processing by a shell.
    If arg would pass through unquoted, return unmodified arg."""
    if not isinstance(arg, unicode):
        arg = unicode(arg, 'utf-8')
    safechars = '.,/~_-+:'
    safeonly = True
    safe_space_only = True
    def is_letter_or_number(c):
        return is_category_letter(c) or is_category_number(c)
    for c in arg:
        cat = get_unichar_category(c)
        if (not is_letter_or_number(cat)) and c not in safechars:
            safeonly = False
            if c != ' ':
                safe_space_only = False
    if safeonly:
        return arg
    if safe_space_only:
        return arg.replace(' ', '\\ ')
    return quote_shell_arg(arg) def dirglob(dir, pat):
def unix_basename(path):
def path_unabs(path, base):
def path_fromurl(url):
def path_tourl(path):
def win32_normpath(path):
def win32_fast_normpath(path):
def win32_expanduser(path):
def win32_pathjoin(*args):
def win32_abspath(path):
def path_unexpanduser(path):
def copy_file_or_dir(src, dest, dest_is_dir):
def open_text_file(path, mode='r', buffering=None):
def file_is_valid_utf8(path):
    def __new__(cls, value, dir=None):
    def path_join(self, path):
def iterd(dpath, fpath=False):
def iterd_sorted(dpath, **kwargs):
def atomic_rename(oldp, newp):
class FilePath(unicode): class File(object):
    """An extended crossplatform stat() container, essentially.  
    Extra data retrieved includes symbolic link target (if applicable) and icon."""
    
    path = property(lambda self: self._path, doc="""Complete path to file, expressed in Hotwire notation (always forward slashes)""")
    uri = property(lambda self: self._uri, doc="""URI notation for file""")
    basename = property(lambda self: self._basename, doc="""Name of file (without directory component)""")
    size = property(lambda self: self._get_size(), doc="""Size in bytes of file, or None if unknown""")
    hidden = property(lambda self: self._hidden, doc="""Whether or not this file is normally visible in directory listings""")
    icon = property(lambda self: self._icon, doc="""Icon name (internal Hotwire/GTK+ representation)""")
    is_directory = property(lambda self: self.test_directory(), doc="""Whether or not this object represents a directory""")
    is_executable = property(lambda self: self._is_executable(), doc="""Whether or not this object represents an OS-executable file""")
    is_link = property(lambda self: self._is_link(), doc="""Whether or not this object represents a symbolic link""")
    file_type_char = property(lambda self:self._get_file_type_char(), doc="""Unix-style file type character ('d' for directory, etc.)""")
    stat_mode = property(lambda self: self._get_stat_mode(), doc="""Unix-style access mode""")
    permissions_string = property(lambda self: self._get_permissions_string(), doc="""Unix-style compact permissions string""")
    mtime = property(lambda self: self._get_mtime(), doc="""Modification time, in seconds since the epoch""")
    mimetype = property(lambda self: self._get_mime(), doc="""MIME type""")

    __slots__ = ['fs', 'stat', 'xaccess', 'icon_error', '_permstring', 'target_stat', 'stat_error']
    def __init__(self, path, fs=None):
        super(File, self).__init__()
        if not isinstance(path, unicode):
            path = unicode(path, 'utf-8')
        self._path = path
        self._uri = 'file://' + urllib.pathname2url(path.encode(sys.getfilesystemencoding()))
        self._basename = unix_basename(path)
        self.fs = fs
        self.stat = None
        self.xaccess = None
        self._hidden = None
        self._icon = None
        self.icon_error = False
        self._permstring = None
        self.target_stat = None
        self.stat_error = None
        
    def __cmp__(self, o):
        if isinstance(o, File):
            return cmp(self.path, o.path)
        return cmp(self.path, o)

    def test_directory(self, follow_link=True):
        if not self.stat:
            return False
        if follow_link and stat.S_ISLNK(self.stat[stat.ST_MODE]):
            stbuf = self.target_stat
        else:
            stbuf = self.stat
        return stbuf and stat.S_ISDIR(stbuf[stat.ST_MODE])
    
    def _is_link(self):
        return self.stat and stat.S_ISLNK(self.stat[stat.ST_MODE])
    
    def _is_executable(self):
        return self.xaccess

    def _get_size(self):
        if self.stat and stat.S_ISREG(self.stat[stat.ST_MODE]):
            return self.stat[stat.ST_SIZE]
        return None

    def _get_mtime(self):
        if self.stat:
            return self.stat[stat.ST_MTIME]
        return None
    
    def _get_file_type_char(self):
        if self.is_directory:
            return 'd'
        return '-'
    
    def _get_stat_mode(self):
        return self.stat[stat.ST_MODE]

    def _get_permissions_string(self):
        if self._permstring:
            return self._permstring
        
        perms = self.stat_mode
        if not perms:
            return
        buf = StringIO()
                
        buf.write(self.file_type_char)
        
        if perms & stat.S_ISUID: buf.write('s')
        elif perms &stat.S_IRUSR: buf.write('r')
        else: buf.write('-')
        if perms & stat.S_IWUSR: buf.write('w')
        else: buf.write('-')
        if perms & stat.S_IXUSR: buf.write('x')
        else: buf.write('-')
        
        if perms & stat.S_ISGID: buf.write('s')
        elif perms & stat.S_IRGRP:buf.write('r')
        else: buf.write('-')
        if perms & stat.S_IWGRP: buf.write('w')
        else: buf.write('-')
        if perms & stat.S_IXGRP: buf.write('x')
        else: buf.write('-')
        
        if perms & stat.S_IROTH: buf.write('r')
        else: buf.write('-')
        if perms & stat.S_IWOTH: buf.write('w')
        else: buf.write('-')
        if perms & stat.S_IXOTH: buf.write('x')
        else: buf.write('-')
        
        self._permstring = buf.getvalue()
        return self._permstring
    
    def _get_mime(self):
        return None

    def get_stat(self):
        self._get_stat_async()

    def _get_stat_async(self):
        MiniThreadPool.getInstance().run(self.__get_stat_signal)
        
    def get_stat_sync(self):
        self._do_get_stat(rethrow=True)
        self._do_get_xaccess()
        self._do_get_hidden()
        self._do_get_icon()

    def _do_get_stat(self, rethrow=False):
        try:
            self.stat = hasattr(os, 'lstat') and os.lstat(self.path) or os.stat(self.path)
            if stat.S_ISLNK(self.stat[stat.ST_MODE]):
                try:
                    self.target_stat = os.stat(self.path)
                except OSError, e:
                    self.target_stat = None		
        except OSError, e:
            _logger.debug("Failed to stat '%s': %s", self.path, e)
            self.stat_error = str(e)
            if rethrow:
                raise FileStatError(e)
            
    def _do_get_xaccess(self):
        self.xaccess = os.access(self.path, os.X_OK)
        
    def _do_get_hidden(self):
        pass 
        
    def _do_get_icon(self):
        if not self.stat:
            self._icon = 'gtk-dialog-error'
        elif self.is_directory:
            self._icon = 'gtk-directory'
        else:
            self._icon = 'gtk-file'

    @log_except(_logger)
    def __get_stat_signal(self):
        self.get_stat_sync()
        call_idle(self.__idle_emit_changed, priority=gobject.PRIORITY_LOW)        
        
    @log_except(_logger)
    def __idle_emit_changed(self):
        responses = dispatcher.send(sender=self)
        _logger.debug("idle changed dispatch from %r, responses=%r", self, responses) class Filesystem(object):
    @staticmethod
    def getInstance():
        global _instance
        if _instance is None:
            _instance = _module.getInstance()
        return _instance class CommandExecutionDisplay(gtk.VBox):
    def __init__(self, context, pipeline, odisp):
        super(CommandExecutionDisplay, self).__init__()
        self.odisp = odisp
        self.cmd_header = CommandExecutionHeader(context, pipeline, odisp, overview_mode=False)
        self.pack_start(self.cmd_header, expand=False)
        self.pack_start(odisp, expand=True)
        
    def cancel(self):
        self.odisp.cancel()
        self.cmd_header.get_pipeline().cancel()
        
    def undo(self):
        self.cmd_header.get_pipeline().undo()         class CommandExecutionControl(gtk.VBox):
    # This may be a sucky policy, but it's less sucky than what came before.
    COMPLETE_CMD_EXPIRATION_SECS = 5 * 60
    __gsignals__ = {
        "new-window" : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, (gobject.TYPE_PYOBJECT,)),
    }
    __gproperties__ = { 
                       'pipeline-count' : (gobject.TYPE_INT, '', '',
                       0, 4096, 0, gobject.PARAM_READWRITE),
                       'executing-pipeline-count' : (gobject.TYPE_INT, '', '',
                       0, 4096, 0, gobject.PARAM_READWRITE),                       
                       'unseen-pipeline-count' : (gobject.TYPE_INT, '', '',
                       0, 4096, 0, gobject.PARAM_READWRITE)                       
    }
    
    def __init__(self, context):
        super(CommandExecutionControl, self).__init__()

        self.__ui_string = """
<ui>
  <menubar name='Menubar'>
    <menu action='FileMenu'>
      <placeholder name='FileDetachAdditions'>
        <menuitem action='DetachPipeline'/>
      </placeholder>
    </menu>
    <menu action='EditMenu'>
      <placeholder name='EditMenuAdditions'>
        <menuitem action='Copy'/>
        <separator/>
        <menuitem action='Search'/>
        <menuitem action='Input'/> 
      </placeholder>
    </menu>
    <menu action='ViewMenu'>
      <menuitem action='Overview'/>
      <separator/>
      <menuitem action='Inspector'/>
      <separator/>
      <placeholder name='ViewMenuAdditions'/>
      <menuitem action='PreviousCommand'/>
      <menuitem action='NextCommand'/>
      <separator/>
      <menuitem action='PreviousUnseenCommand'/>
      <menuitem action='LastCommand'/>
    </menu>
    <placeholder name='WidgetMenuAdditions'>
      <menu action='ControlMenu'>
        <menuitem action='Cancel'/>
        <menuitem action='Undo'/>
        <separator/>
        <menuitem action='RemovePipeline'/>
        <menuitem action='UndoRemovePipeline'/>                     
      </menu>
    </placeholder>          
  </menubar>
  <accelerator action='ScrollHome'/>
  <accelerator action='ScrollEnd'/>
  <accelerator action='ScrollPgUp'/>
  <accelerator action='ScrollPgDown'/>  
</ui>"""         
        self.__actions = [
            ('DetachPipeline', gtk.STOCK_JUMP_TO, _('Detach _Pipeline'), '<control><shift>N', _('Create window from output'), self.__to_window_cb),                          
            ('Copy', gtk.STOCK_COPY, _('_Copy'), '<control>c', _('Copy output'), self.__copy_cb),                          
            ('Cancel', gtk.STOCK_CANCEL, _('_Cancel'), '<control><shift>c', _('Cancel current command'), self.__cancel_cb),
            ('Undo', gtk.STOCK_UNDO, _('_Undo'), None, _('Undo current command'), self.__undo_cb),            
            ('Search', gtk.STOCK_FIND, _('_Search'), '<control>s', _('Search output'), self.__search_cb),
            ('Input', gtk.STOCK_EDIT, _('_Input'), '<control>i', _('Send input'), self.__input_cb),
            ('ScrollHome', None, _('Output _Top'), '<control>Home', _('Scroll to output top'), self.__view_home_cb),
            ('ScrollEnd', None, _('Output _Bottom'), '<control>End', _('Scroll to output bottom'), self.__view_end_cb), 
            ('ScrollPgUp', None, _('Output Page _Up'), 'Page_Up', _('Scroll output up'), self.__view_up_cb),
            ('ScrollPgDown', None, _('Output Page _Down'), 'Page_Down', _('Scroll output down'), self.__view_down_cb),
            ('ControlMenu', None, _('_Control')),
            ('RemovePipeline', gtk.STOCK_REMOVE, _('_Remove Pipeline'), '<control><shift>K', _('Remove current pipeline view'), self.__remove_pipeline_cb),
            ('UndoRemovePipeline', gtk.STOCK_UNDO, _('U_ndo Remove Pipeline'), '<control><shift>J', _('Undo removal of current pipeline view'), self.__undo_remove_pipeline_cb),            
            ('PreviousCommand', gtk.STOCK_GO_UP, _('_Previous'), '<control>Up', _('View previous command'), self.__view_previous_cb),
            ('NextCommand', gtk.STOCK_GO_DOWN, _('_Next'), '<control>Down', _('View next command'), self.__view_next_cb),
            ('PreviousUnseenCommand', gtk.STOCK_GO_UP, _('Previous _Unseen'), '<control><shift>Up', _('View most recent unseen command'), self.__view_previous_unseen_cb),
            ('LastCommand', gtk.STOCK_GOTO_BOTTOM, _('Last'), '<control><shift>Down', _('View most recent command'), self.__view_last_cb),            
        ]
        self.__toggle_actions = [
            ('Overview', None, _('_Overview'), '<control><shift>o', _('Toggle overview'), self.__overview_cb),
            ('Inspector', None, _('_Inspector'), '<control><shift>I', _('Toggle inspector'), self.__inspector_cb),             
        ]
        self.__action_group = gtk.ActionGroup('HotwireActions')
        self.__action_group.add_actions(self.__actions) 
        self.__action_group.add_toggle_actions(self.__toggle_actions)
        self.__action_group.get_action('Overview').set_active(False)       
        self.__context = context
        
        # Holds a reference to the signal handler id for the "changed" signal on the current odisp
        # so we know when to reload any metadata
        self.__odisp_changed_connection = None
        
        self.__header = gtk.HBox()    
        def create_arrow_button(action_name):
            action = self.__action_group.get_action(action_name)
            icon = action.create_icon(gtk.ICON_SIZE_MENU)
            button = gtk.Button(label='x')
            button.connect('clicked', lambda *args: action.activate())
            action.connect("notify::sensitive", lambda *args: button.set_sensitive(action.get_sensitive()))
            button.set_property('image', icon)
            button.set_focus_on_click(False)            
            return button
        self.__header_label = create_arrow_button('PreviousCommand')
        self.__header.pack_start(self.__header_label, expand=False)
        self.__header_exec_label = gtk.Label()
        self.__header.pack_start(self.__header_exec_label, expand=False)
        self.pack_start(self.__header, expand=False)
        self.__cmd_paned = gtk.HPaned()
        self.pack_start(self.__cmd_paned, expand=True)
        self.__cmd_notebook = gtk.Notebook()
        self.__cmd_paned.pack1(self.__cmd_notebook, resize=True)
        self.__cmd_notebook.connect('switch-page', self.__on_page_switch)
        self.__cmd_notebook.set_show_tabs(False)
        self.__cmd_notebook.set_show_border(False)
        self.__inspector = ClassInspectorSidebar()
        self.__cmd_paned.pack2(self.__inspector, resize=False)
        self.__cmd_overview = CommandExecutionHistory(self.__context)
        self.__cmd_overview.show_all()
        self.__cmd_overview.set_no_show_all(True)
        self.__cmd_overview.connect('show-command', self.__on_show_command)
        self.__cmd_overview.connect('command-action', self.__handle_cmd_overview_action) 
        self.pack_start(self.__cmd_overview, expand=True)
        self.__footer = gtk.HBox()    
        self.__footer_label = create_arrow_button('NextCommand')
        self.__footer.pack_start(self.__footer_label, expand=False)
        self.__footer_exec_label = gtk.Label()
        self.__footer.pack_start(self.__footer_exec_label, expand=False)
        self.pack_start(self.__footer, expand=False)        

        self.__complete_unseen_pipelines = set()
        self.__history_visible = False
        self.__inspector_visible = False
        self.__prevcmd_count = 0
        self.__prevcmd_executing_count = 0
        self.__nextcmd_count = 0
        self.__nextcmd_executing_count = 0
        self.__idle_command_gc_id = 0
        
        self.__actively_destroyed_pipeline_box = []

        self.__sync_visible()
        self.__sync_cmd_sensitivity()
        
    def get_ui(self):
        return (self.__ui_string, self.__action_group, None)
    
    def __get_complete_commands(self):
        for child in self.__iter_cmds():
            if child.get_state() != 'executing':
                yield child
            
    def __iter_cmds(self):
        for child in self.__cmd_notebook.get_children():
            yield child.cmd_header            
     
    def add_cmd_widget(self, cmd):
        pipeline = cmd.cmd_header.get_pipeline()
        dispatcher.connect(self.__on_pipeline_state_change, 'state-changed', pipeline)
        self.__cmd_overview.add_pipeline(pipeline, cmd.odisp)
        pgnum = self.__cmd_notebook.append_page(cmd)
        self.__cmd_notebook.set_current_page(pgnum)
        self.__sync_visible()
        self.__sync_display()
        gobject.idle_add(lambda: self.__sync_display())
        
    def add_pipeline(self, pipeline):
        _logger.debug("adding child %s", pipeline)
        dispatcher.connect(self.__on_pipeline_state_change, 'state-changed', pipeline)
        odisp = MultiObjectsDisplay(self.__context, pipeline) 
        cmd = CommandExecutionDisplay(self.__context, pipeline, odisp)
        cmd.cmd_header.connect('action', self.__handle_cmd_action)
        cmd.cmd_header.connect('expand-inspector', self.__on_expand_inspector)        
        cmd.show_all()
        pgnum = self.__cmd_notebook.append_page(cmd)
        self.__cmd_notebook.set_current_page(pgnum)
        self.__cmd_overview.add_pipeline(pipeline, odisp)
                
        self.__sync_visible()                
        self.__sync_display(pgnum)
                
        self.notify('pipeline-count')                
        # Garbage-collect old commands at this point        
        gobject.idle_add(self.__command_gc)

    @log_except(_logger)
    def __command_gc(self):
        curtime = time.time()
        changed = False
        for cmd in self.__iter_cmds():
            pipeline = cmd.get_pipeline()
            if pipeline in self.__complete_unseen_pipelines:
                continue
            compl_time = pipeline.get_completion_time()
            if not compl_time:
                continue
            lastview_time = cmd.get_viewed_time()            
            if curtime - lastview_time > self.COMPLETE_CMD_EXPIRATION_SECS:
                changed = True
                self.remove_pipeline(pipeline, destroy=True)
                
        for cmdview in self.__actively_destroyed_pipeline_box:
            cmdview.destroy()
        self.__actively_destroyed_pipeline_box = []
        self.__sync_cmd_sensitivity()      

    def __mark_pipeline_unseen(self, pipeline, unseen):
        (cmdview, overview) = self.__get_widgets_for_pipeline(pipeline)
        cmdview.cmd_header.set_unseen(unseen)
        overview.set_unseen(unseen)
        self.notify('unseen-pipeline-count')        

    def __get_widgets_for_pipeline(self, pipeline):
        cmdview, overview = (None, None)
        for child in self.__cmd_notebook.get_children():
            if not child.cmd_header.get_pipeline() == pipeline:
                continue
            cmdview = child
        for child in self.__cmd_overview.get_overview_list():
            if not child.get_pipeline() == pipeline:
                continue
            overview = child
        return (cmdview, overview)
        
    def remove_pipeline(self, pipeline, disconnect=True, destroy=False):
        if disconnect:
            pipeline.disconnect()
        try:
            self.__complete_unseen_pipelines.remove(pipeline)
        except KeyError, e:
            pass
        (cmdview, overview) = self.__get_widgets_for_pipeline(pipeline)
        self.__cmd_notebook.remove(cmdview)
        self.__cmd_overview.remove_overview(overview)
        if destroy:
            cmdview.destroy()
            overview.destroy()
            return None
        self.notify('pipeline-count')        
        return (cmdview, overview)
    
    @log_except(_logger)
    def __handle_cmd_complete(self, *args):
        self.__sync_cmd_sensitivity()
        
    @log_except(_logger)
    def __handle_cmd_overview_action(self, oview, cmd):
        self.__handle_cmd_action(cmd)
        
    @log_except(_logger)
    def __handle_cmd_action(self, cmd):
        pipeline = cmd.get_pipeline()
        _logger.debug("handling action for %s", pipeline)        
        if pipeline.validate_state_transition('cancelled'):
            _logger.debug("doing cancel")
            pipeline.cancel()
        elif pipeline.validate_state_transition('undone'):
            _logger.debug("doing undo")            
            pipeline.undo()                    
        else:
            raise ValueError("Couldn't do action %s from state %s" % (action,cmd.cmd_header.get_pipeline().get_state()))        
      
    @log_except(_logger)        
    def __on_show_command(self, overview, cmd):
        _logger.debug("showing command %s", cmd)
        target = None
        for child in self.__cmd_notebook.get_children():
            if child.cmd_header.get_pipeline() == cmd.get_pipeline():
                target = child
                break
        if target:
            pgnum = self.__cmd_notebook.page_num(target)
            self.__cmd_notebook.set_current_page(pgnum)
            self.__action_group.get_action("Overview").activate()
            from hotwire_ui.shell import locate_current_shell
            hw = locate_current_shell(self)
            hw.grab_focus()
 
    def get_current(self):
        cmd = self.get_current_cmd(full=True)
        return cmd and cmd.odisp
 
    def get_current_cmd(self, full=False, curpage=None):
        if curpage is not None:
            page = curpage
        else:
            page = self.__cmd_notebook.get_current_page()
        if page < 0:
            return None
        cmd = self.__cmd_notebook.get_nth_page(page)
        if full:
            return cmd
        return cmd.cmd_header

    def __copy_cb(self, a):
        _logger.debug("doing copy cmd")
        cmd = self.get_current_cmd(full=True)
        cmd.odisp.do_copy()
    
    def __cancel_cb(self, a):
        _logger.debug("doing cancel cmd")
        cmd = self.get_current_cmd(full=True)
        cmd.cancel()
        
    def __undo_cb(self, a):
        _logger.debug("doing undo cmd")
        cmd = self.get_current_cmd(full=True)
        cmd.undo()        
        
    def __search_cb(self, a):
        cmd = self.get_current_cmd(full=True)
        top = self.get_toplevel()
        lastfocused = top.get_focus()
        cmd.odisp.start_search(lastfocused)
        
    def __input_cb(self, a):
        cmd = self.get_current_cmd(full=True)
        top = self.get_toplevel()
        lastfocused = top.get_focus()
        cmd.odisp.start_input(lastfocused)        
    
    def __view_previous_cb(self, a):
        self.open_output(True)
        
    def __view_next_cb(self, a):
        self.open_output(False)
        
    def __view_previous_unseen_cb(self, a):
        target = None
        for cmd in self.__iter_cmdslice(False):
            pipeline = cmd.odisp.get_pipeline()            
            if pipeline in self.__complete_unseen_pipelines:
                target = cmd
                break
        if target:
            pgnum = self.__cmd_notebook.page_num(target)
            self.__cmd_notebook.set_current_page(pgnum) 
        
    def __view_last_cb(self, a):
        self.__cmd_notebook.set_current_page(self.__cmd_notebook.get_n_pages()-1)
        
    def __view_home_cb(self, a):
        self.__do_scroll(True, True)
        
    def __view_end_cb(self, a):
        self.__do_scroll(False, True)    
        
    def __view_up_cb(self, a):
        self.__do_scroll(True, False)
        
    def __view_down_cb(self, a):
        self.__do_scroll(False, False)
    
    def __to_window_cb(self, a):
        cmd = self.get_current_cmd(full=True)
        pipeline = cmd.cmd_header.get_pipeline()
        #pipeline.disconnect('state-changed', self.__on_pipeline_state_change)                 
        (cmdview, overview) = self.remove_pipeline(pipeline, disconnect=False)
        self.emit('new-window', cmdview)
        self.__sync_display()
        
    def __remove_pipeline_cb(self, a):
        cmd = self.get_current_cmd(full=True)
        pipeline = cmd.cmd_header.get_pipeline()
        _logger.debug("doing remove of %s", pipeline)
        (cmdview, overview) = self.remove_pipeline(pipeline, disconnect=False)
        overview.destroy()
        self.__actively_destroyed_pipeline_box.append(cmdview)        
        self.__sync_display()
        
    def __undo_remove_pipeline_cb(self, a):
        cmd = self.__actively_destroyed_pipeline_box.pop()
        _logger.debug("undoing remove of %s", cmd)        
        pgnum = self.__cmd_notebook.append_page(cmd)
        self.__cmd_notebook.set_current_page(pgnum)
        self.__cmd_overview.add_pipeline(cmd.cmd_header.get_pipeline(), cmd.odisp)
        self.__sync_display(pgnum)

    def __overview_cb(self, a): 
        self.__toggle_history_expanded()
        
    def __on_expand_inspector(self, header, expand):
        if self.__inspector_visible == (not not expand):
            return
        self.__action_group.get_action('Inspector').set_active(expand)
        
    def __inspector_cb(self, a): 
        self.__inspector_visible = not self.__inspector_visible
        self.__sync_inspector_expanded()
        
    def __sync_inspector_expanded(self, nth=None):
        self.__sync_visible()
        curcmd = self.get_current_cmd(True, curpage=nth)
        curcmd.cmd_header.set_inspector_expander_active(self.__inspector_visible)
    
    def __vadjust(self, scroll, pos, full):
        adjustment = scroll.get_vadjustment()
        if not full:
            val = scroll.get_vadjustment().page_increment
            if not pos:
                val = 0 - val;
            newval = adjustment.value + val
        else:
            if pos:
                newval = adjustment.upper
            else:
                newval = adjustment.lower
        newval = max(min(newval, adjustment.upper-adjustment.page_size), adjustment.lower)
        adjustment.value = newval
    
    def __do_scroll(self, prev, full):
        if self.__history_visible:
            scroll = self.__cmd_overview.get_scroll()
            self.__vadjust(scroll, not prev, full)
            return
        cmd = self.get_current_cmd()
        if prev:
            cmd.scroll_up(full)
        else:
            cmd.scroll_down(full)
        
    def __toggle_history_expanded(self):
        self.__history_visible = not self.__history_visible
        _logger.debug("history visible: %s", self.__history_visible)
        self.__sync_visible()
        self.__sync_cmd_sensitivity()
        self.__sync_display()
        if self.__history_visible:
            self.__cmd_overview.scroll_to_bottom()            
        
    def __sync_visible(self):
        if self.__history_visible:
            self.__cmd_overview.show()
            self.__cmd_paned.hide()
            self.__header.hide()
            self.__footer.hide()
        else:
            self.__cmd_overview.hide()
            self.__cmd_paned.show()
            if self.__inspector_visible:
                self.__inspector.show()
            else:
                self.__inspector.hide()
            if self.__nextcmd_count > 0:                
                self.__header.show()
            self.__footer.show()
            
    @log_except(_logger)
    def __on_pipeline_state_change(self, signal=None, sender=None):
        pipeline = sender
        _logger.debug("handling state change to %s", pipeline.get_state())
        if pipeline.is_complete():
            self.__complete_unseen_pipelines.add(pipeline)
            self.__mark_pipeline_unseen(pipeline, True)      
        self.__sync_display()
            
    def __sync_cmd_sensitivity(self, curpage=None):
        actions = map(self.__action_group.get_action, ['Copy', 'Cancel', 'PreviousCommand', 'NextCommand', 'Undo', 
                                                       'Input', 'RemovePipeline', 'DetachPipeline', 
                                                       'PreviousUnseenCommand', 'LastCommand', 'UndoRemovePipeline'])
        if self.__history_visible:
            for action in actions:
                action.set_sensitive(False)
            cmd = None
            return
        else:
            undoidx = 10
            actions[undoidx].set_sensitive(len(self.__actively_destroyed_pipeline_box) > 0)             
            cmd = self.get_current_cmd(full=True, curpage=curpage)
            if not cmd:
                for action in actions[:undoidx]:
                    action.set_sensitive(False)                  
                return
            pipeline = cmd.cmd_header.get_pipeline()   
            _logger.debug("sync sensitivity page %s pipeline: %s", curpage, cmd.cmd_header.get_pipeline().get_state())                
            cancellable = not not (pipeline.validate_state_transition('cancelled'))
            undoable = not not (pipeline.validate_state_transition('undone'))
            _logger.debug("cancellable: %s undoable: %s", cancellable, undoable)
            actions[1].set_sensitive(cancellable)
            actions[4].set_sensitive(undoable)
            actions[5].set_sensitive(pipeline.get_state() == 'executing' and cmd.odisp.supports_input() or False)
            actions[6].set_sensitive(pipeline.is_complete())
            actions[7].set_sensitive(True)
        actions[2].set_sensitive(self.__prevcmd_count > 0)
        actions[3].set_sensitive(self.__nextcmd_count > 0)
        actions[8].set_sensitive(len(self.__complete_unseen_pipelines) > 0)
        npages = self.__cmd_notebook.get_n_pages()
        if curpage is None:
            curpage = self.__cmd_notebook.get_current_page()
        actions[9].set_sensitive(npages > 0 and curpage < npages-1)       
        
    def __sync_display(self, nth=None):
        def set_label(container, label, n, label_exec, n_exec, n_done):
            if n <= 0 or self.__history_visible:
                container.hide_all()
                return
            container.show_all()
            label.set_label(gettext.ngettext(' %d pipeline' % (n,), ' %d pipelines' % (n,), n))
            if n_exec > 0 and n_done > 0:
                label_exec.set_markup(_(' %d executing, <b>%d complete</b>') % (n_exec, n_done))
            elif n_done > 0:
                label_exec.set_markup(_(' <b>%d complete</b>') % (n_done,))
            elif n_exec > 0:
                label_exec.set_label(_(' %d executing') % (n_exec,))
            else:
                label_exec.set_label('')
        # FIXME - this is a bit of a hackish place to put this
        curcmd = self.get_current_cmd(True, curpage=nth)
        if curcmd:
            current = curcmd.cmd_header
            pipeline = current.get_pipeline()
            _logger.debug("sync display, current=%s", pipeline)
            if pipeline in self.__complete_unseen_pipelines:
                self.__complete_unseen_pipelines.remove(pipeline)
                self.__mark_pipeline_unseen(pipeline, False)                
            current.update_viewed_time()
        self.__prevcmd_count = 0
        self.__prevcmd_executing_count = 0
        self.__prevcmd_complete_count = 0
        self.__nextcmd_count = 0
        self.__nextcmd_executing_count = 0
        self.__nextcmd_complete_count = 0
        for cmd in self.__iter_cmdslice(False, nth):
            self.__prevcmd_count += 1
            pipeline = cmd.odisp.get_pipeline()            
            if pipeline.get_state() == 'executing':
                self.__prevcmd_executing_count += 1
            if pipeline in self.__complete_unseen_pipelines:
                self.__prevcmd_complete_count += 1
        for cmd in self.__iter_cmdslice(True, nth):
            self.__nextcmd_count += 1
            pipeline = cmd.odisp.get_pipeline()
            if pipeline.get_state() == 'executing':
                self.__nextcmd_executing_count += 1
            if pipeline in self.__complete_unseen_pipelines:
                self.__nextcmd_complete_count += 1
        self.notify('executing-pipeline-count')                
        # The idea here is to not take up the vertical space if we're viewing the last command.
        if self.__nextcmd_count == 0:
            self.__header.hide()
        else:                
            set_label(self.__header, self.__header_label, self.__prevcmd_count, self.__header_exec_label, self.__prevcmd_executing_count, self.__prevcmd_complete_count)
        set_label(self.__footer, self.__footer_label, self.__nextcmd_count, self.__footer_exec_label, self.__nextcmd_executing_count, self.__nextcmd_complete_count)
        self.__sync_cmd_sensitivity(curpage=nth)        
        
        if curcmd:
            if self.__odisp_changed_connection is not None:
                (o, id) = self.__odisp_changed_connection
                o.disconnect(id)
            odisp = curcmd.odisp
            self.__odisp_changed_connection = (odisp, odisp.connect("changed", self.__sync_odisp))
            self.__sync_odisp(odisp)
        
    @log_except(_logger)
    def __sync_odisp(self, odisp):
        self.__inspector.set_otype(odisp.get_output_common_supertype())        
        
    def __iter_cmdslice(self, is_end, nth_src=None):
        if nth_src is not None:
            nth = nth_src
        else:
            nth = self.__cmd_notebook.get_current_page()
        n_pages = self.__cmd_notebook.get_n_pages()
        if is_end:
            r = xrange(nth+1, n_pages)
        else:
            r = xrange(0, nth)
        for i in r:
            yield self.__cmd_notebook.get_nth_page(i)

    def __on_page_switch(self, notebook, page, nth):
        self.__sync_display(nth=nth)
 
    def open_output(self, do_prev=False, dry_run=False):
        nth = self.__cmd_notebook.get_current_page()
        n_pages = self.__cmd_notebook.get_n_pages()
        _logger.debug("histmode: %s do_prev: %s nth: %s n_pages: %s", self.__history_visible, do_prev, nth, n_pages)
        if do_prev and nth > 0:
            target_nth = nth - 1
        elif (not do_prev) and nth < n_pages-1:
            target_nth = nth + 1
        else:
            return False
        if dry_run:
            return True
        self.__cmd_notebook.set_current_page(target_nth)
        from hotwire_ui.shell import locate_current_shell
        hw = locate_current_shell(self)
        hw.grab_focus()         
        
    def do_get_property(self, property):
        if property.name == 'pipeline-count':
            return self.__cmd_notebook.get_n_pages()
        elif property.name == 'unseen-pipeline-count':
            return len(self.__complete_unseen_pipelines)
        elif property.name == 'executing-pipeline-count':
            return self.__prevcmd_executing_count + self.__nextcmd_executing_count         
        else:
            raise AttributeError('unknown property %s' % property.name)
        
    def create_overview_button(self):
        return OverviewButton(self, self.__action_group.get_action('Overview'))
    
    def create_unseen_button(self):
        return UnseenNotifyButton(self, self.__action_group.get_action('PreviousUnseenCommand')) class CompletionStatusDisplay(hotwidgets.TransientPopup):
    __gsignals__ = {
        "histitem-selected" : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, (gobject.TYPE_PYOBJECT,)),                    
        "completion-selected" : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, (gobject.TYPE_PYOBJECT,)),
        "completions-loaded" : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE, []),
    }    
    def __init__(self, entry, window, context=None, tabhistory=[], **kwargs):
        super(CompletionStatusDisplay, self).__init__(entry, window, **kwargs)
        self.__entry = entry
        self.__window = window
        self.__context = context
        self.__tabhistory = tabhistory
        self.__token = None
        self.__completer = None
        self.__complsys = CompletionSystem()
        self.__current_completion = None
        self.__current_history = None
        self.__pending_completion_load = False
        self.__completion_display = MatchPopup(_('Completions (%s)') % ('TAB',),
                                               TabCompletionView,                                                
                                               self.__entry, self.__window, self.__context)
        self.__completion_display.connect('item-selected', self.__on_completion_selected)
        self.__tab_history_display = MatchPopup(_('Tab History'), 
                                                MatchingHistoryView,                                                 
                                                self.__entry, self.__window, self.__context) 
        self.__tab_history_display.connect('item-selected', self.__on_histitem_selected)
        self.__global_history_display = MatchPopup(_('Global History Search (%s)') % ('Ctrl-R',), 
                                                   MatchingHistoryView,
                                                   self.__entry, self.__window, self.__context) 
        self.__global_history_display.connect('item-selected', self.__on_histitem_selected)
        
        self.__overview_visible = False
        self.__completion_visible = False
        self.__tab_history_visible = False
        self.__global_history_visible = False        

        self.get_box().pack_start(self.__completion_display.get_miniview(), expand=True)
        self.get_box().pack_start(gtk.VSeparator(), expand=False)
        self.get_box().pack_start(self.__global_history_display.get_miniview(), expand=True)
         
    def __on_histitem_selected(self, th, histitem):
        self.emit('histitem-selected', histitem)
         
    def __on_completion_selected(self, ac, compl):
        self.emit('completion-selected', compl)

    def invalidate(self):
        self.__token = None
        self.__completer = None
        self.__current_completion = None
        self.__pending_completion_load = False
        self.hide_all()

    def hide_all(self):
        if self.__completion_visible:
            self.__completion_display.hide()
        self.__completion_visible = False
        if self.__tab_history_visible:
            self.__tab_history_display.hide()
        self.__tab_history_visible = False
        if self.__global_history_visible:
            self.__global_history_display.hide()
        self.__global_history_visible = False
        if self.__overview_visible:
            super(CompletionStatusDisplay, self).hide()
        self.__overview_visible = False

    def set_completion(self, completer, text, context):
        if text == self.__token and completer == self.__completer:
            return
        _logger.debug("new completion: %s", text)
        self.invalidate()
        self.__token = text
        self.__completer = completer
        if completer:
            self.__complsys.async_complete(completer, text, context.get_cwd(), self.__completions_result)
        
    def completion_request(self):      
        if self.__current_completion is not None:
            if not self.__completion_visible:
                self.hide_all()
                self.__completion_visible = True
                self.__completion_display.show()
            self.__completion_display.reposition()
            self.__completion_display.queue_reposition()
            return self.__current_completion
        if self.__completer:
            self.hide_all()
            self.__pending_completion_load = True
            return True
        return None
    
    def show(self):
        self.__overview_visible = True
        super(CompletionStatusDisplay, self).show()
        self.reposition()
        self.queue_reposition()
        
    def hide(self):
        self.__overview_visible = False
        super(CompletionStatusDisplay, self).hide()
        
    def __completions_result(self, completer, text, results):
        if not (text == self.__token and completer == self.__completer):
            _logger.debug("stale completion result")
            return
        self.__current_completion = results
        self.__completion_display.set_content(self.__current_completion.results)        
        if self.__pending_completion_load:
            self.__current_completion = results            
            self.emit('completions-loaded')
            self.__pending_completion_load = False
        else:
            if self.__current_completion.results or self.__current_history:
                self.show()
                self.queue_reposition()

    def _set_size_request(self):            
        (ref_x, ref_y, ref_w, ref_h, bits) = self.__entry.get_parent_window().get_geometry()
        _logger.debug("setting size request width to %d*0.75", ref_w)
        self.set_size_request((int(ref_w*0.75)), -1)
        
    def set_history_search(self, lang_uuid, histsearch):
        histitems = map(lambda result: (lang_uuid,result), self.__context.history.search_commands(lang_uuid, histsearch))
        self.__current_history = not not histitems
        self.__global_history_display.set_content(histitems, uniquify=True)
        self.__global_history_display.set_matchtext(histsearch)
            
    def popup_tab_history(self):
        if self.__tab_history_visible:
            return
        _logger.debug("doing tab history popup")
        self.hide()
        self.__tab_history_display.set_content(self.__tabhistory, uniquify=False)         
        self.__tab_history_display.reposition()
        self.__tab_history_display.queue_reposition()
        self.__tab_history_visible = True
        self.__tab_history_display.show()
        
    def popup_global_history(self):
        if self.__global_history_visible:
            return
        self.hide()
        self.__global_history_display.reposition()
        self.__global_history_display.queue_reposition()
        self.__global_history_visible = True
        self.__global_history_display.show()            

    def get_state(self):
        if self.__tab_history_visible:
            return 'tabhistory'
        elif self.__global_history_visible:
            return 'globalhistory'
        elif self.__completion_visible:
            return 'completions'
        return None

    def select_next(self):
        if self.__tab_history_visible:
            self.__tab_history_display.select_next()
            return True
        elif self.__global_history_visible:
            self.__global_history_display.select_next()
            return True
        elif self.__completion_visible:
            self.__completion_display.select_next()
            return True
        return False
        
    def select_prev(self):
        if self.__tab_history_visible:
            self.__tab_history_display.select_prev()
            return True
        elif self.__global_history_visible:
            self.__global_history_display.select_prev()
            return True
        elif self.__completion_visible:
            self.__completion_display.select_prev()
            return True
        return False
    
    def page_up(self):
        if self.__tab_history_visible:
            self.__tab_history_display.page_up()
            return True
        elif self.__global_history_visible:
            self.__global_history_display.page_up()
            return True
        elif self.__completion_visible:
            self.__completion_display.page_up()
            return True
        return False

    def page_down(self):
        if self.__tab_history_visible:
            self.__tab_history_display.page_down()
            return True
        elif self.__global_history_visible:
            self.__global_history_display.page_down()
            return True
        elif self.__completion_visible:
            self.__completion_display.page_down()
            return True
        return False

    def activate_selected(self):
        if self.__tab_history_visible:
            self.__tab_history_display.emit_itemselected()
            return True
        elif self.__global_history_visible:
            self.__global_history_display.emit_itemselected()
            return True
        elif self.__completion_visible:
            self.__completion_display.emit_itemselected()
            return True
        return False def log_except(logger=None, text=''):
    def annotate(func):
        def _exec_cb(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except:
                log_target = logger or logging
                log_target.exception('Exception in callback%s', text and (': '+text) or '')
        return _exec_cb
    return annotate
import os, sys, re, logging, string, locale, weakref
import gtk, gobject, pango
import hotwire.command
import hotwire.version
import hotwire_ui.widgets as hotwidgets
import hotwire_ui.pyshell
from hotwire.command import PipelineFactory,Pipeline,Command,HotwireContext
from hotwire.command import PipelineLanguageRegistry,BaseCommandResolver
from hotwire.completion import Completion, VerbCompleter, TokenCompleter
from hotwire_ui.pixbufcache import PixbufCache
from hotwire.externals.singletonmixin import Singleton
from hotwire.sysdep.term import Terminal
from hotwire.builtin import BuiltinRegistry
from hotwire.cmdalias import Alias, AliasRegistry
from hotwire.gutil import *
from hotwire.util import markup_for_match, quote_arg
from hotwire.fs import path_unexpanduser, path_expanduser, unix_basename, path_fromurl
from hotwire.sysdep import is_unix
from hotwire.sysdep.fs import File, Filesystem
from hotwire.state import History, Preferences, ViewState
from hotwire_ui.command import CommandExecutionDisplay,CommandExecutionControl
from hotwire_ui.completion import CompletionStatusDisplay
from hotwire_ui.aboutdialog import HotwireAboutDialog
from hotwire_ui.msgarea import MsgArea,MsgAreaController
from hotwire_ui.quickfind import QuickFindWindow
from hotwire_ui.prefs import PrefsWindow
from hotwire_ui.dirswitch import DirSwitchWindow
from hotwire.logutil import log_except
from hotwire.externals.dispatch import dispatcher
from hotwire_ui.navigationbar import NavigationBar
        <separator/>
        <menuitem action='DirSwitch'/>
        <separator/>
        <menuitem action='AddBookmark'/>
        <separator/>
        <menuitem action='Home'/>
      </menu>
    </placeholder>
  </menubar>
</ui>'''
        self.__actions = [
            ('SwitchLanguage', 'hotwire', _('Switch _Language'), '<control><shift>L', _('Change the input language'), self.__on_switch_language_cb),
            ('GoMenu', None, _('_Go')),
            ('Up', 'gtk-go-up', _('Up'), '<alt>Up', _('Go to parent directory'), self.__up_cb),
            ('Back', 'gtk-go-back', _('Back'), '<alt>Left', _('Go to previous directory'), self.__back_cb),
            ('Forward', 'gtk-go-forward', _('Forward'), '<alt>Right', _('Go to next directory'), self.__forward_cb),
            ('AddBookmark', None, _('Add Bookmark'), '<alt><shift>B', _('Set a bookmark at this directory'), self.__add_bookmark_cb),
            ('Home', 'gtk-home', _('_Home'), '<alt>Home', _('Go to home directory'), self.__home_cb),
            ('DirSwitch', 'gtk-find', _('Quick Switch'), '<alt>Down', _('Search for a directory'), self.__dirswitch_cb)
        ]
        self.__toggle_actions = [
            ('NavigationBar', None, _('_Navigation Bar'), None, _('Toggle display of navigation bar'), self.__navbar_cb)
        ]
        self.__action_group = gtk.ActionGroup('HotwireActions')
        self.__action_group.add_actions(self.__actions)
        self.__action_group.add_toggle_actions(self.__toggle_actions)

        self.context = HotwireClientContext(self, initcwd=initcwd)
        self.context.history = History.getInstance()
        self.__tabhistory = []
        dispatcher.connect(self.__on_cwd, 'cwd', self.context)

        self.__cwd = self.context.get_cwd()
        
        bookmarks = Filesystem.getInstance().get_bookmarks()
        dispatcher.connect(self.__handle_bookmark_change, sender=bookmarks)
        gobject.idle_add(self.__sync_bookmarks)

        self.drag_dest_set(gtk.DEST_DEFAULT_MOTION | gtk.DEST_DEFAULT_HIGHLIGHT | gtk.DEST_DEFAULT_DROP,
                           [('text/uri-list', 0, 0)],
                           gtk.gdk.ACTION_COPY) 
        self.connect("drag-data-received", self.__on_drag_data_received)
        self.connect("key-press-event", self.__on_toplevel_keypress) 

        self.__paned = gtk.VBox()
        self.__topbox = gtk.VBox()
        self.__welcome = gtk.Label('Welcome to Hotwire.')
        self.__welcome_align = hotwidgets.Align(self.__welcome, yscale=1.0, xscale=1.0)
        self.__paned.pack_start(self.__welcome_align, expand=True)
        self.pack_start(self.__paned, expand=True)

        self.__navigation_bar = NavigationBar(self.context)
        # Visibility is synced by __sync_navbar_display
        self.__navigation_bar.show_all()
        self.__navigation_bar.set_no_show_all(True)
        view_state = ViewState.getInstance()
        navigation_bar_show = view_state.get_state('NavigationBar')
        _logger.debug('Show address bar? %s' % navigation_bar_show)
        if navigation_bar_show == None:
            _logger.debug("No NavigationBar record")
            self.__navigation_bar.hide()
            view_state.set_state('NavigationBar', 0)
        elif not navigation_bar_show:
            _logger.debug("Hide NavigationBar")
            self.__navigation_bar.hide()
        else:
            _logger.debug("Show NavigationBar")
            self.__navigation_bar.show()
        
        self.__topbox.pack_start(self.__navigation_bar, expand = False)
        self.__outputs = CommandExecutionControl(self.context)
        self.__outputs.connect("new-window", self.__on_commands_new_window)      
        self.__topbox.pack_start(self.__outputs, expand=True)

        self.__bottom = gtk.VBox()
        self.__paned.pack_end(hotwidgets.Align(self.__bottom, xscale=1.0, yalign=1.0), expand=False)
        
        self.__emacs_bindings = None
        self.__active_input_completers = []
        
        self.__msgarea_control = MsgAreaController()
        self.__bottom.pack_start(self.__msgarea_control, expand=False)
        
        self.__inputline = gtk.HBox()
        
        self.__overview_button = self.__outputs.create_overview_button()
        self.__inputline.pack_start(self.__overview_button, expand=False)
        self.__unseen_button = self.__outputs.create_unseen_button()
        self.__inputline.pack_start(self.__unseen_button, expand=False)
        
        self.__lang_button = PipelineLanguageButton()
        self.__lang_button.set_lang(PipelineLanguageRegistry.getInstance()['62270c40-a94a-44dd-aaa0-689f882acf34'])
        self.__lang_button.connect('notify::lang', self.__on_lang_button_changed)
        self.__inputline.pack_start(self.__lang_button, expand=False) 
        self.__doing_recentdir_sync = False
        self.__doing_recentdir_navigation = False
        
        self.__recentdir_navigation_index = None
        store = gtk.ListStore(gobject.TYPE_STRING, gobject.TYPE_STRING)
        self.__recentdirs = CwdDisplay()
        self.__recentdirs.connect('changed', self.__on_recentdir_selected)
        self.__inputline.pack_start(hotwidgets.Align(self.__recentdirs), expand=False)           
        
        self.__bottom.pack_start(self.__inputline, expand=False)        
        self.__input = gtk.Entry()
        self.__input.connect("notify::scroll-offset", self.__on_scroll_offset)
        self.__input.connect("notify::text", lambda *args: self.__on_input_changed())
        self.__input.connect("key-press-event", lambda i, e: self.__on_input_keypress(e))
        self.__input.connect("focus-out-event", self.__on_entry_focus_lost)
        self.__inputline.pack_start(self.__input, expand=True)

        self.__idle_parse_id = 0
        self.__parse_stale = False
        self.__parse_resolved = None
        self.__parse_partial = None
        self.__resolver = ShellCommandResolver()
        self.__pipeline_factory = PipelineFactory(self.context, self.__resolver)
        self.__parsed_pipeline = None
        self.__langtype = PipelineLanguageRegistry.getInstance()['62270c40-a94a-44dd-aaa0-689f882acf34']
        self.__verb_completer = VerbCompleter()

        self.__token_completer = TokenCompleter()