class UndertakerCore(loopa.TaskLooper, metaclass=API):
    ''' Note: what about post-facto removal of bindings that have
    illegal targets? For example, if someone uploads a binding for a
    target that isn't currently known, and then it turns out that the
    target, once uploaded, actually doesn't support that binding, what
    should we do?
    
    In theory it shouldn't affect other operations. Should we just bill
    for it and call it a day? We'd need to make some kind of call to the
    bookie to handle that.
    '''
    _librarian = weak_property('__librarian')
    _postman = weak_property('__postman')
    
    def __init__(self, *args, maxlen=25, **kwargs):
        super().__init__(*args, **kwargs)
        self._maxlen = maxlen
        self._triage = None
        
        self._check_lookup = {
            _GidcLite: self._check_gidc,
            _GeocLite: self._check_geoc,
            _GobsLite: self._check_gobs,
            _GobdLite: self._check_gobd,
            _GdxxLite: self._check_gdxx,
            _GarqLite: self._check_garq
        }
        
    async def await_idle(self):
        ''' Wait until the undertaker has no more GC to perform.
        '''
        while self._triage is None:
            await asyncio.sleep(.01)
        
        await self._triage.join()
        
    async def loop_init(self):
        ''' Set up the staging queue.
        '''
        self._triage = asyncio.Queue(maxsize=self._maxlen)
        
    async def loop_stop(self):
        ''' Clear the staging queue.
        '''
        self._triage = None
        
    async def loop_run(self):
        ''' Wait for stuff to be added to the queue, then execute GC on
        it.
        '''
        ghid_to_collect, skip_conn = await self._triage.get()
        logger.debug(str(ghid_to_collect) + ' GC check starting.')
        try:
            obj = await self._librarian.summarize(ghid_to_collect)
            
            try:
                collection_check = self._check_lookup[type(obj)]
            
            except KeyError as exc:
                raise TypeError('Invalid object type received from ' +
                                'librarian.') from exc
                
            else:
                removable = await collection_check(obj, skip_conn)
                if removable:
                    logger.info(str(obj) + ' garbage collection starting.')
                    await self._gc_execute(obj, skip_conn)
                else:
                    logger.debug(str(obj) + ' garbage collection unneeded.')
            
        # except DoesNotExist:
        except KeyError:
            logger.warning(str(ghid_to_collect) + ' missing; could not ' +
                           'garbage collect it.')
            
        finally:
            self._triage.task_done()
        
    def assemble(self, librarian, postman):
        # Call before using.
        self._librarian = librarian
        self._postman = postman
            
    async def _check_gidc(self, obj, skip_conn=None):
        ''' Check whether we should remove a GIDC, and then remove it
        if appropriate. Currently we don't do that, so just leave it
        alone.
        '''
        return False
            
    async def _check_geoc(self, obj, skip_conn=None):
        ''' Check whether we should remove a GEOC, and then remove it if
        appropriate. Pretty simple: is it bound?
        '''
        # Keep if bound
        if (await self._librarian.is_bound(obj)):
            return False
        # Remove if unbound
        else:
            return True
            
    async def _check_gobs(self, obj, skip_conn=None):
        if (await self._librarian.is_debound(obj)):
            # Add our target to the list of GC checks
            await self._triage.put((obj.target, skip_conn))
            return True
        else:
            return False
            
    async def _check_gobd(self, obj, skip_conn=None):
        # If we've been debound, it might be time to die
        if (await self._librarian.is_debound(obj)):
            # Except child bindings can prevent GCing GOBDs
            if (await self._librarian.is_bound(obj)):
                return False
            
            # Dead binding.
            else:
                # Still need to add target
                await self._triage.put((obj.target, skip_conn))
                return True
        
        # Nope, still alive.
        else:
            return False
            
    async def _check_gdxx(self, obj, skip_conn=None):
        # Note that removing a debinding cannot result in a downstream target
        # being GCd, because it wouldn't exist.
        if (await self._librarian.is_debound(obj)):
            return True
        else:
            return False
            
    async def _check_garq(self, obj, skip_conn=None):
        if (await self._librarian.is_debound(obj)):
            return True
        else:
            return False
        
    async def _gc_execute(self, obj, skip_conn):
        # Next, goodbye object.
        await self._librarian.abandon(obj)
        # Now notify the postman, and tell her it's a removal.
        await self._postman.schedule(obj, removed=True, skip_conn=skip_conn)
    
    @fixture_return(None)
    @public_api
    async def alert_gidc(self, obj, skip_conn=None):
        ''' GIDC do not affect GC.
        '''
        # GIDC creates zero triage calls.
        return None
        
    @fixture_return(None)
    @public_api
    async def alert_geoc(self, obj, skip_conn=None):
        ''' GEOC do not affect GC.
        '''
        # GEOC creates zero triage calls.
        return None
        
    @fixture_return(None)
    @public_api
    async def alert_gobs(self, obj, skip_conn=None):
        ''' GOBS do not affect GC.
        '''
        return None
        
    @fixture_return(None)
    @public_api
    async def alert_gobd(self, obj, skip_conn=None):
        ''' GOBD require triage for previous targets.
        '''
        # This will always happen if it's the first frame, so let's be sure
        # to ignore that for logging (also, performance).
        if len(obj.target_vector) > 1:
            try:
                existing = await self._librarian.summarize(obj.ghid)
            
            except KeyError:
                logger.warning(str(obj) + ' existing binding missing; could ' +
                               'not triage old target.')
                triaged = None
            
            else:
                triaged = existing.target
                await self._triage.put((triaged, skip_conn))
        
        else:
            triaged = None
            
        return triaged
        
    @public_api
    async def alert_gdxx(self, obj, skip_conn=None):
        ''' GDXX require triage for new targets.
        '''
        triaged = obj.target
        await self._triage.put((triaged, skip_conn))
        return triaged
        
    @alert_gdxx.fixture
    async def alert_gdxx(self, obj, skip_conn=None):
        ''' Return the obj.target without triaging when fixtured.
        '''
        return obj.target
        
    @fixture_return(None)
    @public_api
    async def alert_garq(self, obj, skip_conn=None):
        ''' GARQ do not affect GC.
        '''
        return None class LibrarianCore(metaclass=API):
    ''' Base class for caching systems common to non-volatile librarians
    such as DiskLibrarian, S3Librarian, etc.
    
    TODO: make ghid vs frame ghid usage more consistent across things.
    
    NOTE: ideally, everything stateful should be in the librarian. In
    other words, you shouldn't use a bookie to track binding status and
    stuff; that should all be handled within librarian. That would make
    the librarian the single source of truth. Unfortunately, there's not
    really a good way of doing that right now ops-wise; you'd need some
    kind of relational filesystem.
    
    TODO: instead, should create a dedicated "cache" system. It can be
    responsible for restoration of stuff, atomic state updates, etc.
    '''
    _enforcer = weak_property('__enforcer')
    _lawyer = weak_property('__lawyer')
    _percore = weak_property('__percore')
    
    @public_api
    def __init__(self, *args, memory_cache=10000, **kwargs):
        super().__init__(*args, **kwargs)
        
        # Lookup for ghid -> hypergolix description
        # This may be GC'd by the python process.
        self._catalog = FiniteDict(maxlen=memory_cache)
        
    @__init__.fixture
    def __init__(self, *args, **kwargs):
        ''' Construct an in-memory-only version of librarian.
        '''
        super(LibrarianCore.__fixture__, self).__init__(*args, **kwargs)
        
        # Lookup <ghid>: <raw data>
        self._shelf = {}
        
        # Lookup for dynamic ghid -> frame ghid
        self._dyn_resolver = {}
        
        # Lookup <bound ghid>: set(<binding obj>)
        self._bound_by_ghid = SetMap()
        
        # Lookup <debound ghid>: set(<debinding ghid>)
        self._debound_by_ghid = SetMap()
        
        # Lookup <recipient>: set(<request ghid>)
        self._requests_for_recipient = SetMap()
        
    @fixture_api
    def RESET(self):
        ''' Reset all of the librarian.
        '''
        self._catalog.clear()
        self._shelf.clear()
        self._dyn_resolver.clear()
        self._bound_by_ghid.clear_all()
        self._debound_by_ghid.clear_all()
        self._requests_for_recipient.clear_all()
        
    def assemble(self, enforcer, lawyer, percore):
        ''' Assign stuff hereto avoid circuitous problems.
        '''
        # We need to be able to check the legality of debindings whose targets
        # are uploaded after the debinding itself is.
        self._enforcer = enforcer
        self._lawyer = lawyer
        # And we need to be able to load things back up from the cache
        self._percore = percore
    
    @public_api
    async def is_bound(self, obj):
        ''' Check to see if the object has been bound.
        '''
        bindings = await self.bind_status(obj.ghid)
        return bool(bindings)
    
    @public_api
    async def is_debound(self, obj):
        # We need to validate any unvalidated bindings. For now, just always
        # validate debindings when an object is being checked.
        debindings = await self.debind_status(obj.ghid)
        invalidated = 0
        for debinding_ghid in debindings:
            # Get the existing debinding object
            debinding = await self.summarize(debinding_ghid)
            
            # Validate existing binding against newly-known target
            try:
                await self._enforcer.validate_gdxx(debinding, target_obj=obj)
                await self._lawyer.validate_gdxx(debinding, target_obj=obj)
                
            # Validation failed. Remove illegal debinding.
            except (InvalidTarget, InconsistentAuthor):
                logger.error(''.join((
                    'Removed invalid existing debinding.\n',
                    '    Debinding author:     ', str(debinding.author), '\n',
                    '    Valid object author:  ', str(obj.author), '\n',
                    '    Debinding target:     ', str(debinding.target), '\n',
                    '    Target type:          ', str(type(obj))
                )))
                # There's no need to do anything with the undertaker, because
                # debindings don't get subscribed to, and we (the librarian)
                # are the sole source of truth.
                await self.abandon(debinding)
                
                # Record that this was invalidated
                invalidated += 1
                
            # If we were differentiating between validated debindings and
            # provisional (unvalidated) debindings, this would be the place to
            # commit the provisional to the validated.
        
        # Instead of making another call to debind_status, we can calculate it
        # directly. NOTE! There is a race condition between the initial
        # validity check on upload, and the commit call.
        
        return bool(len(debindings) - invalidated)
        
    @is_debound.fixture
    async def is_debound(self, obj):
        ''' Create ad-hoc enforcers and lawyers when fixturing self.
        '''
        enforcer = Enforcer.__fixture__(librarian=self)
        lawyer = LawyerCore.__fixture__(librarian=self)
        self._enforcer = enforcer
        self._lawyer = lawyer
        result = await super(LibrarianCore.__fixture__, self).is_debound(obj)
        del enforcer
        del lawyer
        return result
    
    @public_api
    async def store(self, obj, data):
        ''' Starts tracking an object.
        obj is a hypergolix representation object.
        raw is bytes-like.
        '''
        if isinstance(obj, _GobdLite):
            reference_ghid = obj.frame_ghid
            # If we have an existing frame, get ITS frame ghid, so we can clear
            # it if the storage succeeds.
            try:
                old_ghid = await self.resolve_frame(obj.ghid)
            except KeyError:
                old_ghid = None
            
        else:
            reference_ghid = obj.ghid
            old_ghid = None
            
        await self.add_to_cache(obj, data)
        self._catalog[reference_ghid] = obj
        
        # If successful (which is any time we get to here), we also need to get
        # rid of any old dynamic frames and pop them from the catalog.
        if old_ghid is not None:
            await self.remove_from_cache(old_ghid)
            # We need the None regardless of bugs, in case the old frame is
            # "stale" enough to have been released from memory
            self._catalog.pop(old_ghid, None)
    
    @public_api
    async def retrieve(self, ghid):
        ''' Returns the raw data associated with the ghid, checking only
        locally.
        '''
        try:
            ghid = await self.resolve_frame(ghid)
        except KeyError:
            pass
            
        return (await self.get_from_cache(ghid))
    
    @public_api
    async def summarize(self, ghid):
        ''' Returns a lightweight Hypergolix description of the object.
        Checks only locally.
        '''
        try:
            ghid = await self.resolve_frame(ghid)
        except KeyError:
            pass
        
        try:
            obj = self._catalog[ghid]
        
        except KeyError:
            logger.debug('Attempting lazy-load for ' + str(ghid))
            # This will raise DoesNotExist if missing.
            data = await self.get_from_cache(ghid)
            # This does NOT ingest the data into the persistence system!
            obj = await self._percore.attempt_load(data, quiet=False)
            self._catalog[ghid] = obj
            
        return obj
        
    @public_api
    async def abandon(self, obj):
        ''' Forces erasure of an object without notifying anyone else.
        Idempotent. Should never raise KeyError.
        '''
        if isinstance(obj, _GobdLite):
            ghid = obj.frame_ghid
        else:
            ghid = obj.ghid
        
        await self.remove_from_cache(ghid)
        
        # Delete it from the catalog (if it exists there)
        self._catalog.pop(ghid, None)
    
    # Subclasses MAY define this, but are not required to do so.
    @fixture_api
    async def restore(self):
        ''' For LibrarianCore, do nothing. This is just here as an
        endpoint for restore calls. If subclasses want/need to support
        restoration, they should override this.
        '''
    
    # Subclasses MUST define this to work!
    # @abc.abstractmethod
    @fixture_api
    async def contains(self, ghid):
        ''' Checks the ghidcache for the ghid.
        '''
        try:
            ghid = await self.resolve_frame(ghid)
        except KeyError:
            pass
        
        # Catalog may only be accurate locally. Shelf is accurate globally.
        return ghid in self._shelf
    
    # Subclasses MUST define this to work!
    # @abc.abstractmethod
    @fixture_api
    async def resolve_frame(self, ghid):
        ''' Get the current frame ghid from the dynamic ghid.
        '''
        if not isinstance(ghid, Ghid):
            raise TypeError('Ghid must be a Ghid.')
            
        if ghid in self._dyn_resolver:
            return self._dyn_resolver[ghid]
        else:
            raise KeyError(str(ghid) + ' not known as dynamic ghid.')
    
    # Subclasses MUST define this to work!
    # @abc.abstractmethod
    @fixture_api
    async def recipient_status(self, ghid):
        ''' Return a frozenset of ghids assigned to the passed ghid as
        a recipient.
        '''
        return self._requests_for_recipient.get_any(ghid)
    
    # Subclasses MUST define this to work!
    # @abc.abstractmethod
    @fixture_api
    async def bind_status(self, ghid):
        ''' Return a frozenset of ghids binding the passed ghid.
        '''
        return self._bound_by_ghid.get_any(ghid)
    
    # Subclasses MUST define this to work!
    # @abc.abstractmethod
    @fixture_api
    async def debind_status(self, ghid):
        ''' Return either a ghid, or None.
        '''
        # Note that any particular object can have exactly zero or one VALID
        # debinds, but that a malicious actor could find a race condition and
        # debind something FOR SOMEONE ELSE before the bookie knows about the
        # original object authorship.
        return self._debound_by_ghid.get_any(ghid)
            
    # Subclasses MUST define this to work!
    # @abc.abstractmethod
    @fixture_api
    async def add_to_cache(self, obj, data):
        ''' Adds the passed raw data to the cache.
        '''
        if isinstance(obj, _GobsLite):
            reference_ghid = obj.ghid
            self._bound_by_ghid.add(obj.target, obj.ghid)
            
        elif isinstance(obj, _GobdLite):
            reference_ghid = obj.frame_ghid
            try:
                existing = await self.summarize(obj.ghid)
            except KeyError:
                pass
            else:
                self._bound_by_ghid.remove(existing.target, obj.ghid)
                
            # Now we have a clean slate and need to update things accordingly.
            self._bound_by_ghid.add(obj.target, obj.ghid)
            self._dyn_resolver[obj.ghid] = obj.frame_ghid
                
        elif isinstance(obj, _GdxxLite):
            reference_ghid = obj.ghid
            self._debound_by_ghid.add(obj.target, obj.ghid)
            
        elif isinstance(obj, _GarqLite):
            reference_ghid = obj.ghid
            # NOTE: this is only necessary for a persistence server
            self._requests_for_recipient.add(obj.recipient, obj.ghid)
            
        else:
            reference_ghid = obj.ghid
            
        self._shelf[reference_ghid] = data
        
    # Subclasses MUST define this to work!
    # @abc.abstractmethod
    @fixture_api
    async def remove_from_cache(self, ghid):
        ''' Removes the data associated with the passed ghid from the
        cache.
        '''
        obj = await self.summarize(ghid)
        
        if isinstance(obj, _GobsLite):
            reference_ghid = obj.ghid
            self._bound_by_ghid.discard(obj.target, obj.ghid)
            
        elif isinstance(obj, _GobdLite):
            reference_ghid = obj.frame_ghid
            self._bound_by_ghid.discard(obj.target, obj.ghid)
            
            if self._dyn_resolver.get(obj.ghid, None) == obj.frame_ghid:
                self._dyn_resolver.pop(obj.ghid, None)
            
        elif isinstance(obj, _GdxxLite):
            reference_ghid = obj.ghid
            self._debound_by_ghid.discard(obj.target, obj.ghid)
            
        elif isinstance(obj, _GarqLite):
            reference_ghid = obj.ghid
            self._requests_for_recipient.discard(obj.recipient, obj.ghid)
            
        else:
            reference_ghid = obj.ghid
        
        try:
            self._shelf.pop(reference_ghid)
        except KeyError as exc:
            # Protect the full GHID from accidental exposure
            raise DoesNotExist(str(ghid)) from None
        
    # Subclasses MUST define this to work!
    # @abc.abstractmethod
    @fixture_api
    async def get_from_cache(self, ghid):
        ''' Returns the raw data associated with the ghid.
        '''
        try:
            return self._shelf[ghid]
        except KeyError as exc:
            # Protect the full GHID from accidental exposure
            raise DoesNotExist(str(ghid)) from None class PostalCore(loopa.TaskLooper, metaclass=API):
    ''' Tracks, delivers notifications about objects using **only weak
    references** to them. Threadsafe.
    
    ♫ Please Mister Postman... ♫
    
    Question: should the distributed state management of GARQ recipients
    be managed here, or in the bookie (where it currently is)?
    '''
    _librarian = weak_property('__librarian')
    
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        
        # The scheduling queue is created at loop init.
        self._scheduled = None
        # The delayed lookup. <awaiting ghid>: set(<subscribed ghids>)
        self._deferred = SetMap()
        
        # Resolve primitives into their schedulers.
        self._scheduler_lookup = {
            _GidcLite: self._schedule_gidc,
            _GeocLite: self._schedule_geoc,
            _GobsLite: self._schedule_gobs,
            _GobdLite: self._schedule_gobd,
            _GdxxLite: self._schedule_gdxx,
            _GarqLite: self._schedule_garq
        }
        
    @fixture_api
    def RESET(self):
        ''' In general this would be where you'd reset self._scheduled,
        but since self.schedule() is fixtured as a NOOP, there's no
        real reason to do anything here.
        '''
        
    def assemble(self, librarian):
        # Links the librarian.
        self._librarian = librarian
        
    async def await_idle(self):
        ''' Wait until the postman has no more deliveries to perform.
        '''
        while self._scheduled is None:
            await asyncio.sleep(.01)
        
        await self._scheduled.join()
        
    async def loop_init(self):
        ''' Init all of the needed async primitives.
        '''
        self._scheduled = asyncio.Queue()
        
    async def loop_run(self):
        ''' Deliver notifications as soon as they are available.
        TODO: support parallel sending.
        '''
        subscription, notification, skip_conn = await self._scheduled.get()
        
        try:
            logger.info(str(subscription) + ' subscription out for delivery.')
            # We can't spin this out into a thread because some of our
            # delivery mechanisms want this to have an event loop.
            await self._deliver(subscription, notification, skip_conn)
            
        except asyncio.CancelledError:
            logger.debug('PostalCore cancelled.')
            raise
        
        except Exception:
            logger.error(str(subscription) + ' subscription FAILED for ' +
                         'notification ' + str(notification) +
                         ' w/ traceback:\n' +
                         ''.join(traceback.format_exc()))
            
        finally:
            self._scheduled.task_done()
        
    async def loop_stop(self):
        ''' Clear the async primitives.
        '''
        # Ehhhhh, should the queue be emptied before being destroyed?
        self._scheduled = None
        
    @fixture_return(True)
    @public_api
    async def schedule(self, obj, removed=False, skip_conn=None):
        ''' Schedules update delivery for the passed object.
        '''
        # It's possible we're being told to schedule nothing, so catch that
        # here.
        if obj is None:
            return
        
        else:
            try:
                scheduler = self._scheduler_lookup[type(obj)]
            
            except KeyError:
                raise TypeError('Postman scheduling failed for ' +
                                type(obj).__name__) from None
            
            else:
                await scheduler(obj, removed, skip_conn)
                
            return True
        
    async def _schedule_gidc(self, obj, removed, skip_conn):
        # GIDC will never trigger a subscription.
        pass
        
    async def _schedule_geoc(self, obj, removed, skip_conn):
        # GEOC will never trigger a subscription directly, though they might
        # have deferred updates.
        # Note that these have already been put into _SubsUpdate form.
        for deferred in self._deferred.pop_any(obj.ghid):
            await self._scheduled.put(deferred)
        
    async def _schedule_gobs(self, obj, removed, skip_conn):
        # GOBS will never trigger a subscription.
        pass
        
    async def _schedule_gobd(self, obj, removed, skip_conn):
        # GOBD might trigger a subscription! But, we also might to need to
        # defer it. Or, we might be removing it.
        if removed:
            debinding_ghids = await self._librarian.debind_status(obj.ghid)
            
            # Check to see that there are the proper number of debindings
            num_debindings = len(debinding_ghids)
            if num_debindings != 1:
                raise ValueError('Improper debinding number: ' +
                                 str(num_debindings) + ' for ' + str(obj))
                
            # Debinding_ghids is a frozenset; this is the fastest way of
            # getting the single element from it.
            debinding_ghid = next(iter(debinding_ghids))
            await self._scheduled.put(
                _SubsUpdate(obj.ghid, debinding_ghid, skip_conn)
            )
            
        else:
            notifier = _SubsUpdate(obj.ghid, obj.frame_ghid, skip_conn)
            if (await self._librarian.contains(obj.target)):
                logger.debug(str(obj) +
                             ' subscription notification scheduled for: ' +
                             str(obj.target))
                await self._scheduled.put(notifier)
            
            else:
                self._deferred.add(obj.target, notifier)
                logger.debug(str(obj) +
                             ' subscription notification deferred for: ' +
                             str(obj.target))
        
    async def _schedule_gdxx(self, obj, removed, skip_conn):
        # GDXX will never directly trigger a subscription. If they are removing
        # a subscribed object, the actual removal (in the undertaker GC) will
        # trigger a subscription without us.
        pass
        
    async def _schedule_garq(self, obj, removed, skip_conn):
        # GARQ might trigger a subscription! Or we might be removing it.
        if removed:
            debinding_ghids = await self._librarian.debind_status(obj.ghid)
            
            # Check to see that there are the proper number of debindings
            num_debindings = len(debinding_ghids)
            if num_debindings != 1:
                raise ValueError('Improper debinding number: ' +
                                 str(num_debindings) + ' for ' + str(obj))
                
            # Debinding_ghids is a frozenset; this is the fastest way of
            # getting the single element from it.
            debinding_ghid = next(iter(debinding_ghids))
            await self._scheduled.put(
                _SubsUpdate(obj.recipient, debinding_ghid, skip_conn)
            )
        else:
            await self._scheduled.put(
                _SubsUpdate(obj.recipient, obj.ghid, skip_conn)
            )
            
    async def _deliver(self, subscription, notification, skip_conn):
        ''' Do the actual subscription update.
        '''
        # We need to freeze the listeners before we operate on them, but we
        # don't need to lock them while we go through all of the callbacks.
        # Instead, just sacrifice any subs being added concurrently to the
        # current delivery run.
        pass class Enforcer(metaclass=API):
    ''' Enforces valid target selections.
    '''
    _librarian = weak_property('__librarian')
    
    @fixture_api
    def __init__(self, librarian, *args, **kwargs):
        super(Enforcer.__fixture__, self).__init__(*args, **kwargs)
        self._librarian = librarian
        
    def assemble(self, librarian):
        # Call before using.
        self._librarian = librarian
        
    async def validate_gidc(self, obj):
        ''' GIDC need no target verification.
        '''
        return True
        
    async def validate_geoc(self, obj):
        ''' GEOC need no target validation.
        '''
        return True
        
    async def validate_gobs(self, obj):
        ''' Check if target is known, and if it is, validate it.
        '''
        try:
            target = await self._librarian.summarize(obj.target)
        # TODO: think more about this, and whether everything has been updated
        # appropriately to raise a DoesNotExist instead of a KeyError.
        # This could be more specific and say DoesNotExist
        except KeyError:
            logger.debug(str(obj) + ' target missing from librarian: ' +
                         str(obj.target))
        else:
            for forbidden in (_GidcLite, _GobsLite, _GdxxLite, _GarqLite):
                if isinstance(target, forbidden):
                    raise InvalidTarget(str(obj) + ' target invalid: ' +
                                        str(target))
        return True
        
    async def validate_gobd(self, obj):
        ''' Check if target is known, and if it is, validate it.
        
        Also do a state check on the dynamic binding.
        '''
        try:
            target = await self._librarian.summarize(obj.target)
        except KeyError:
            logger.debug(str(obj) + ' target missing from librarian: ' +
                         str(obj.target))
        else:
            for forbidden in (_GidcLite, _GobsLite, _GdxxLite, _GarqLite):
                if isinstance(target, forbidden):
                    raise InvalidTarget(str(obj) + ' target invalid: ' +
                                        str(target))
                    
        await self._validate_dynamic_history(obj)
                    
        return True
        
    async def validate_gdxx(self, obj, target_obj=None):
        ''' Check if target is known, and if it is, validate it.
        '''
        try:
            if target_obj is None:
                target = await self._librarian.summarize(obj.target)
            else:
                target = target_obj
        except KeyError:
            logger.warning(str(obj) + ' validated by Enforcer, but its ' +
                           'target was unknown: ' + str(obj.target))
            logger.debug(str(obj) + ' missing target traceback:\n' +
                         ''.join(traceback.format_exc()))
            
        else:
            # NOTE: if this changes, will need to modify place_gdxx in _Bookie
            for forbidden in (_GidcLite, _GeocLite):
                if isinstance(target, forbidden):
                    raise InvalidTarget(str(obj) + ' target invalid: ' +
                                        str(target))
        return True
        
    async def validate_garq(self, obj):
        ''' No additional validation needed.
        '''
        return True
        
    async def _validate_dynamic_history(self, obj):
        ''' Enforces state flow / progression for dynamic objects. In
        other words, ensures monotonic counter.
        '''
        # Try getting an existing binding.
        try:
            existing = await self._librarian.summarize(obj.ghid)
        
        # TOFU (trust on first upload lulz)
        except KeyError:
            pass
                
        else:
            if existing.counter >= obj.counter:
                logger.debug('New obj frame:     ' +
                             str(obj.frame_ghid))
                logger.debug('New obj tarvec:    ' +
                             str(obj.target_vector))
                logger.debug('Existing frame:    ' +
                             str(existing.frame_ghid))
                logger.debug('Existing tarvec:   ' +
                             str(existing.target_vector))
                raise IllegalDynamicFrame(str(obj.ghid) + ' counter ' +
                                          'decreased from ' +
                                          str(existing.counter) + ' to ' +
                                          str(obj.counter)) class PersistenceCore(metaclass=API):
    ''' Provides the core functions for storing Golix objects. Required
    for the hypergolix service to start.
    
    Can coordinate with both "upstream" and "downstream" persisters.
    Other persisters should pass through the "ingestive tract". Local
    objects can be published directly through calling the ingest_<type>
    methods.
    
    TODO: add librarian validation, so that attempting to update an
    object we already have an identical copy to silently exits.
    '''
    _doorman = weak_property('__doorman')
    _enforcer = weak_property('__enforcer')
    _lawyer = weak_property('__lawyer')
    _bookie = weak_property('__bookie')
    _postman = weak_property('__postman')
    _undertaker = weak_property('__undertaker')
    _librarian = weak_property('__librarian')
    _salmonator = weak_property('__salmonator')
    
    # This is a messy way of getting the suffix for validation and stuff but
    # it's getting the job done.
    _ATTR_LOOKUP = {
        _GidcLite: 'gidc',
        _GeocLite: 'geoc',
        _GobsLite: 'gobs',
        _GobdLite: 'gobd',
        _GdxxLite: 'gdxx',
        _GarqLite: 'garq'
    }
    
    @public_api
    def __init__(self, loop=None, *args, **kwargs):
        ''' Create a KeyedAsyncioLock for ingestion.
        '''
        self._ingestion_mutex = KeyedAsyncioLock(loop)
        
    @__init__.fixture
    def __init__(self, librarian=None, *args, **kwargs):
        ''' Right now, well, this is basically a noop. Anticipating
        substantial changes!
        '''
        super(PersistenceCore.__fixture__, self).__init__(
            *args,
            **kwargs
        )
        
        if librarian is not None:
            self._librarian = librarian
        
    def assemble(self, doorman, enforcer, lawyer, bookie, librarian, postman,
                 undertaker, salmonator):
        self._doorman = doorman
        self._enforcer = enforcer
        self._lawyer = lawyer
        self._bookie = bookie
        self._postman = postman
        self._undertaker = undertaker
        self._librarian = librarian
        self._salmonator = salmonator
    
    @public_api
    async def direct_ingest(self, obj, packed, remotable, skip_conn=None):
        ''' Standard ingestion flow for stuff. To be called from ingest
        above, or directly (for objects created "in-house").
        '''
        # Check for a redundant object, which will immediately short-circuit.
        if isinstance(obj, _GobdLite):
            check_ghid = obj.frame_ghid
            log_frame = True
            counter = obj.counter
            target = obj.target
        else:
            check_ghid = obj.ghid
            log_frame = False
            counter = 0
            target = None
        
        async with self._ingestion_mutex(check_ghid):
            if (await self._librarian.contains(check_ghid)):
                logger.debug(
                    str(check_ghid) + ' not ingested: already exists.'
                )
                return False
            
            else:
                logger.info(
                    'Ingesting ' + str(obj) +
                    (
                        ' frame ' + str(counter) + ': ' + str(check_ghid) +
                        ', target: ' + str(target)
                    ) * log_frame + '...'
                )
                # Calculate "gidc", etc
                suffix = self._ATTR_LOOKUP[type(obj)]
                validation_method = 'validate_' + suffix
                
                # Validate the object... (will raise for invalid)
                # ########################
                # Enforce target selection
                await getattr(self._enforcer, validation_method)(obj)
                # Now make sure authorship requirements are satisfied
                await getattr(self._lawyer, validation_method)(obj)
                # Finally make sure persistence rules are followed
                await getattr(self._bookie, validation_method)(obj)
                
                # Ingest the object
                # ########################
                # Alert the undertaker for any necessary GC of targets, etc. Do
                # that before storing at the librarian, so that the undertaker
                # has access to the old state.
                await getattr(self._undertaker, 'alert_' + suffix)(obj,
                                                                   skip_conn)
                # Finally, add it to the librarian.
                await self._librarian.store(obj, packed)
                
                if remotable:
                    await self._salmonator.push(obj.ghid)
            
                return True
    
    @direct_ingest.fixture
    async def direct_ingest(self, obj, packed, remotable, skip_conn=None):
        ''' Fixture direct ingest to just store at librarian directly
        with no verification (if librarian exists), otherwise simply
        return None.
        '''
        try:
            librarian = self._librarian
        except AttributeError:
            return None
        else:
            await librarian.store(obj, packed)
    
    @public_api
    async def attempt_load(self, packed, quiet=True):
        ''' Attempt to load a packed golix object. If quiet=False, it
        will raise if there is no loader.
        
        TODO: move into doorman.
        '''
        # This is kinda silly, but instead of spewing off a million different
        # threads that will immediately die, let's pre-select the loader
        magic = bytes(packed[:4])
        loaders = {
            b'GIDC': self._doorman.load_gidc,
            b'GEOC': self._doorman.load_geoc,
            b'GOBS': self._doorman.load_gobs,
            b'GOBD': self._doorman.load_gobd,
            b'GDXX': self._doorman.load_gdxx,
            b'GARQ': self._doorman.load_garq
        }
        
        try:
            loader = loaders[magic]
            
        # If no successful loader was found, return None, and allow the parent
        # to raise.
        except KeyError as exc:
            if quiet:
                return None
            else:
                raise MalformedGolixPrimitive('No loader found for magic: ' +
                                              str(magic)) from exc
        
        obj = await loader(packed)
            
        return obj
        
    @attempt_load.fixture
    async def attempt_load(self, packed):
        ''' Create an ad-hoc doorman fixture.
        '''
        # Note that, because of the weak ref, we need to actually hold this
        # here to prevent the doorman from being gc'd. So, just create a
        # disposable one.
        doorman = Doorman.__fixture__()
        self._doorman = doorman
        result = \
            await super(PersistenceCore.__fixture__, self).attempt_load(packed)
        del doorman
        return result
    
    @fixture_noop
    @public_api
    async def ingest(self, packed, remotable=True, skip_conn=None):
        ''' Called on an untrusted and unknown object. May be bypassed
        by locally-created, trusted objects (by calling the individual
        ingest methods directly). Parses, validates, and stores the
        object, and returns True; or, raises an error.
        '''
        # This may return None, but that will be caught by the KeyError below.
        obj = await self.attempt_load(packed)
        if obj is None:
            raise MalformedGolixPrimitive(
                'Packed bytes do not appear to be a Golix primitive.'
            )
        
        ingested = await self.direct_ingest(obj, packed, remotable)
        
        # Note that individual ingest methods are only called directly for
        # locally-built objects, which do not need a mail run.
        # If the object is identical to what we already have, the ingester
        # will return None.
        if ingested:
            await self._postman.schedule(obj, skip_conn=skip_conn)
            
        else:
            logger.debug(str(obj.ghid) + ' postal scheduling aborted on ' +
                         'unchanged object.')
        
        # Note: this is not the place for salmonator pushing! Locally
        # created/updated objects call the individual ingest methods
        # directly, so they have to be the ones that actually deal with it.
        
        return ingested class LawyerCore(metaclass=API):
    ''' Enforces authorship requirements, including both having a known
    entity as author/recipient and consistency for eg. bindings and
    debindings.
    
    Threadsafe.
    '''
    _librarian = weak_property('__librarian')
    
    @fixture_api
    def __init__(self, librarian, *args, **kwargs):
        super(LawyerCore.__fixture__, self).__init__(*args, **kwargs)
        self._librarian = librarian
        
    def assemble(self, librarian):
        # Call before using.
        self._librarian = librarian
    
    @fixture_return(True)
    @public_api
    async def _validate_author(self, obj):
        try:
            author = await self._librarian.summarize(obj.author)
        
        except KeyError as exc:
            raise InvalidIdentity('Unknown author: ' +
                                  str(obj.author)) from exc
        
        else:
            if not isinstance(author, _GidcLite):
                raise InvalidIdentity('Invalid author: ' + str(author))
                
        return True
        
    async def validate_gidc(self, obj):
        ''' GIDC need no validation.
        '''
        return True
        
    async def validate_geoc(self, obj):
        ''' Ensure author is known and valid.
        '''
        return (await self._validate_author(obj))
        
    async def validate_gobs(self, obj):
        ''' Ensure author is known and valid.
        '''
        return (await self._validate_author(obj))
        
    async def validate_gobd(self, obj):
        ''' Ensure author is known and valid, and consistent with the
        previous author for the binding (if it already exists).
        '''
        await self._validate_author(obj)
        
        if obj.counter > 0:
            try:
                existing = await self._librarian.summarize(obj.ghid)
            
            except KeyError:
                logger.warning(str(obj) + ' previous binding missing; ' +
                               'cannot validate author self-consistency.')
            
            else:
                if existing.author != obj.author:
                    raise InconsistentAuthor('Existing author for ' +
                                             str(obj) + ': ' +
                                             str(existing.author) +
                                             ' (existing) vs ' +
                                             str(obj.author) + ' (attempted)')
        
        return True
        
    async def validate_gdxx(self, obj, target_obj=None):
        ''' Ensure author is known and valid, and consistent with the
        previous author for the binding.
        
        If other is not None, specifically checks it against that object
        instead of obtaining it from librarian.
        '''
        await self._validate_author(obj)
        
        try:
            if target_obj is None:
                existing = await self._librarian.summarize(obj.target)
            else:
                existing = target_obj
                
        except KeyError:
            logger.debug(str(obj) + ' target missing from librarian; ' +
                         'cannot validate author self-consistency: ' +
                         str(obj.target))
            
        else:
            if isinstance(existing, _GarqLite):
                if existing.recipient != obj.author:
                    raise InconsistentAuthor('Existing author for ' +
                                             str(obj) + ': ' +
                                             str(existing.recipient) +
                                             ' (existing) vs ' +
                                             str(obj.author) + ' (attempted)')
                
            else:
                if existing.author != obj.author:
                    raise InconsistentAuthor('Existing author for ' +
                                             str(obj) + ': ' +
                                             str(existing.author) +
                                             ' (existing) vs ' +
                                             str(obj.author) + ' (attempted)')
        return True
        
    async def validate_garq(self, obj):
        ''' Validate recipient.
        '''
        try:
            recipient = await self._librarian.summarize(obj.recipient)
        
        except KeyError as exc:
            raise InvalidIdentity('Unknown recipient: ' +
                                  str(obj.recipient)) from exc
        
        else:
            if not isinstance(recipient, _GidcLite):
                raise InvalidIdentity('Invalid recipient: ' + str(recipient))
                
        return True class _GidcLite(_BaseLite):
    ''' Lightweight description of a GIDC.
    '''
    __slots__ = [
        'identity'
    ]
    
    def __init__(self, ghid, identity):
        self.ghid = ghid
        self.identity = identity
        
    @classmethod
    def from_golix(cls, golix_obj):
        ''' Convert the golix object to a lightweight representation.
        '''
        identity = SecondParty.from_identity(golix_obj)
        return cls(
            ghid = golix_obj.ghid,
            identity = identity,
        ) class _GeocLite(_BaseLite):
    ''' Lightweight description of a GEOC.
    '''
    __slots__ = [
        'author',
    ]
    
    def __init__(self, ghid, author):
        self.ghid = ghid
        self.author = author
        
    def __eq__(self, other):
        try:
            return (
                super().__eq__(other) and
                self.author == other.author
            )
        # This will not catch a super() TyperError, so we want to be able to
        # compare anything with a ghid. In reality, any situation where the
        # authors don't match but the ghids do is almost certainly a bug; but,
        # compare it anyways just in case.
        except AttributeError as exc:
            return False
        
    @classmethod
    def from_golix(cls, golix_obj):
        ''' Convert the golix object to a lightweight representation.
        '''
        return cls(
            ghid = golix_obj.ghid,
            author = golix_obj.author,
        ) class _GobsLite(_BaseLite):
    ''' Lightweight description of a GOBS.
    '''
    __slots__ = [
        'author',
        'target',
    ]
    
    def __init__(self, ghid, author, target):
        self.ghid = ghid
        self.author = author
        self.target = target
        
    def __eq__(self, other):
        try:
            return (
                super().__eq__(other) and
                self.author == other.author and
                self.target == other.target
            )
            
        # This will not catch a super() TyperError, so we want to be able to
        # compare anything with a ghid. In reality, any situation where the
        # authors don't match but the ghids do is almost certainly a bug; but,
        # compare it anyways just in case.
        except AttributeError as exc:
            return False
        
    @classmethod
    def from_golix(cls, golix_obj):
        ''' Convert the golix object to a lightweight representation.
        '''
        return cls(
            ghid = golix_obj.ghid,
            author = golix_obj.binder,
            target = golix_obj.target,
        ) class _GobdLite(_BaseLite):
    ''' Lightweight description of a GOBD.
    '''
    __slots__ = [
        'author',
        'counter',
        'target_vector',
        'frame_ghid'
    ]
    
    def __init__(self, ghid, author, counter, target_vector, frame_ghid):
        self.ghid = ghid
        self.author = author
        self.counter = counter
        self.target_vector = target_vector
        self.frame_ghid = frame_ghid
        
    def __eq__(self, other):
        try:
            return (
                super().__eq__(other) and
                self.author == other.author and
                self.target == other.target and
                self.frame_ghid == other.frame_ghid
                # Skip history, because it could potentially vary
                # self.history == other.history
            )
            
        # This will not catch a super() TyperError, so we want to be able to
        # compare anything with a ghid. In reality, any situation where the
        # authors don't match but the ghids do is almost certainly a bug; but,
        # compare it anyways just in case.
        except AttributeError as exc:
            return False
        
    @classmethod
    def from_golix(cls, golix_obj):
        ''' Convert the golix object to a lightweight representation.
        '''
        return cls(
            ghid = golix_obj.ghid_dynamic,
            author = golix_obj.binder,
            counter = golix_obj.counter,
            target_vector = golix_obj.target_vector,
            frame_ghid = golix_obj.ghid
        )
        
    @property
    def target(self):
        ''' Access the current target.
        '''
        return self.target_vector[0] class _GdxxLite(_BaseLite):
    ''' Lightweight description of a GDXX.
    '''
    __slots__ = [
        'author',
        'target',
        '_debinding',
    ]
    
    def __init__(self, ghid, author, target):
        self.ghid = ghid
        self.author = author
        self.target = target
        self._debinding = True
        
    def __eq__(self, other):
        try:
            return (
                super().__eq__(other) and
                self.author == other.author and
                self._debinding == other._debinding
            )
            
        # This will not catch a super() TyperError, so we want to be able to
        # compare anything with a ghid. In reality, any situation where the
        # authors don't match but the ghids do is almost certainly a bug; but,
        # compare it anyways just in case.
        except AttributeError as exc:
            return False
        
    @classmethod
    def from_golix(cls, golix_obj):
        ''' Convert the golix object to a lightweight representation.
        '''
        return cls(
            ghid = golix_obj.ghid,
            author = golix_obj.debinder,
            target = golix_obj.target,
        ) class _GarqLite(_BaseLite):
    ''' Lightweight description of a GARQ.
    '''
    __slots__ = [
        'recipient',
    ]
    
    def __init__(self, ghid, recipient):
        self.ghid = ghid
        self.recipient = recipient
        
    def __eq__(self, other):
        try:
            return (
                super().__eq__(other) and
                self.recipient == other.recipient
            )
            
        # This will not catch a super() TyperError, so we want to be able to
        # compare anything with a ghid. In reality, any situation where the
        # authors don't match but the ghids do is almost certainly a bug; but,
        # compare it anyways just in case.
        except AttributeError as exc:
            return False
        
    @classmethod
    def from_golix(cls, golix_obj):
        ''' Convert the golix object to a lightweight representation.
        '''
        return cls(
            ghid = golix_obj.ghid,
            recipient = golix_obj.recipient,
        ) class InvalidIdentity(RemoteNak):
    ''' This RemoteNak is raised when a persistence provider has received
    a primitive with an unknown or invalid author or recipient.
    '''
    pass class InconsistentAuthor(RemoteNak):
    ''' This RemoteNak is raised when a persistence provider has received
    a primitive targeting an inappropriate object (for example, trying
    to debind a GEOC directly).
    '''
    pass
import unittest
import queue
import random
import inspect
import asyncio
from loopa import NoopLoop
from loopa.utils import await_coroutine_threadsafe
from golix._getlow import GIDC
from hypergolix.undertaker import UndertakerCore
from hypergolix.librarian import LibrarianCore
from hypergolix.postal import PostalCore
from hypergolix.persistence import Enforcer
from hypergolix.persistence import PersistenceCore
from hypergolix.lawyer import LawyerCore
from hypergolix.persistence import _GidcLite
from hypergolix.persistence import _GeocLite
from hypergolix.persistence import _GobsLite
from hypergolix.persistence import _GobdLite
from hypergolix.persistence import _GdxxLite
from hypergolix.persistence import _GarqLite
from hypergolix.exceptions import InvalidIdentity
from hypergolix.exceptions import InconsistentAuthor
from _fixtures.ghidutils import make_random_ghid
from _fixtures.identities import TEST_AGENT1
from _fixtures.identities import TEST_AGENT2
from _fixtures.identities import TEST_READER1
from _fixtures.identities import TEST_READER2
from _fixtures.remote_exchanges import gidc1
from _fixtures.remote_exchanges import gidc2
from _fixtures.remote_exchanges import cont1_1  # Known author
from _fixtures.remote_exchanges import cont3_1  # Unknown author
from _fixtures.remote_exchanges import bind1_1  # Known author
from _fixtures.remote_exchanges import bind3_1  # Unknown author
from _fixtures.remote_exchanges import dyn1_1a  # Known author frame 1
from _fixtures.remote_exchanges import dyn1_1b  # Known author frame 2
from _fixtures.remote_exchanges import dyn3_1a  # Unknown author frame 1
from _fixtures.remote_exchanges import dyn3_1b  # Unknown author frame 2
from _fixtures.remote_exchanges import dynF_a   # Inconsistent author frame 1
from _fixtures.remote_exchanges import dynF_b   # Inconsistent author frame 2
from _fixtures.remote_exchanges import dynF_c   # Inconsistent, unk author frm2
from _fixtures.remote_exchanges import debind1_1        # Consistent author
from _fixtures.remote_exchanges import debind1_F        # Inconsistent author
from _fixtures.remote_exchanges import debind3_1        # Unknown author
from _fixtures.remote_exchanges import debindR_1
from _fixtures.remote_exchanges import debindR_F
from _fixtures.remote_exchanges import debind3_TF
from _fixtures.remote_exchanges import dyndebind1_1
from _fixtures.remote_exchanges import dedebind1_1
from _fixtures.remote_exchanges import handshake1_1     # Known recipient
from _fixtures.remote_exchanges import handshake3_1     # Unknown recipient
    from hypergolix import logutils
        # But removed otherwise.
        await_coroutine_threadsafe(
            coro = self.librarian.store(xbind1, debind1_1.packed),
            loop = self.nooploop._loop
        )
        self.assertTrue(
            await_coroutine_threadsafe(
                coro = self.undertaker._check_gobs(sbind1),
                loop = self.nooploop._loop
            )
        )
        
    def test_gobd(self):
        ''' Test gobd operations.
        '''
        await_coroutine_threadsafe(
            coro = self.librarian.store(dbind1a, dyn1_1a.packed),
            loop = self.nooploop._loop
        )
        
        # Gobd should be kept unless explicitly DEbound (or if also explicitly
        # bound; TODO.)
        self.assertFalse(
            await_coroutine_threadsafe(
                coro = self.undertaker._check_gobd(dbind1a),
                loop = self.nooploop._loop
            )
        )
        
        # But removed otherwise.
        await_coroutine_threadsafe(
            coro = self.librarian.store(xbind1d, dyndebind1_1.packed),
            loop = self.nooploop._loop
        )
        self.assertTrue(
            await_coroutine_threadsafe(
                coro = self.undertaker._check_gobd(dbind1a),
                loop = self.nooploop._loop
            )
        )
        
    def test_gdxx(self):
        ''' Test gdxx operations.
        '''
        await_coroutine_threadsafe(
            coro = self.librarian.store(xbind1, debind1_1.packed),
            loop = self.nooploop._loop
        )
        
        # Gdxx should be kept if not DEbound.
        self.assertFalse(
            await_coroutine_threadsafe(
                coro = self.undertaker._check_gdxx(xbind1),
                loop = self.nooploop._loop
            )
        )
        
        # But removed otherwise.
        await_coroutine_threadsafe(
            coro = self.librarian.store(xbind1x, dedebind1_1.packed),
            loop = self.nooploop._loop
        )
        self.assertTrue(
            await_coroutine_threadsafe(
                coro = self.undertaker._check_gdxx(xbind1),
                loop = self.nooploop._loop
            )
        )
        
    def test_garq(self):
        ''' Test garq operations.
        '''
        await_coroutine_threadsafe(
            coro = self.librarian.store(req1, handshake1_1.packed),
            loop = self.nooploop._loop
        )
        
        # Garq should be kept if not DEbound.
        self.assertFalse(
            await_coroutine_threadsafe(
                coro = self.undertaker._check_garq(req1),
                loop = self.nooploop._loop
            )
        )
        
        # But removed otherwise.
        await_coroutine_threadsafe(
            coro = self.librarian.store(xbind1R, debindR_1.packed),
            loop = self.nooploop._loop
        )
        self.assertTrue(
            await_coroutine_threadsafe(
                coro = self.undertaker._check_garq(req1),
                loop = self.nooploop._loop
            )
        )


class UndertakerAlertTest(unittest.TestCase):
    ''' Test the standard UndertakerCore external interface (alerting).
    '''
    
    @classmethod
    def setUpClass(cls):
        cls.nooploop = NoopLoop(
            debug = True,
            threaded = True
        )
        cls.nooploop.start()
        
    @classmethod
    def tearDownClass(cls):
        # Kill the running loop.
        cls.nooploop.stop_threadsafe_nowait()
    
    def setUp(self):
        ''' Do a per-test fresh init of the undertaker, as well as the
        fixtures for both librarian and postman.
        '''
        # First prep fixtures

        self.librarian = LibrarianCore.__fixture__()