class NNTPGroup(object):
    """
    An object for manipulating and looking up group information.

    It handles short-form references such as a.b can be transformed
    to alt.binaries., etc.

    """

    # used for translation lookups
    _translations = None

    def __init__(self, name, *args, **kwargs):
        """
        Initialize NNTP Group

        """
        # The Group Name
        self.name = NNTPGroup.normalize(name)

        if self.name is None:
            raise AttributeError(
                "Invalid group {} set specified.".format(name))

    @staticmethod
    def normalize(group, shorthand=True):
        """
        takes a group identifier and normalizes it. For example:
        a.b.test           returns alt.binaries.test
        a.b.pictures.test  returns alt.binaries.pictures.test
        A.Binaries.test    returns alt.binaries.test

        items that can be normalized or are of an invalid group causes the
        function to return None.

        """

        if isinstance(group, NNTPGroup):
            # Support passing in ourselves
            return group.name

        try:
            group = GROUP_INVALID_CHAR_RE.sub('', group).lower()

        except (AttributeError, TypeError):
            # Invalid content passed in
            pass

        if not group:
            return None

        if not shorthand:
            return group

        # If we reach here, we want to additionally try to look up any
        # shorthand notation and expand it
        entries = re.split('[.]+', group)

        if NNTPGroup._translations is None:
            # Populate it
            line_re = re.compile(
                r'^(?P<index>\d)\s+'
                r'(?P<key>[a-z0-9.-]+)\s+'
                r'(?P<lookup>[a-z0-9.-]+)', re.I)

            NNTPGroup._translations = list()
            path = join(dirname(abspath(__file__)), 'var', 'groups.dat')
            with open(path) as fd:
                for line in fd:
                    result = line_re.match(line)
                    if not result:
                        continue

                    index = int(result.group('index'))

                    while len(NNTPGroup._translations) < index+1:
                        NNTPGroup._translations.append({})

                    # store our lookup
                    NNTPGroup._translations[index][result.group('key')] = \
                        result.group('lookup')

        # Iterate over our names
        for depth in range(len(NNTPGroup._translations)):
            if depth >= len(entries):
                # we're done
                break

            # Perform our lookup and translate if we can
            match = NNTPGroup._translations[depth].get(entries[depth])
            if not match:
                # We can't skip some entries and translate others, the chain
                # is broken on our first-mismatch
                break

            # Store our match and keep going
            entries[depth] = match

        # Return our compiled list
        return '.'.join(entries)

    @staticmethod
    def split(groups):
        """
        Takes a string containing multiple groups and returns a normalized
        group set()
        """

        # Initialize our return result set
        result = set()

        if isinstance(groups, basestring):
            groups = GROUP_INVALID_CHAR_RE.split(groups)

        if isinstance(groups, (set, list, tuple)):
            for group in groups:
                if isinstance(group, NNTPGroup):
                    result.add(group)

                elif isinstance(group, basestring):
                    try:
                        result.add(NNTPGroup(group))

                    except AttributeError:
                        pass

                elif isinstance(group, (set, list, tuple)):
                    # a little bit of recursion
                    result |= NNTPGroup.split(group)

        return result

    def shorthand(self):
        """
        Returns the shorthand version of the group
        """

        # TODO
        return self.name

    def __str__(self):
        """
        Return a printable version of the article
        """
        return '%s' % self.name

    def __unicode__(self):
        """
        Return a printable version of the article
        """
        return u'%s' % self.name

    def __lt__(self, other):
        """
        Handles group sorting while sticking 'None' type records (if they
        exist at the end)
        """
        return self.name < other.name

    def __eq__(self, other):
        """
        Handles equality

        """
        if isinstance(other, NNTPGroup):
            return self.name == other.name

        elif isinstance(other, basestring):
            return self.name == NNTPGroup.normalize(other)

        return False

    def __hash__(self):
        """
        allows us to make use of the 'in' keyword. Hence this object can
        reside within a set, and you can still type:
        if 'alt.binaries.test' in set(NNTPGroup(), NNTPGroup())
        """
        return hash(self.name)

    def __len__(self):
        """
        Returns the length of our group
        """
        return len(self.name)

    def __repr__(self):
        """
        Return an unambigious version of the object
        """
        return '<NNTPGroup name="%s" />' % (
            self.name,
        ) MESSAGE_ID_RE = re.compile(r'^\s*<?\s*(?P<id>[a-z0-9@!.$-]+)\s*>?\s*$', re.I) class NNTPArticle(object):
    """
    An object for maintaining retrieved article content. The primary
    reason for this object is it makes it easier to sort the results
    in memory tracking both the content downloaded (using the
    NNTPConnection.get() call along with the file stored locally on disk
    that it's responsible for.

    This function does it's best to behave like a stream. But provides
    some functions to make manipulating and merging with other articles
    easier to do.

    Articles by default assume a roll of 'attached'.  This means that
    the files written to disk are removed the second the object is
    destroyed.  This is intentional!  You can call detach() at any
    time you want but now you are responsible for cleaning up the
    filename.

    """

    def __init__(self, id=None, subject=None, poster=None, groups=None,
                 work_dir=None, body=None, codecs=None, *args, **kwargs):
        """
        Initialize NNTP Article

        """

        # The Article Message-ID
        self.id = id if isinstance(id, basestring) else ''

        # The NNTP Group Index #
        self.no = 1000
        try:
            self.no = int(kwargs.get(u'no', self.no))
        except:
            # Just use the default
            pass

        # The Subject
        self.subject = subject
        if self.subject is None:
            self.subject = DEFAULT_NNTP_SUBJECT

        # The Poster
        self.poster = poster
        if self.poster is None:
            self.poster = DEFAULT_NNTP_POSTER

        if work_dir is None:
            self.work_dir = DEFAULT_TMP_DIR
        else:
            self.work_dir = abspath(expanduser(work_dir))

        # Contains a list of decoded content; it's effectively the articles
        # attachments
        self.decoded = sortedset(key=lambda x: x.key())

        # The group(s) associated with our article
        self.groups = NNTPGroup.split(groups)

        # A hash of header entries
        self.header = NNTPHeader(work_dir=self.work_dir)

        if isinstance(body, NNTPAsciiContent):
            self.body = body
        else:
            # Our body contains non-decoded content
            self.body = NNTPAsciiContent(work_dir=self.work_dir)

        if isinstance(body, basestring) and len(body) > 0:
            # Store our body content
            self.body.write(body)

        # Load our Codecs
        self._codecs = codecs

        # Provide default codecs if required
        if not self._codecs:
            self._codecs = [CodecYenc(), ]

        elif isinstance(self._codecs, CodecBase):
            self._codecs = [self._codecs, ]

    def load(self, response):
        """
        Loads an article by it's NNTPResponse or from another NNTPArticle

        """
        if isinstance(response, NNTPResponse):
            # Our body contains non-decoded content
            self.body = response.body

            # Store decoded content
            self.decoded = response.decoded

            # Store Header
            self.header = next(
                (d for d in self.decoded if isinstance(d, NNTPHeader)), None)

            # Our groups associated with the post (if we know it)
            self.groups = set()

            if self.header is not None:
                # Remove Header from decoded list
                self.decoded.remove(self.header)

                if u'Newsgroups' in self.header:
                    # Parse our groups out of the header
                    self.groups = NNTPGroup.split(self.header[u'Newsgroups'])

        elif isinstance(response, NNTPArticle):
            # We basically save everything except the work-dir since
            # our new work dir could be a new location

            # The NNTP Message-ID
            self.id = response.id

            # The NNTP Group Index
            self.no = response.no

            # Store the subject
            self.subject = response.subject

            # Store the poster
            self.poster = response.poster

            # The NNTP Groups
            self.groups = response.groups

            # Store the header from the other side
            self.header = response.header

            # Our body contains non-decoded content
            self.body = response.body

            # Store decoded content
            self.decoded = response.decoded

        else:
            # Unsupported
            return False

        return True

    def post_iter(self, update_headers=True):
        """
        Returns NNTP string as it would be required for posting
        """

        if not self.id:
            # Generate ourselves a Message-ID
            self.msgid()

        if not len(self.body) and not len(self.decoded):
            # Nothing to post
            logger.error(
                'Post Denied / Article %s has no content to post.' %
                self.msgid())
            return None

        if not len(self.groups):
            # Nothing to post
            logger.error(
                'Post Denied / Article %s has no groups defined.' %
                self.msgid())
            return None

        try:
            if not self.subject.strip():
                logger.error(
                    'Post Denied / Article %s has no subject.' %
                    self.msgid())
                return None

        except AttributeError:
            logger.error(
                'Post Denied / Article %s has no subject.' % self.msgid())
            return None

        try:
            if not self.poster.strip():
                logger.error(
                    'Post Denied / Article %s has no poster.' % self.msgid())
                return None

        except AttributeError:
            logger.error(
                'Post Denied /Article %s has no poster.' % self.msgid())
            return None

        if update_headers:
            self.update_headers()

        args = [
            iter(self.header.post_iter()),
        ]

        if len(self.body):
            args.append(iter(self.body.post_iter()))
            args.append(NNTP_EOL)

        if len(self.decoded):
            for entry in self.decoded:
                args.append(iter(entry.post_iter()))
                args.append(NNTP_EOL)

        return chain(*args)

    def encode(self, encoders):
        """
        A wrapper to the encoding of content. The function returns None if
        a problem occurs, otherwise the function returns an NNTPArtice()
        object.

        The power of this function comes from the fact you can pass in
        multiple encoders to have them all fire after one another.
        """
        if len(self) == 0:
            # Nothing to encode
            return None

        objs = sortedset(key=lambda x: x.key())
        for content in self:
            obj = content.encode(encoders)
            if obj is None:
                return None

            # Successful, add our object to our new list
            objs.add(obj)

        # If we reach here we encoded our entire article
        # Create a copy of our article
        article = self.copy(include_attachments=False)

        # In our new copy; store our new encoded content
        article.decoded = objs

        # Return our article
        return article

    def split(self, size=81920, mem_buf=1048576, body_mirror=0):
        """
        Split returns a set of NNTPArticle() objects containing the split
        version of the data it already represents.

        Even if the object can't be split any further given the parameters, a
        set of at least 1 entry will always be returned.  None is returned if
        an error occurs. None is also returned if split() is called while there
        is more then one NNTPContent objects since it makes the situation
        Ambiguous.

        The body_mirror flag has a series of meanings; since we start with a
        single post before calling this (which goes on and splits the post) we
        have to decide if we want the contents of our message body to appear
        in every post. So here is how it breaks down.

        If you set it to False, None then the body content is ignored
        and nothing is done with it.

        If you set it to True, then the body is copied to every single part.

        If you set it to -X (negative #) then the body is stored in the
        part generated as if you called the python slice expression:
            parts[-X].body <-- store body here

        If you set it to X (positive #), then the body is stored in
        that index value similar to above:
            parts[X].body <-- store body here

        IndexErrors are automatially and silently handled.  If you access
        a part that is out of bounds then the bounds are limited to fit
        automtatically.  For instance, if you specified -100 as the
        part and there were only 4 parts generated, (-4 is the maximum
        it could have been) then -4 is used in place of the -100.

        consiquently if you passed in 100 (a positive value). since
        (positive) indexes are measured started at 0 (zero), this means a
        with a list of 4 items the largest value can be a 3.  Therefore
        the index of 3 is used instead.
        """
        if len(self) > 1:
            # Ambiguous
            return None

        if len(self) == 0:
            # Nothing to split
            return None

        content = next((
            c for c in self.decoded if isinstance(c, NNTPContent)), None)

        if not content:
            # Well this isn't good; we have decoded entries that are not
            # of type NNTPContent.
            return None

        # ensure our size is set to some value
        size = strsize_to_bytes(size)
        if size is None:
            # You can't call split() with a size value set at None
            return None

        # Size needs to consider our message body
        # The length of our headers are not applicable
        if len(self.body):
            size -= len(self.body) + len(NNTP_EOL)

        if size < 0:
            # we can't perform the split
            return None

        # Split our content
        new_content = content.split(size=size, mem_buf=mem_buf)

        if new_content is None:
            # something bad happened
            return None

        # If we get here, we have content to work with.  We need to generate
        # a list of articles based on our existing one.
        articles = sortedset(key=lambda x: x.key())

        for no, c in enumerate(new_content):
            a = NNTPArticle(
                subject=self.subject,
                poster=self.poster,
                groups=self.groups,
                # Increment our index #
                no=self.no + no,
                work_dir=self.work_dir,
            )

            # Set our header to be a copy of what we already have
            a.header = deepcopy(self.header)

            # Store our NNTPContent() object
            a.add(c)

            if body_mirror is True:
                # body is mirrored to everything
                a.body = self.body

            # Store our Article
            articles.add(a)

        # Now we transfer over our body if nessisary
        if isinstance(body_mirror, int):
            if body_mirror > -1:
                try:
                    articles[body_mirror].body = self.body

                except IndexError:
                    # Store at last entry
                    articles[-1].body = self.body
            else:
                # Negative Number
                try:
                    articles[body_mirror].body = self.body

                except IndexError:
                    # Store at first entry
                    articles[0].body = self.body

        # Return our articles
        return articles

    def append(self, articles):
        """
        Appends one article onto another. This is the reverse of split()
        articles must be an NNTPArticle object, or a set-of NNTPArticle()
        objects.

        """
        if isinstance(articles, NNTPArticle):
            # Convert our entry into a list object
            articles = [articles, ]

        # Change to an iterator
        articles = iter(articles)
        if len(self) > 1:
            # Ambiguous; we can't append content if we have
            # more then one possible thing to append to
            return False

        for article in articles:
            if len(article) == 0:
                # No attachments; move along silently
                continue

            elif len(article) > 1:
                # To many attachments in article
                return False

            if len(self.decoded) == 0:
                # Store a copy of our decoded object
                content = article[0].copy()
                if content is None:
                    # We failed to create a copy
                    return False

                # Add our content to our object
                self.decoded.add(content)
                continue

            # If we reach here, we have 1 entry to append our content to
            if not self.decoded[0].append(article[0]):
                return False

        # Return our success
        return True

    def copy(self, include_attachments=True):
        """
        Creates a copy of the article returning one

        """

        article = NNTPArticle(
            subject=self.subject,
            poster=self.poster,
            groups=self.groups,
        )

        article.id = self.id
        article.no = self.no
        article.work_dir = self.work_dir
        article.header = self.header.copy()
        article.body = self.body.copy()

        if include_attachments:
            for content in self.decoded:
                obj = content.copy()
                if obj is None:
                    return None

                # Add our copied NNTP object
                article.decoded.add(obj)

        # Return our copy
        return article

    def is_valid(self):
        """
        Iterates over article content an returns True if all of it is valid
        based on it's crc32/md5 and/or whatever mechanism is used to determine
        a NNTPContents validity
        """
        if len(self.decoded) == 0:
            # No articles means no validity
            return False

        return next((
            False for c in self.decoded if c.is_valid() is False), True)

    def files(self):
        """
        Returns a list of the files within article
        """
        return [x.path() for x in self.decoded]

    def key(self):
        """
        Returns a key that can be used for sorting with:
            lambda x : x.key()
        """
        return '%.5d%s' % (self.no, self.id)

    def detach(self):
        """
        Detach the article stored on disk from being further managed by this
        class
        """
        for a in self.decoded:
            if isinstance(a, NNTPBinaryContent):
                a.detach()
        return

    def attach(self):
        """
        Detach the article stored on disk from being further managed by this
        class
        """
        for a in self.decoded:
            if isinstance(a, NNTPBinaryContent):
                a.attach()
        return

    def add(self, content):
        """
        Used for adding content to the self.decoded class
        """
        if isinstance(content, basestring):
            # A mime object we can use to detect the type of file
            m = Mime()

            if isfile(content):
                # file relative to current path; get our mime response
                mr = m.from_bestguess(content)

                # Our NNTPContent object will depend on whether or not we're
                # dealing with an ascii file or binary
                instance = NNTPBinaryContent \
                    if mr.is_binary() else NNTPAsciiContent

                content = instance(
                    filepath=content,
                    work_dir=self.work_dir,
                )

            else:
                # Not relative; is the file relative to the work_dir?
                with pushd(self.work_dir, create_if_missing=True):
                    if isfile(content):

                        # file relative to current path; get our mime response
                        mr = m.from_bestguess(content)

                        # Our NNTPContent object will depend on whether or not
                        # we're dealing with an ascii file or binary
                        instance = NNTPBinaryContent \
                            if mr.is_binary() else NNTPAsciiContent

                        # load our file
                        content = instance(
                            filepath=content,
                            work_dir=self.work_dir,
                        )

                    else:
                        # we're dealing with a new file; just save what we have
                        content = NNTPBinaryContent(
                            filepath=content,
                            work_dir=self.work_dir,
                        )

        if not isinstance(content, NNTPContent):
            return False

        # duplicates are ignored in a blist and therefore
        # we just capture the length of our list before
        # and after so that we can properly return a True/False
        # value
        _bcnt = len(self.decoded)
        self.decoded.add(content)

        return len(self.decoded) > _bcnt

    def msgid(self, host=None, reset=False):
        """
        Returns a message ID; if one hasn't been generated yet, it is
        based on the content in the article and used

        If reset is set to True, then a new ID is generated
        """
        if not reset and self.id:
            return self.id

        if not host:
            # Generate a 32-bit string we can use
            host = random_str(32)

        if len(self.decoded):
            partno = self.decoded[0].part
        else:
            partno = 1

        m = hashlib.sha1()
        reference = datetime.utcnow()
        m.update(datetime.utcnow().strftime('%Y%m%d%H%M%S%f'))
        key = m.hexdigest()

        # If we reach here an ID hasn't been generated yet; generate one
        self.id = '%s%x.%x!%x%x%x%x%x@%s%s%s' % (
            key[0:5],
            reference.second,
            reference.microsecond,
            reference.year,
            reference.month,
            reference.day,
            reference.minute,
            partno,
            reference.hour,
            host,
            key[5:10],
        )

        # Then return it
        return self.id

    def save(self, filepath=None, copy=False):
        """
        Saves all of the NNTPContent into the directory specified by
        the filepath; if no filepath is specified, then content is written
        into it's work_dir
        """

        if filepath:
            if not isdir(filepath) and not mkdir(filepath):
                # We failed
                return False

        for attachment in self.decoded:
            if not attachment.save(filepath=filepath, copy=copy):
                return False

        return True

    def deobsfucate(self, filebase='', codecs=None):
        """
        Using the information we have in the article generate a filename to
        the best of our ability.

        filebase: provide a fallback filename base (the part of the file
                  before the extension) to build on if we can't detect
                  the file on our own.

        codecs:   The codec(s) you wish to use to assist in the deobsfucation
        """

        if len(self.decoded) != 1:
            # There is ambiguity if there is not one attachment
            return None

        if filebase is None:
            # Safety
            filebase = ''

        # The detected article filename and extension
        a_name = ''
        a_fext = ''
        a_mime = None

        # The detected attachment filename and extension
        d_name = ''
        d_fext = ''
        d_mime = None

        # Create our Mime object since we'll be using it a lot
        m = Mime()

        if codecs is None:
            codecs = self._codecs

        elif isinstance(codecs, CodecBase):
            codecs = [codecs, ]

        # Use our Codec(s) to extract our Yenc Subject
        matched = None
        for c in codecs:
            # Use our Defined Codec(s) to extract content from our subject
            matched = c.parse_article(
                id=self.id,
                article_no=self.no,
                subject=self.subject,
                poster=self.poster,
            )

            if matched:
                # We succesfully got a filename from our subject line
                a_fname = matched.get('fname', '').strip()
                if a_fname:
                    # always allow the name to over-ride the detected filename
                    # if we actually have a real name we can assciated with it
                    # by
                    a_fext = m.extension_from_filename(a_fname)
                    a_name = a_fname[:-len(a_fext)]
                    a_mime = m.from_filename(a_fname)
                    if a_mime and a_mime.type() == DEFAULT_MIME_TYPE:
                        a_mime = None

                # We can break out
                break

        ####################################
        # Now we want to scan our attachment
        ####################################

        # Our mime object for our attachment
        d_mime = self.decoded[0].mime()
        d_fname = self.decoded[0].filename if self.decoded[0].filename \
            else basename(self.decoded[0].path())
        d_mime = m.from_bestguess(d_fname)
        if d_mime and d_mime.type() == DEFAULT_MIME_TYPE:
            d_mime = None

        d_fext = m.extension_from_filename(d_fname)
        d_name = d_fname[:-len(d_fext)]

        # At this point we have to make a decision to go with the filename
        # pulled from our content object, or go with the filename detected
        # from the article.

        _name = filebase
        _fext = None

        # Using the mime types, we decide which one is more likely the name
        if a_mime is None:
            # Attachment was not parsable
            if d_mime is not None:
                # However we did detect a file through this method
                _fext = d_fext
                _name = filebase if filebase else d_name

        elif d_mime is not None:
            # Attachment is parsable and so is our attachment
            if d_mime == a_mime:
                # Same type
                _fext = d_fext if d_fext else a_fext
                _name = _name if _name else d_name

            else:
                # We need to choose one over the other
                if len(d_fext):
                    # we have a filesize associated with our content which
                    # means there is a good chance we parsed our mime typefrom
                    # here
                    _fext = d_fext if d_fext else d_mime.extension()
                else:
                    # Use whatever we parsed
                    _fext = a_fext if a_fext else a_mime.extension()

                _name = _name if _name else d_name

        else:
            # Attachment is parsable and our Article isn't, easy-peasy
            # just use the content
            _fext = a_fext if a_fext else a_mime.extension()
            _name = filebase if filebase else a_name

        if not _fext:
            # Last resort
            _fext = d_fext if d_fext else a_fext
            if not _fext:
                return None

        if not _name:
            # Last resort
            _name = d_name if d_name else a_name
            if not _name:
                return None

        # Return our assembled identifier
        return "{0}{1}".format(_name, _fext)

    def size(self):
        """
        return the total size of our decoded content; we factor in the body if
        it exists.
        """
        if len(self.body):
            return sum(len(d) for d in self.decoded) + \
                len(self.body) + len(NNTP_EOL)
        return sum(len(d) for d in self.decoded)

    def strsize(self):
        """
        return the total size of our decoded content in a human readable
        string.
        """
        return bytes_to_strsize(self.size())

    def update_headers(self):
        """
        Updates our header information based on specified content
        """
        # Our Newsgroups can be of type NNTPGroup, but it can also just
        # be a plain string 'alt.binaries.test', etc. to support both
        # types we need to do a loop inside of our join below
        self.header['Newsgroups'] = ','.join([str(x) for x in self.groups])
        self.header['Subject'] = self.subject.strip()
        self.header['From'] = self.poster.strip()
        self.header['X-Newsposter'] = '%s v%s' % (__title__, __version__)
        self.header['Message-ID'] = '<%s>' % self.msgid()

    def __iter__(self):
        """
        Grants usage of the next()
        """
        # Ensure our stream is open with read
        return iter(self.decoded)

    def __len__(self):
        """
        Returns the number of decoded content entries found
        """
        return len(self.decoded)

    def __lt__(self, other):
        """
        Handles less than for storing in btrees
        """
        return self.key() < other.key()

    def __eq__(self, other):
        """
        Handles equality

        """
        return self.__dict__ == other.__dict__

    def __getitem__(self, index):
        """
        Support accessing NNTPContent objects by index
        """
        return self.decoded[index]

    def __str__(self):
        """
        Return a printable version of the article
        """
        return '%s' % self.id

    def __unicode__(self):
        """
        Return a printable version of the article
        """
        return u'%s' % self.id

    def __repr__(self):
        """
        Return an unambigious version of the object
        """

        return '<NNTPArticle Message-ID="%s" attachments="%d" />' % (
            self.id,
            len(self.decoded),
        ) class NNTPSegmentedPost(object):
    """
    An object for maintaining retrieved nzb content. Large files need
    to be split across multiple Articles in order to be posted.

    When combined into one, They create a SegmentedPost

    Similarily; this file makes posting easy by allowing you to add files
    to it in order to prepare an object worthy of posting.

    Every file you wish to post to an NNTP Server should be done through
    this file for the most power/control

    """

    def __init__(self, filename, subject=DEFAULT_NNTP_SUBJECT,
                 poster=DEFAULT_NNTP_POSTER, groups=None,
                 utc=None, work_dir=None, sort_no=None, codecs=None,
                 *args, **kwargs):
        """Initialize NNTP Segmented File

        Args:
            filename (str): Either a path of an existing filename or if
                            you're initializing a new file from scratch this
                            should be the name of the filename being
                            assembled.
        Kwargs:
            subject (str): The subject to associate with the file. This is
                            only required if you're posting the content (to
                            Usenet).
            poster (str): The poster to associate with the file. This is
                           only required if you're posting the content (to
                           Usenet).
            group (str, set): This can be either a string identifying a single
                               Usenet group, or it can be a set() of strings
                               identifying all of the Usenet groups you want
                               to cross-post to. This is only required if you
                               intend on posting the content (to Usenet).
            utc (int, datetime): Should always be in UTC format and can be
                                either defined as the number of seconds from
                                epoch (usually produced from gmtime()) or it
                                can be a datetime() object. This identifies the
                                date to associate with the file.  It's mostly
                                used for sorting.  If no time is specified
                                then it defaults to datetime.utcnow(). If a
                                valid `filename` was specified then the utc
                                defaults to the time associated with the files
                                modification date.
            codecs (CodecBase): The codec to use as our deobsfucation engine
                                You can specify as many Codecs as you want in
                                an interable form.

        Returns:
            Nothing

        Raises:
            AttributeError() if you don't follow group isn't a set() of str(),
                              None or str().

        """
        # The Filename
        self.filename = filename

        # These fields get populated when reading in an nzb file
        self.poster = poster
        self.utc = utc
        self.subject = subject

        # The sort order which helps with file ordering, if set to None
        # or a common value, then it has no bearing on anything.  Its
        # main goal is to provide a method of sorting multiple SegmentedPosts
        self.sort_no = sort_no

        # When writing files, this is the memory buffer to fill when
        # dealing with very large files. If this is set to None, then
        # no buffer is used. Default 500K
        self.mem_buffer = 512000

        if not isinstance(self.utc, datetime):
            # Convert into datetime
            try:
                self.utc = datetime.fromtimestamp(int(self.utc))

            except (TypeError, ValueError):
                # Used None, uninitialized, or Used bad value
                # Default timezone to 'now' but make it consistent
                # with the world, use the UTC as a common source
                self.utc = datetime.utcnow()

        # Load our Codecs
        self._codecs = codecs

        # Provide default codecs if required
        if not self._codecs:
            self._codecs = [CodecYenc(), ]

        elif isinstance(self._codecs, CodecBase):
            self._codecs = [self._codecs, ]

        # A sorted set of articles
        self.articles = sortedset(key=lambda x: x.key())

        if work_dir is None:
            self.work_dir = DEFAULT_TMP_DIR
        else:
            self.work_dir = abspath(expanduser(work_dir))

        # The group(s) associatd with our article(s)
        self.groups = NNTPGroup.split(groups)

        if self.filename:
            # attempt to add our filename
            self.add(filename)

    def pop(self, index=0):
        """
        Pops an Article at the specified index out of the segment table
        """
        return self.articles.pop(index)

    def add(self, content):
        """
        Add an NNTPArticle()
        """

        if isinstance(content, basestring):
            # A mime object we can use to detect the type of file
            m = Mime()

            if isfile(content):
                # file relative to current path; get our mime response
                mr = m.from_bestguess(content)

                # Our NNTPContent object will depend on whether or not we're
                # dealing with an ascii file or binary
                instance = NNTPBinaryContent \
                    if mr.is_binary() else NNTPAsciiContent

                content = instance(
                    filepath=content,
                    work_dir=self.work_dir,
                )

            else:
                with pushd(self.work_dir, create_if_missing=True):
                    # Not relative; is the file relative to the work_dir?
                    if isfile(content):
                        # Create a content object from the data
                        # This isn't always the best route because no part #'s
                        # are assigned this way; but if it's only a single file
                        # the user is working with; this way is much easier.

                        # file relative to current path; get our mime response
                        mr = m.from_bestguess(content)

                        # Our NNTPContent object will depend on whether or not
                        # we're dealing with an ascii file or binary
                        instance = NNTPBinaryContent \
                            if mr.is_binary() else NNTPAsciiContent

                        content = instance(
                            filepath=content,
                            work_dir=self.work_dir,
                        )

                    else:
                        # Nothing to add
                        return False

                    # At this point we fall through and the next set of
                    # if checks will catch our new content object we created

        # duplicates are ignored in a blist and therefore
        # we just capture the length of our list before
        # and after so that we can properly return a True/False
        # value
        _bcnt = len(self.articles)

        if isinstance(content, NNTPContent):
            # Create an Article and store our content
            article = NNTPArticle(
                subject=self.subject,
                poster=self.poster,
                groups=self.groups,
                work_dir=self.work_dir,
            )

            # Add the content to our article
            article.add(content)

            # Store the article in our collection
            self.articles.add(article)

        elif isinstance(content, NNTPArticle):
            # Add our article
            self.articles.add(content)

        return len(self.articles) > _bcnt

    def apply_template(self, custom=None, relative=None, strftime=True):
        """
        Iterates over all of the articles defined in the segment and replaces
        any defined {{directive}} with their translated value. All tokens must
        be wrapped in {{ }} braces.

        There are some built in directives, but you can alternatively provide
        your own as well.  The built in directives are:

            {{filename}}        : Translates to the filename associated with
                                    the 'first' attachment inside of the
                                    article.

            {{filesize}}        : Translates to the size of the 'first'
                                    attachment inside of the article

            {{filenameXXX}}     : Handles situations where you know there are
                                    multiple attachments and you want to
                                    exclusively index the filename of one of
                                    these entries. XXX is a numeric
                                    represetation starting a 001 and can range
                                    to 999

            {{filesizeXXX}}     : Handles situations where you know there are
                                    multiple attachments and you want to
                                    exclusively index the filesize of one of
                                    these entries. XXX is a numeric
                                    represetation starting a 001 and can range
                                    to 999

            {{totalsize}}       : Translates to the total size of all
                                    accumulated attachments in the article.

            {{index}}           : Translates to NNTPArticles index as
                                    represented inside of the SegmentedPost

            {{count}}           : Translates to total number of NNTPArticles
                                    defined inside of the SegmentedPost

        You can add additionally use your own custom dictionary to which you
        just provide it keys and their translated values.  If you provide any
        custom dictionary, it will always be applied 'before' the built in
        ones.  This allows you to over-ride them if you like.

        All braces are replaced, so in the event braces are detected that
        could not be translated, then they are replaced with 'nothing'

        if strftime is set to True, then the contents of the subject is also
        passed through datetime.strftime() as a last task. this allows custom
        templating to even insert datetime directives if required knowing they
        will be masked shortly thereafter.

        The time strftime() bases itself off is relative to the time specified
        (the actual variable 'relative').  If 'relative' is not specified then
        'now()' is used instead

        The following information was taken from:
                https://docs.python.org/2/library/datetime.html

        It will give you an idea of what datetime objects can be set to. Note
        that datetime.stftime() entries are 'NOT' wrapped in {{ }} objects.

        %a  Weekday as locale’s abbreviated name.
                Sun, Mon, …, Sat (en_US)
                So, Mo, …, Sa (de_DE)

        %A  Weekday as locale’s full name.
                Sunday, Monday, …, Saturday (en_US)
                Sonntag, Montag, …, Samstag (de_DE)

        %w  Weekday as a decimal number, where 0 is Sunday and 6 is Saturday.
                0, 1, …, 6

        %d  Day of the month as a zero-padded decimal number.
                01, 02, …, 31

        %b  Month as locale’s abbreviated name.
                Jan, Feb, …, Dec (en_US)
                Jan, Feb, …, Dez (de_DE)

        %B  Month as locale’s full name.
                January, February, …, December (en_US);
                Januar, Februar, …, Dezember (de_DE)

        %m  Month as a zero-padded decimal number.
                01, 02, …, 12

        %y  Year without century as a zero-padded decimal number.
                00, 01, …, 99

        %Y  Year with century as a decimal number.
                1970, 1988, 2001, 2013

        %H  Hour (24-hour clock) as a zero-padded decimal number.
                00, 01, …, 23

        %I  Hour (12-hour clock) as a zero-padded decimal number.
                01, 02, …, 12

        %p  Locale's equivalent of either AM or PM.
                AM, PM (en_US)
                am, pm (de_DE)

        %M  Minute as a zero-padded decimal number.
                00, 01, …, 59

        %S  Second as a zero-padded decimal number.
                00, 01, …, 59

        %f  Microsecond as a decimal number, zero-padded on the left.
                000000, 000001, …, 999999

        %z  UTC offset in the form +HHMM or -HHMM (empty string if the the
            object is naive).
                (empty), +0000, -0400, +1030

        %Z  Time zone name (empty string if the object is naive).
                (empty), UTC, EST, CST

        %j  Day of the year as a zero-padded decimal number.
                001, 002, …, 366

        %U  Week number of the year (Sunday as the first day of the week) as a
            zero padded decimal number. All days in a new year preceding the
            first Sunday are considered to be in week 0.
                00, 01, …, 53

        %W  Week number of the year (Monday as the first day of the week) as a
            decimal number. All days in a new year preceding the first Monday
            are considered to be in week 0.
                00, 01, …, 53

        %c  Locale’s appropriate date and time representation.
                Tue Aug 16 21:30:00 1988 (en_US);
                Di 16 Aug 21:30:00 1988 (de_DE)

        %x  Locale’s appropriate date representation.
                08/16/88 (None)
                08/16/1988 (en_US)
                16.08.1988 (de_DE)

        %X  Locale’s appropriate time representation.
                21:30:00 (en_US)
                21:30:00 (de_DE)

        %%  A literal '%' character.
                %
        """
        if custom is None:
            custom = {}

        for index, article in enumerate(self.articles):

            # Make a copy of our subject
            subject = self.subject \
                if isinstance(self.subject, basestring) else ''

            # Make a copy of our poster
            poster = self.poster \
                if isinstance(self.poster, basestring) else ''

            if custom:
                # Apply our custom object if one is defined; first escape
                # content
                cmask = {re.escape(str(key)): str(value)
                         for (key, value) in custom.items()}

                # Build ourselves a translation map
                cmask_r = re.compile(
                    r'(' + '|'.join(cmask.keys()) + r')',
                    re.IGNORECASE,
                )

                # Apply our custom masks
                subject = cmask_r.sub(
                    lambda x: cmask[re.escape(x.group())], subject)
                poster = cmask_r.sub(
                    lambda x: cmask[re.escape(x.group())], poster)

            # Initialize ourselves a master translation table
            mask = {
                re.escape('{{count}}'): str(len(self)),
                re.escape('{{index}}'): str(index+1),
                re.escape('{{totalsize}}'): str(article.size()),
            }

            # Our first item can be referenced as
            #   {{filename001}} or {{filename}}
            #
            # Similarily the first time (only) can be reference as
            #   {{filesize001}} or {{filesize}}
            #
            try:
                mask[re.escape('{{filename}}')] = article[0].filename
                mask[re.escape('{{filesize}}')] = str(len(article[0]))

            except IndexError:
                mask[re.escape('{{filename}}')] = ''
                mask[re.escape('{{filesize}}')] = ''

            # for each file, create a filenameNo and filesizeNo
            for no, content in enumerate(article):
                try:
                    mask[re.escape('{{filename%.3d}}' % (no+1))] = \
                        article[0].filename
                    mask[re.escape('{{filesize%.3d}}' % (no+1))] = \
                        str(len(article[0]))

                except IndexError:
                    # No files found
                    mask[re.escape('{{filename%.3d}}' % (no+1))] = ''
                    mask[re.escape('{{filesize%.3d}}' % (no+1))] = ''

            # Build ourselves a translation map
            mask_r = re.compile(
                r'(' + '|'.join(mask.keys()) + r')',
                re.IGNORECASE,
            )

            # Apply our common masks
            subject = mask_r.sub(lambda x: mask[re.escape(x.group())], subject)
            poster = mask_r.sub(lambda x: mask[re.escape(x.group())], poster)

            # Final Tidy
            subject = re.sub('{{[^}]*}}', '', subject)
            poster = re.sub('{{[^}]*}}', '', poster)

            if strftime:
                # Time reference
                _relative = relative
                if _relative is None:
                    # A reference time
                    _relative = datetime.now()

                # apply time support
                subject = _relative.strftime(subject)
                poster = _relative.strftime(poster)

            # Store our subject
            article.subject = subject
            # Store our poster
            article.poster = poster

        return True

    def is_valid(self):
        """
        Iterates over article content an returns True if all of it is valid
        based on it's crc32/md5 and/or whatever mechanism is used to determine
        a NNTPContents validity
        """
        if len(self.articles) == 0:
            # No articles means no validity
            return False

        return next((False for c in self.articles
                     if c.is_valid() is False), True)

    def split(self, size=81920, mem_buf=1048576):
        """
        If there is one Article() and one (valid) NNTPContent() object within
        it, this object will split the Article() into several and break apart
        the contents of the file into several pieces.

        This function returns True if this action was successful, otherwise
        this function returns False.  Newly split content is 'not' in a
        detached form meaning content is removed if the object goes out of
        scope
        """
        if len(self.articles) != 1:
            # Not possible to split a post that is already split
            return False

        articles = self.articles[0].split(size=size, mem_buf=mem_buf)
        if articles is None:
            return False

        # Otherwise store our goods
        self.articles = articles
        return True

    def encode(self, encoders):
        """
        Iterates over all articles and encodes them based on the specified
        encoders.
        """

        # Prepare a new article set
        articles = sortedset(key=lambda x: x.key())
        for article in self.articles:
            result = article.encode(encoders)
            if not result:
                return False

            articles.add(result)

        # Store our new article set
        self.articles = articles

        return True

    def join(self):
        """
        If there are more then one Article() objects containing one valid()
        NNTPContent() object, they can be joined together into one single
        Article() object.

        This function returns True if this action was successful, otherwise
        this function returns False.  Newly joined content is 'not' in a
        detached form meaning content is removed if the object goes out of
        scope
        """

        if len(self.articles) == 1:
            # Nothing to do; no need to fail
            return True

        if len(self.articles) < 1:
            # Not possible to join less than 1 article
            return False

        # acquire the first entry; this will be what we build from
        articles = iter(self.articles)

        # Create a copy of our first object
        head_article = articles.next().copy()

        # Iterate over all our remaining article entries and stack their
        # content onto our head_article
        for content in articles:

            # Append our content
            if not head_article.append(content):

                # Clean up our copy
                del head_article

                # We failed
                return False

        # Reset with a new sorted set of articles
        self.articles.clear()

        # Add our single head_article entry as our primary entry
        self.articles.add(head_article)

        # We're done!
        return True

    def deobsfucate(self, filebase='', codecs=None):
        """
        Using the article information we have, attempt to generate the
        filename to the best of our ability
        """
        if not len(self.articles) > 0:
            # we need at least 1 article
            return None

        if filebase is None:
            # Safety
            filebase = ''

        if codecs is None:
            codecs = self._codecs

        elif isinstance(codecs, CodecBase):
            codecs = [codecs, ]

        # Create our Mime object
        m = Mime()

        # Initialize our objects
        _name = filebase
        _mime = m.from_filename(self.filename)
        _fext = None

        if _mime and _mime.type() != DEFAULT_MIME_TYPE:
            # Store our potential extension
            _fext = _mime.extension()

        if not _name and self.filename:
            # create a filebase
            _name = splitext(basename(self.filename))[0]

        # Because our articles could amount to a lot of crap, we can only hope
        # that one of them provides us with a valid file we can work with.
        # our map allows us to pick and choose what looks good amongst
        # everything else.

        # A sorted set of matches; index 0 is our index our element was found
        # we want this to prevail when match hunting
        ext_map = dict()

        for no, article in enumerate(self.articles):
            # Detect our article _fname
            _fname = article.deobsfucate(filebase=_name, codecs=codecs)
            # Detect our type
            mr = m.from_filename(_fname)
            if mr:
                # Store our tuple if we can
                ext_map[mr.type()] = (no, _fname, mr)

        # Now comes the hard part... if we're lucky we'll have just 2 types at
        # the most (one being DEFAULT_MIME_TYPE).  We can ignore these entries
        # unless they're all we have to pick from.  Otherwise hopefully we
        # have an option #2, that will be our official extension and filename

        # Initialize our findings
        match = None
        for _, potential in ext_map.iteritems():
            if potential[2].type() == DEFAULT_MIME_TYPE or _mime is None:
                if match is None:
                    # We allow this match if we have to
                    match = potential

            elif _mime and _mime.type() == potential[2].type():
                # Our detected mime type is exactly what we expected

                # Store our match
                match = potential

                if not _name and _mime and _mime.type() != DEFAULT_MIME_TYPE:
                    # Update our basename
                    _name = splitext(match[1])[0]

                # Safely break because we can't match any better than this
                break

            elif _mime and _mime.type() == DEFAULT_MIME_TYPE:
                # Update our candidate
                match = potential

        if not match:
            return None

        # use the extension gathered from our match and apply it to our base
        # if we can
        if not _fext:
            _fext = match[2].extension()
            # Return our matched filename
            return '%s%s' % (_name, _fext)

        # Return your file
        return match[1]

    def save(self, filepath=None, copy=False):
        """
        Saves all of the NNTPContent into the directory specified by
        the filepath; if no filepath is specified, then content is written
        into it's work_dir
        """

        for article in self.articles:
            if not article.save(filepath=filepath, copy=copy):
                return False

        return True

    def files(self):
        """
        Returns a list of the files within article
        """
        _files = []
        for article in self.articles:
            _files.extend(article.files())
        return _files

    def size(self):
        """
        return the total size of our articles
        """
        return sum(a.size() for a in self.articles)

    def strsize(self):
        """
        return the total size of our articles
        """
        return bytes_to_strsize(self.size())

    def key(self):
        """
        Returns a key that can be used for sorting with:
            lambda x : x.key()
        """
        return '%s' % self.filename

    def __iter__(self):
        """
        Grants usage of the next()
        """
        # Ensure our stream is open with read
        return iter(self.articles)

    def __len__(self):
        """
        Return the length of the articles
        """
        return len(self.articles)

    def __lt__(self, other):
        """
        Handles less than for storing in btrees
        """
        if self.sort_no is not None:
            # Compare based on a sort number, undefined sorting always
            # trumps those with sorting defined.
            if other.sort_no is not None:
                if self.sort_no == other.sort_no:
                    return str(self.filename) < str(other.filename)

                # Compare our sorting values
                return str(self.sort_no) < str(other.sort_no)

            # If no other sort_no then we are not less than it; those without
            # a sort_no should always trump those with one
            return False

        elif other.sort_no is not None:
            # We don't have a sort_no, and the other does; we're less than it
            # based on our set rules.
            return True

        # If we reach here, neither comparison objects have a sort_no defined.
        # as a result, we just base our match on the filename specified
        return str(self.filename) < str(other.filename)

    def __eq__(self, other):
        """
        Handles equality

        """
        if self.sort_no:
            # Compare based on a sort number, undefined sorting always
            # trumps those with sorting defined.
            if other.sort_no:
                if self.sort_no == other.sort_no:
                    return self.__dict__ == other.__dict__

            # Any other posibility is False
            return False

        elif other.sort_no:
            # they have a sort and we don't. there is no equality in that
            return False

        return self.__dict__ == other.__dict__

    def __getitem__(self, index):
        """
        Support accessing NNTPArticle objects by index
        """
        return self.articles[index]

    def __str__(self):
        """
        Return a printable version of the article
        """
        return '%s' % self.filename

    def __unicode__(self):
        """
        Return a printable version of the article
        """
        return u'%s' % self.filename

    def __repr__(self):
        """
        Return an unambigious version of the object
        """
        if not self.sort_no:
            return '<NNTPSegmentedPost filename="%s" articles=%d />' % (
                self.filename,
                len(self.articles),
            )

        return '<NNTPSegmentedPost sort="%d" filename="%s" articles=%d />' % (
            self.sort_no,
            self.filename,
            len(self.articles),
        ) class NNTPnzb(NNTPContent):
    """
    This class reads and writes to nzb files.

    It can also be passed into a NNTPConnection() get()
    call for the retrieval of articles identified.

    """

    def __init__(self, nzbfile=None, encoding=XML_ENCODING, work_dir=None,
                 mode=NZBParseMode.Simple, codecs=None, *args, **kwargs):
        """
        Initialize NNTP NZB object

        A nzbfile can optionally be specified identifying the location of the
        nzb file to open/read or even eventually write to if you're generating
        one.

        codecs: define the codecs you want to use to parse the articles
                contents
        """

        self._lazy_is_valid = None
        self._lazy_gid = None

        # XML Stream/Iter Pointer
        self.xml_iter = None
        self.xml_root = None
        self.xml_itr_count = 0

        # Meta information placed into (or read from) the <head/> tag
        self.meta = None

        # Encoding to use when handling the NZB File
        self.encoding = encoding

        # Track segmented files when added
        self.segments = sortedset(key=lambda x: x.key())

        # Segments are loaded on our first iteration through the NZB-File
        # Once these are loaded, most of the functions references come from the
        # self.segments instead of re-parsing the XML file again and again
        # We can use self.segments to rebuild a brand new XML file too
        self._segments_loaded = None
        self._segment_iter = None

        # An NZB-File Mode which can control out it responds
        self._nzb_mode = mode

        # Initialize our parent
        super(NNTPnzb, self).__init__(work_dir=work_dir, *args, **kwargs)

        # Used for it's ability to convert to and
        self._htmlparser = None

        # NNTPContent Object
        self._detached = True

        # The nzbfile
        self.filepath = nzbfile

        # Some pretty nzb printing meta information the padding character to
        # use. You probably wouldn't want to set this to anything other then a
        # space (' ') or a tab ('\t').
        self.padding = ' '

        # The multiplier is how many of these padding characters you want to
        # increase by per indent. Hence if you set this to 2 and the padding
        # to a space (' '), then each time a new sub-element is created, it
        # is indented 2 more spaces.
        #
        # Example of padding_multiplier set to 2 (and padding to space):
        #     1|<root>
        #     2|  <level1>
        #     3|    <level2>
        #     4|      <level3>
        #     5|      </level3>
        #     6|    </level2>
        #     7|  </level1>
        #     8|</root>
        #
        self.padding_multiplier = 2

        # Create our Mime object since we'll be using it a lot
        self._mime = Mime()

        # Load our Codecs
        self._codecs = codecs

        # Provide default codecs if required
        if not self._codecs:
            self._codecs = [CodecYenc(), ]

        elif isinstance(self._codecs, CodecBase):
            self._codecs = [self._codecs, ]

    def save(self, nzbfile=None, pretty=True, dtd_type=XMLDTDType.Public):
        """
        Write an nzbfile to the file and path specified. If no path is
        specified, then the one used to open the class is used.

        If that wasn't specified, then this function will return False.
        The function returns True if the save was successful

        If pretty is set to True then the output is formatted gently
        on the eyes; otherwise it is packed for disk size
        """
        if self.open(filepath=nzbfile, mode=NNTPFileMode.BINARY_RW_TRUNCATE):
            eol = '\n'
            indent = ''

            self.write('<?xml version="%s" encoding="%s"?>%s' % (
                XML_VERSION,
                self.encoding,
                eol,
            ))
            self.write('<!DOCTYPE %s %s %s>%s' % (
                XML_DOCTYPE,
                dtd_type,
                NZB_XML_DTD_MAP[dtd_type],
                eol,
            ))

            if not pretty:
                # No more end-of-lines are nessisary from this point on if
                # we're not out to format the content in any human-readable
                # fashion
                eol = ''

            self.write('<nzb xmlns="%s">%s%s' % (
                NZB_XML_NAMESPACE,
                eol, eol,
            ))

            if self.meta:
                if pretty:
                    indent = ''.ljust(self.padding_multiplier, self.padding)

                # Handle the meta information if there is anything at all to
                # print
                self.write('%s<head>%s' % (
                    indent,
                    eol,
                ))

                if pretty:
                    indent = ''.ljust(self.padding_multiplier*2, self.padding)

                for k, v in self.meta.items():
                    self.write('%s<meta type="%s">%s</meta>%s' % (
                        indent,
                        self.escape_xml(k.encode(self.encoding)),
                        self.escape_xml(v.encode(self.encoding)),
                        eol,
                    ))

                if pretty:
                    indent = ''.ljust(self.padding_multiplier, self.padding)

                self.write('%s</head>%s%s' % (
                    indent,
                    eol,
                    eol,
                ))

            for segno, segment in enumerate(self.segments):
                if not len(segment):
                    logger.error(
                        "NZB-File '%s' has no defined articles." % self.path())
                    self.remove()
                    return False

                if not len(segment.groups):
                    logger.warning(
                        "NZB-File '%s' (segno %d) has no defined groups." % (
                            self.path(),
                            segno,
                        ),
                    )

                if pretty:
                    indent = ''.ljust(self.padding_multiplier, self.padding)

                self.write('%s<file poster="%s" date="%s" subject="%s">%s' % (
                    indent,
                    self.escape_xml(segment.poster),
                    segment.utc.strftime('%s'),
                    self.escape_xml(segment.subject),
                    eol,
                ))

                if pretty:
                    indent = ''.ljust(
                        self.padding_multiplier*2, self.padding)

                self.write('%s<groups>%s' % (
                    indent,
                    eol,
                ))

                if pretty:
                    indent = ''.ljust(
                        self.padding_multiplier*3, self.padding)

                for group in segment.groups:
                    self.write('%s<group>%s</group>%s' % (
                        indent,
                        group,
                        eol,
                    ))

                if pretty:
                    indent = ''.ljust(
                        self.padding_multiplier*2, self.padding)
                self.write('%s</groups>%s' % (
                    indent,
                    eol,
                ))

                self.write('%s<segments>%s' % (
                    indent,
                    eol,
                ))

                for part, article in enumerate(segment):
                    if pretty:
                        indent = ''.ljust(
                            self.padding_multiplier*3, self.padding)

                    for attachment in article:
                        # use enumerated content and not the part assigned.
                        # this is by design because it gives developers the
                        # ability to add/remove items from the attachment and
                        # only use the part numbers for their own personal
                        # ordering.

                        _bytes = len(attachment)
                        self.write(
                            '%s<segment %snumber="%d">%s</segment>%s' % (
                                indent,
                                (('bytes="%d" ' % _bytes) if _bytes else ''),
                                part+1,
                                self.escape_xml(article.msgid()),
                                eol,
                            )
                        )

                if pretty:
                    indent = ''.ljust(self.padding_multiplier*2, self.padding)

                self.write('%s</segments>%s' % (
                    indent,
                    eol,
                ))

                if pretty:
                    indent = ''.ljust(self.padding_multiplier, self.padding)

                self.write('%s</file>%s%s' % (
                    indent,
                    eol, eol,
                ))

            if pretty:
                indent = ''

            self.write('%s<!-- Generated by %s v%s -->%s' % (
                indent, __title__, __version__, eol,
            ))

            self.write('%s</nzb>%s' % (
                indent,
                eol,
            ))
            # close our file
            self.close()
            return True

        logger.error("NZB-File '%s' could not be accessed." % self.path())
        return False

    def gid(self):
        """
        Returns the Global Identifier associated with an NZB File.
        This is just a unique way of associating this file with another
        posted to usenet.

        None is returned if the GID can not be acquired

        """
        if self._lazy_gid is None:

            if self._segments_loaded is True:
                # Use our segments already loaded in memory
                if len(self.segments) > 0:
                    try:
                        # Use the md5 hash of the first message-id of the
                        # first segment
                        _md5sum = hashlib.md5()
                        _md5sum.update(self.segments[0][0].msgid())

                        # Store our data
                        self._lazy_gid = _md5sum.hexdigest()

                    except (TypeError, AttributeError, IndexError):
                        # Can't be done
                        logger.warning(
                            'Cannot calculate GID from NZB-Segments: %s' %
                            self.filepath)

                return self._lazy_gid

            if self.is_valid() is False:
                # Save ourselves the time and cpu of parsing further
                return None

            # get ourselves the gid which is just the md5sum of the first
            # Article-ID
            iter(self)

            if self.xml_iter is None:
                return None

            # get the root element
            try:
                _, self.xml_root = self.xml_iter.next()

                segment = self.xml_root.xpath(
                    '/ns:nzb/ns:file[1]/ns:segments/ns:segment[1]',
                    namespaces=NZB_LXML_NAMESPACES,
                )[0]

                self.xml_root.clear()
                self.xml_root = None
                self.xml_itr_count = 0
                self.xml_iter = None

            except IndexError:
                logger.warning(
                    'NZB-File is missing initial </segment> element: %s' %
                    self.filepath,
                )
                # Thrown if no segment elements were found in the first file
                # entry; this NZBFile is not good.

                # We intentionally do not mark the invalidity of the situation
                # because we allow for nzbfiles that are still being
                # constructed.
                return None

            except StopIteration:
                logger.warning(
                    'NZB-File is missing </file> elements: %s' % self.filepath)
                # Thrown if no <file/> elements were found at all.
                # This NZBFile is not good.

                # We intentionally do not mark the invalidity of the situation
                # because we allow for nzbfiles that are still being
                # constructed.
                return None

            except IOError:
                logger.warning('NZB-File is missing: %s' % self.filepath)
                self.xml_root = None
                self.xml_itr_count = 0
                # Mark situation
                self._lazy_is_valid = False

            except XMLSyntaxError as e:
                logger.error("NZB-File '%s' is corrupt" % self.filepath)
                logger.debug('NZB-File XMLSyntaxError Exception %s' % str(e))
                # Mark situation
                self._lazy_is_valid = False

            except Exception:
                logger.error("NZB-File '%s' is corrupt" % self.filepath)
                logger.debug('NZB-File Exception %s' % str(e))
                # Mark situation
                self._lazy_is_valid = False
                return None

            try:
                # Use the md5 hash of the first message-id of the
                # first segment
                _md5sum = hashlib.md5()
                _md5sum.update(segment.text.strip())

                # Store our data
                self._lazy_gid = _md5sum.hexdigest()

            except (TypeError, AttributeError):
                # Can't be done
                logger.warning(
                    'Cannot calculate GID from NZB-File: %s' % self.filepath)

        return self._lazy_gid

    def load(self, filepath=None, mode=NZBParseMode.Simple, detached=True):
        """
        Loads segments into memory (this object)
        """

        # Reset all entries
        self.segments.clear()
        self._segments_loaded = None
        self._segment_iter = None
        self._nzb_mode = mode

        if filepath is not None:
            # Reset our variables
            self._lazy_is_valid = None
            self._lazy_gid = None
            self.close()

            if not super(NNTPnzb, self).load(filepath=filepath):
                self._segments_loaded = False
                return False

        for sp in self:
            # Load our segmented post entries into memory
            self.segments.add(sp)

        # Toggle our segments loaded flag
        self._segments_loaded = True

        return True

    def is_valid(self):
        """
        Validate if the NZB File is okay; this will generate some overhead
        but at the same time it caches a lot of the results it returns so
        future calls will be speedy

        The function returns True if the nzb file is valid, otherwise it
        returns False
        """

        if self._lazy_is_valid is None:
            if self.open():
                # Open DTD file and create dtd object
                dtdfd = open(NZB_XML_DTD_FILE)
                dtd = etree.DTD(dtdfd)
                # Verify our dtd file against our current stream
                try:
                    nzb = etree.parse(self.filepath)

                except XMLSyntaxError as e:
                    if e[0] is not None:
                        # We have corruption
                        logger.error(
                            "NZB-File '%s' is corrupt" % self.filepath)
                        logger.debug(
                            'NZB-File XMLSyntaxError Exception %s' % str(e))
                        # Mark situation
                        self._lazy_is_valid = False
                        # We failed
                        return False

                self._lazy_is_valid = dtd.validate(nzb)

        return self._lazy_is_valid is True

    def escape_xml(self, unescaped_xml, encoding=None):
        """
        A Simple wrapper to Escape XML charaters from a string
        """

        if isinstance(unescaped_xml, unicode):
            # Encode content as per defined in our XML
            if encoding is None:
                encoding = self.encoding
            unescaped_xml = unescaped_xml.encode(encoding)

        return sax_escape(unescaped_xml, {"'": "&apos;", "\"": "&quot;"})

    def add(self, obj):
        """
        Adds an NNTPSegmentedPost to an nzb object
        """

        if not isinstance(obj, NNTPSegmentedPost):
            return False

        # duplicates are ignored in a blist and therefore
        # we just capture the length of our list before
        # and after so that we can properly return a True/False
        # value
        _bcnt = len(self.segments)

        # Add our segment
        self.segments.add(obj)

        # A reference value we can use to compare against
        file_count = len(self.segments)

        # We're manually loading the segments so we can trust wherever
        # we leave off
        self._segments_loaded = True

        return file_count > _bcnt

    def unescape_xml(self, escaped_xml, encoding=None):
        """
        A Simple wrapper to UnEscape XML characters from a string
        """

        if encoding is None:
            encoding = self.encoding

        if self._htmlparser is None:
            # Define on first use
            self._htmlparser = HTMLParser()

        return self._htmlparser.unescape(escaped_xml).decode(encoding)

    def segcount(self):
        """
        Returns the total number of segments in the NZB File
        """
        seg_count = 0
        seg_count += sum(len(c) for c in self)
        return seg_count

    def deobsfucate(self, filebase=None):
        """
        Scans through the segments, articles and content associated with an
        NZB-File and sets up the filenames defined in the segments to it's
        best effort

        filebase: provide a fallback filename base (the part of the file
                  before the extension) to build on if we can't detect
                  the file on our own.

        """
        # The name from the meta tag
        _name = self.meta.get('name', '').decode(self.encoding).strip()
        if not _name:
            if filebase:
                # Use base provided as a base
                _name = filebase

            else:
                # If we can't get a name from the meta tag to start with, then
                # we use the NZB-Filename itself as a backup
                _name = splitext(basename(self.path()))[0]

        for segment in self.segments:
            filename = segment.deobsfucate(_name)
            if filename:
                # Update
                segment.filename = filename

    def next(self):
        """
        Python 2 support
        Support stream type functions and iterations
        """

        # We track our iterator since we move along if our mode tells us to do
        # so.
        _iter = None

        if self.xml_root is not None:
            # clear our unused memory
            self.xml_root.clear()

        if self._segment_iter:
            while 1:
                _iter = self._segment_iter.next()
                if self._valid_by_mode(_iter):
                    return _iter

        # get the root element
        try:
            _, self.xml_root = self.xml_iter.next()

            # Increment our iterator
            self.xml_itr_count += 1

        except StopIteration:
            # let this pass through
            self.xml_root = None
            self.xml_itr_count = 0

        except IOError:
            logger.warning('NZB-File is missing: %s' % self.filepath)
            self.xml_root = None
            self.xml_itr_count = 0
            # Mark situation
            self._lazy_is_valid = False

        except XMLSyntaxError as e:
            if e[0] is not None:
                # We have corruption
                logger.error("NZB-File '%s' is corrupt" % self.filepath)
                logger.debug('NZB-File XMLSyntaxError Exception %s' % str(e))
                # Mark situation
                self._lazy_is_valid = False
            # else:
            # this is a bug with lxml in earlier versions
            # https://bugs.launchpad.net/lxml/+bug/1185701
            # It occurs when the end of the file is reached and lxml
            # simply just doesn't handle the closure properly
            # it was fixed here:
            # https://github.com/lxml/lxml/commit\
            #       /19f0a477c935b402c93395f8c0cb561646f4bdc3
            # So we can relax and return ok results here
            self.xml_root = None
            self.xml_itr_count = 0

        except Exception as e:
            logger.error("NZB-File '%s' is corrupt" % self.filepath)
            logger.debug('NZB-File Exception %s' % str(e))
            # Mark situation
            self._lazy_is_valid = False

        if self.xml_root is None or len(self.xml_root) == 0:
            self.xml_iter = None
            self.xml_root = None
            self.xml_itr_count = 0

            raise StopIteration()

        if self.meta is None:
            # Attempt to populate meta information
            self.meta = {}

            for meta in self.xml_root.xpath('/ns:nzb/ns:head[1]/ns:meta',
                                            namespaces=NZB_LXML_NAMESPACES):
                # Store the Meta Information Detected
                self.meta[meta.attrib['type'].decode(self.encoding)] = \
                    self.unescape_xml(meta.text.strip())

        # Acquire the Segments Groups
        groups = [
            group.text.strip().decode(self.encoding)
            for group in self.xml_root.xpath(
                'ns:groups/ns:group',
                namespaces=NZB_LXML_NAMESPACES,
            )
        ]

        # The detected filename
        _filename = ''

        # The name from the meta tag
        _name = self.meta.get('name', '').decode(self.encoding).strip()

        if not _name and self.filepath is not None:
            # Lets try to generate a name frome our NZB-File
            tmpfname = basename(self.filepath)

            # Strip our extension off the end (if present)
            result = NZB_EXTENSION_RE(tmpfname)
            if result and result.group('fname'):
                # Store our new filename as our name
                _name = result.group('fname')

        # Subject
        _subject = self.unescape_xml(
                self.xml_root.attrib.get('subject', '')).decode(self.encoding)

        # Poster
        _poster = self.unescape_xml(
                self.xml_root.attrib.get('poster', '')).decode(self.encoding)

        # Use our Codec(s) to extract our Yenc Subject
        matched = None
        for c in self._codecs:
            # for each entry, parse our article
            matched = c.parse_article(
                subject=_subject,
                poster=_poster,
            )
            if matched:
                # We matched
                break

        if matched:
            # We succesfully got a filename from our subject line
            _filename = matched.get('fname', '').strip()
            if _filename and _name:
                # always allow the name to over-ride the detected filename if
                # we actually have a real name we can assciated with it by
                _ext = self._mime.extension_from_filename(_filename)
                if _ext:
                    _filename = '{0}{1}'.format(_name, _ext)

        # Initialize a NNTPSegmented File Object using the data we read
        _file = NNTPSegmentedPost(
            _filename,
            poster=_poster,
            epoch=self.xml_root.attrib.get('date', '0'),
            subject=_subject,
            groups=groups,
            work_dir=self.work_dir,
            sort_no=self.xml_itr_count,
        )

        # index tracker
        _last_index = 0

        # Now append our segments
        for segment in self.xml_root.xpath(
                'ns:segments/ns:segment', namespaces=NZB_LXML_NAMESPACES):

            _cur_index = int(segment.attrib.get('number', _last_index+1))
            try:
                _size = int(segment.attrib.get('bytes'))
                if _size < 0:
                    _size = 0

            except (TypeError, ValueError):
                _size = 0

            article = NNTPArticle(
                subject=_file.subject,
                poster=_file.poster,
                id=self.unescape_xml(segment.text),
                no=_cur_index,
                work_dir=self.work_dir,
                codecs=self._codecs,
            )

            # Store our empty content Placeholder
            article.add(
                NNTPEmptyContent(
                    filepath=_filename,
                    part=self.xml_itr_count,
                    total_size=_size,
                    work_dir=self.work_dir,
                )
            )

            # Add article
            _file.add(article)

            # Track our index
            _last_index = _cur_index

        if not self._valid_by_mode(_file):
            # Not used; recursively move along
            return self.next()

        # Return our object
        return _file

    def _valid_by_mode(self, segmented):
        """
        Internally called function that takes a segmented object and matches
        it against the mode defined by this class. We return True if the mode
        would otherwise accept this entry, otherwise we return False.
        """

        if self._nzb_mode == NZBParseMode.Simple:
            # We accept everything
            return True

        # Our Mime Response
        mr = None

        # Check our segmented content to see if it's a par-file
        if segmented.filename:
            # use filename
            mr = self._mime.from_filename(segmented.filename)

        if mr is None or mr == DEFAULT_MIME_TYPE:
            try:
                for article in segmented:
                    for content in article:
                        if content.filename:
                            mr = self._mime.from_filename(content.filename)

                        if mr is None or mr == DEFAULT_MIME_TYPE:
                            mr = self._mime.from_file(content.path())

                        if not (mr is None or mr == DEFAULT_MIME_TYPE):
                            raise StopIteration()

            except StopIteration:
                # we're done
                pass

        if mr is None or mr == DEFAULT_MIME_TYPE:
            # We don't know the type, so we can't do much here
            return True

        if (self._nzb_mode & NZBParseMode.IgnorePars):
            # our mime type is going to be one of the following
            #  - application/par2
            #  - application/x-par2
            #
            # it wil always be in lower case, so the following should
            # get our match for us:
            if mr.type().endswith('par2'):
                return False

        if (self._nzb_mode & NZBParseMode.IgnoreAttached):
            if segmented[0][0].is_attached():
                return False

        # If we get here, we can go ahead and process our file
        return True

    def __next__(self):
        """
        Python 3 support
        Support stream type functions and iterations
        """
        return self.next()

    def __iter__(self):
        """
        Grants usage of the next()
        """

        # First get a ptr to the head of our data
        super(NNTPnzb, self).__iter__()

        if self._segments_loaded is True:
            self._segment_iter = iter(self.segments)
            return self

        elif len(self.segments) > 0:
            # make sure we don't have any segments lingering
            self.segments.clear()

        if self.xml_root is not None:
            # clear our unused memory
            self.xml_root.clear()
            # reset our variable
            self.xml_root = None
            self.xml_itr_count = 0

        try:
            self.xml_iter = iter(etree.iterparse(
                self.filepath,
                tag="{%s}file" % NZB_XML_NAMESPACE,
            ))

        except IOError:
            logger.warning('NZB-File is missing: %s' % self.filepath)
            # Mark situation
            self._lazy_is_valid = False

        except XMLSyntaxError as e:
            if e[0] is not None:
                # We have corruption
                logger.error("NZB-File '%s' is corrupt" % self.filepath)
                logger.debug('NZB-File Exception %s' % str(e))
                # Mark situation
                self._lazy_is_valid = False
            # else:
            # this is a bug with lxml in earlier versions
            # https://bugs.launchpad.net/lxml/+bug/1185701
            # It occurs when the end of the file is reached and lxml
            # simply just doesn't handle the closure properly
            # it was fixed here:
            # https://github.com/lxml/lxml/commit\
            #       /19f0a477c935b402c93395f8c0cb561646f4bdc3
            # So we can relax and return ok results here

        except Exception as e:
            logger.error("NZB-File '%s' is corrupt" % self.filepath)
            logger.debug('NZB-File Exception %s' % str(e))
            # Mark situation
            self._lazy_is_valid = False

        return self

    def size(self):
        """
        Returns the calculated size of all of the contents residing in the
        NZB-File.  If it has not been downloaded yet, the size is calculated
        based on the contents of the NZB-File

        """
        if self._segments_loaded:
            return sum(s.size() for s in self.segments)

        else:
            return sum(c.size() for c in self)

    def __len__(self):
        """
        Returns the number of files in the NZB File
        """
        return len(self.segments) if self._segments_loaded is True\
            else sum(1 for c in self)

    def __getitem__(self, index):
        """
        Support accessing NNTPSegmentedPost objects by index
        """
        if self._segments_loaded is None:
            # Attempt to preload our content
            self.load()

        return self.segments[index]

    def __repr__(self):
        """
        Return a printable version of the file being read
        """
        return '<NNTPnzb filename="%s" />' % (
            self.filepath,
        ) class NNTPGetDatabase(Database):
    """
    A managment class to handle Downloaded Articles
    """

    def __init__(self, engine=None, reset=None):
        """
        Initialize NNTP Retrieval Database
        """
        super(NNTPGetDatabase, self).__init__(
            base=ObjectBase,
            vsp=Vsp,
            engine=engine,
            reset=reset,
        ) class NNTPConnection(SocketBase):
    """
        NNTPConnection is a class that wraps and eases the comunication
        to and from an NNTP Server.

        - Usenet Compression supported (GZIP) if the usenet server allows
          for it.

        - SSL Support (you'll ideally want to use the default of TLSv1
          for security reasons; but can support SSLv3 too if the remote
          server can't handle TLSv1).

        - It uses btree+ to store all of it's fetched results using a
          fantastic addon called blist which never maded it into
          python (https://www.python.org/dev/peps/pep-3128/)

        - can seek_by_date() using a recusive binary style searching
          which can narrow in great time!

        - all socket i/o parsed through a BytesIO block to
          save on the constent allocation strings would normally
          have caused. Content is then further parsed and broken into
          an easy to parse dictionary (stored in a btree) and
          returned for the user for parsing.

        - any filters specified must be of type NNTPFilterBase. If a
          list or tuple is passed in, then each element in the list
          must be inheritied from NNTPFilterBase.
    """
    # 10MB Chunk Sizes read from the stream;  If content is compressed
    # it has the potential to get quite larger.
    MAX_BUFFER_SIZE = 10485760

    # The number of characters to read backwards at a time when scanning
    # for a new line character during situations where we've received
    # partial data we don't want to process yet. A new line delimiter
    # found allows us to safely process what is before it.
    READBACK_CHUNK_SIZE = 128

    def __init__(self, username=None, password=None, secure=False,
                 iostream=NNTPIOStream.RFC3977_GZIP,
                 join_group=False, use_body=False, use_head=True,
                 encoding=None, work_dir=None,
                 filters=None, hooks=None, *args, **kwargs):
        """
        Initialize NNTP Connection

        secure
        ------
        Secure connection with NNTP Server using a SSL/TLS Encryption

        iostream
        --------
        NewsReap was developed in order to communicate with an NNTP server.
        The idea behind the IOStream is to allow us to handle other protocols
        too. At this time, there are only really 2 values to specify here:
           - NNTPIOStream.RFC3977_GZIP : Support GZIP Compression wrapped
                                          around the standard RFC3977 (NNTP)
                                          protocol.

           - NNTPIOStream.RFC3977      : Standard RFC3977 (NNTP) Protocol

        join_group
        ----------
        Some NNTP Servers have gone out of their way to make sure that all of
        the Message-ID's associated with an article is unique which eliminates
        the need to have to select the group prior to retrieving the article.

        It's obviously slower to do this; so if you're Usenet provider
        doesn't need this option; then don't specify it. It 'will' slow down
        processing if you set this!

        use_body
        --------
        This actually refers to quering the NNTP server for it's articles
        via the BODY command over the ARTICLE.  If this is set to true, only
        the body's content is downloaded.  This is faster; however the trade
        off is using the ARTICLE option hands down the Header information too.

        The Header information is really useful for scanning more details about
        the content we're retrieving.  Sometimes the header information can
        provide us enough information to no bother downloading the rest of the
        content.  Hence having this set to false can actually be faster in
        some cases. By default this is set to False so we can acquire as much
        information on the article we're retrieving as we can dispite the
        small overhead that it comes with.
        """

        # get connection mode
        if secure:
            kwargs['secure'] = True
            self.protocol = 'nntps'
        else:
            kwargs['secure'] = False
            self.protocol = 'nntp'

        # Default Filters
        self.filters = []

        # Filter Preparation
        if isinstance(filters, NNTPFilterBase):
            self.filters.append(filters)

        elif isinstance(filters, (list, tuple)):
            self.filters.extend([
                x for x in filters if isinstance(x, NNTPFilterBase)
            ])

        if hooks:
            # Force defined hooks into arguments
            kwargs['hooks'] = hooks

        # Initialize the Socket Base Class
        super(NNTPConnection, self).__init__(*args, **kwargs)

        # Store the default encoding
        self.encoding = encoding
        if not self.encoding:
            self.encoding = NNTP_DEFAULT_ENCODING

        # SSL Does not like unicode characters, so we convert the content
        # to a characterset so we don't get any handshaking errors
        if isinstance(username, unicode):
            username = username.encode(self.encoding)
        self.username = username

        # SSL Does not like unicode characters, so we convert the content
        # to a characterset so we don't get any handshaking errors
        if isinstance(password, unicode):
            password = password.encode(self.encoding)
        self.password = password

        # Attempt to gzip the connection
        if isinstance(iostream, basestring):
            try:
                self._iostream = iostream.lower()
                if self._iostream not in NNTP_SUPPORTED_IO_STREAMS:
                    # Default
                    self._iostream = NNTPIOStream.RFC3977_GZIP
                    logger.warning('An unknown iostream was specified; '
                                   "using default: '%s'." % self._iostream)

            except (TypeError, ValueError):
                # Default
                self._iostream = NNTPIOStream.RFC3977_GZIP
                logger.warning('A malformed iostream was specified; '
                               "using default: '%s'." % self._iostream)

        elif iostream:
            # Default
            logger.warning('An invalid iostream was specified; '
                           "using default: '%s'." % self._iostream)
            self._iostream = NNTPIOStream.RFC3977_GZIP

        else:
            # iostream is None (0, or False)
            # used RFC3977 Standards but without Compresssion
            logger.info('An invalid iostream was specified; '
                        "using default: '%s'." % self._iostream)
            self._iostream = NNTPIOStream.RFC3977

        # can_post is a flag that gets set after authenticating
        # with the news server.  The server responds with the
        # posting information which is set in this flag
        self.can_post = False

        # Article Meta information (Extracted from post)
        self.article = {}

        # Article End Of Data
        # This has 3 states:
        #   * False: The end of the article has not been found; scan and
        #            save what we find. We 'always' start in this state
        #   * None:  The end of the article has not been found; but due
        #            to a problem, we don't want anything else that is
        #            returned by the server.  This flag is usually set
        #            during parsing if CRC errors occur as a way of
        #            letting the server complete what it's doing.
        #   * True:  The end of the article has been found; we are in
        #            a somewhat of completed state
        self.article_eod = False

        # Article Filename
        self.article_fname = None

        # Data that has been successfully read from the
        # _buffer is processed (perhaps uncompressed)
        # and placed into the _data stream.
        self._data = BytesIO()

        # A _data length tracker which is used with
        # trying to track of we've processed all of the data yet
        # or not.
        self._data_len = 0

        # Temporary Buffer of read (unprocessed) data
        self._buffer = BytesIO()

        # Our response code (integer) returned by the NNTPServer on the last
        # request made
        self.last_resp_code = None

        # Our response string returned by the NNTPServer on the last request
        # made
        self.last_resp_str = ''

        # A flag set to True if the payload to follow the response is gzipped.
        # GZipped responses require a bit more CPU crunching but much less
        # bandwidth. This value is set to None if there simply wasn't a payload
        # at all (or an initialization value)
        self.payload_gzipped = None

        self.lines = []
        self.line_count = 0

        self.group_name = None
        self.group_count = 0
        self.group_head = 0
        self.group_tail = 0
        self.group_index = 0

        # use HEAD command vs STAT when retrieving information on an article
        # on usenet.
        self.use_head = use_head

        # Use the BODY command vs ARTICLE when retrieving the data itself.
        # The subtle difference is BODY only returns the raw contents where as
        # ARTICLE returns the header information too.  The header information
        # can provide invaluable information on whether the BODY is even worth
        # retrieving in some cases.  It's purely up to the user which they
        # want to use.  It's also possible that some servers won't support
        # one command over the other.
        self.use_body = use_body

        # Some usenet servers do not require the group to be joined prior to
        # fetching it's content.
        self.join_group = join_group

        # A List of backup servers that attempt to fetch missing content; use
        # append() to append to the list.  The entry must be of type
        # NNTPConnection()
        self._backups = []

        # Used to cache group list responses
        self._grouplist = None

        # Default Working Directory
        # All temporary content is downloaded to this location.  If set to
        # None then the defaults are used instead.
        if work_dir is None:
            self.work_dir = DEFAULT_TMP_DIR
        else:
            self.work_dir = abspath(expanduser(work_dir))

    def append(self, connection, *args, **kwargs):
        """
        Add a backup NNTP Server (Block Account) which is only
        queried in the event content can't be pulled from the
        primary server.
        """
        if not isinstance(connection, NNTPConnection):
            return False

        # block/backup connections are established on demand
        self._backups.append(connection)
        return True

    def connect(self, *args, **kwargs):
        """
        Establishes a connection to an NNTP Server
        """
        if self.connected:
            # nothing to see here
            return True

        # Reset tracking items
        self._soft_reset()

        # initialize our return flag
        status = False

        try:
            # pre_connect hook
            response = self.hooks.call('pre_connect', *args, **kwargs)

            # Allow a response over-ride which forces a denied connection
            if next((r for r in response
                     if r is not False), None) is not False:

                # call _connect()
                status = self._connect(*args, **kwargs)

        except SocketRetryLimit:
            # We can not establish a connection and never will
            return False

        finally:
            self.hooks.call(
                'post_connect',
                status=status,
                conection=weakref.proxy(self)
            )

        return status

    def _connect(self, *args, **kwargs):
        """
        _connect() performs the actual connection with little
        error checking.  This is intended so that if we lose the
        connection we can attempt to resume where we left off
        gracefully.
        """

        try:
            if not super(NNTPConnection, self).connect(*args, **kwargs):
                # Return Fail
                return False

        except (SocketException, SignalCaughtException):
            # Return Fail
            return False

        # Receive Initial Welcome Message
        response = self._recv(timeout=NNTP_WELCOME_MESSAGE_TIMEOUT)
        if response.code not in NNTPResponseCode.SUCCESS:
            logger.error(
                'Failed to establish a handshake response from server.',
            )
            # server error
            self.close()
            return False

        # Tests for specific flags that exist in the welcome message that
        # give us indication on whether or not we can post to usenet or
        # not.
        if POSTING_OK_RE.search(response.code_str):
            # Toggle can_post flag
            logger.info('NNTP Posting enabled.')
            self.can_post = True
        else:
            logger.info('NNTP Posting disabled.')

        # Authenticate
        response = self.send('AUTHINFO USER %s' % self.username)
        if response.code not in NNTPResponseCode.PENDING:
            # We got a 400 or 500 error, no good
            logger.error('The specified username was not accepted.')
            self.close()
            return False

        response = self.send('AUTHINFO PASS %s' % self.password)
        if response.code not in NNTPResponseCode.SUCCESS:
            # We failed to get a 200 error; assume an authentication error
            logger.error('The specified password was not accepted.')
            self.close()
            return False

        logger.info('NNTP USER/PASS Handshake was successful.')

        if self._iostream == NNTPIOStream.RFC3977_GZIP:
            # Do Compression
            response = self.send('XFEATURE COMPRESS GZIP')
            if response.code not in NNTPResponseCode.SUCCESS:
                # Not supported; flip flag
                logger.warning('NNTP Compression not supported.')
                self._iostream = NNTPIOStream.RFC3977
            else:
                logger.info('NNTP Compression enabled.')

        if self.join_group and self.group_name:
            # Change to our group
            self.group(self.group_name)

        return True

    def post(self, payload, update_headers=True, success_only=False):
        """
        Allows posting content to a NNTP Server

        If success_only is set to True, then the result set returned only
        contains the entries of successfully posted content.  otherwise
        the result set returns the actual response code for each
        entry posted.

        """

        # A sorted list of all articles pulled down
        results = sortedset(key=lambda x: x.key())

        postable = []
        if isinstance(payload, (set, tuple, sortedset, list)):
            # iterate over all items and append them to our resultset
            for entry in payload:
                _results = self.post(entry, update_headers=True)
                if _results is not None:
                    # Append our results
                    results |= _results

        elif isinstance(payload, NNTPArticle):
            postable = [payload, ]

        elif isinstance(payload, NNTPSegmentedPost):
            postable = iter(payload)

        # Initialize a bound respose object
        response = None

        for content in postable:
            if content not in results:

                try:
                    weak_content = weakref.proxy(content)

                except TypeError:
                    # Some types just can't be converted into a weak reference
                    # no problem...
                    weak_content = content

                try:
                    # pre hook
                    _response = self.hooks.call(
                        'pre_post',
                        content=weak_content,
                    )

                    # Allow a response over-ride which forces a denied
                    # post
                    if next((r for r in _response
                             if r is not False), None) is not False:

                        response = self._post(content, update_headers=True)

                    else:
                        response = NNTPResponse(
                            NNTPResponseCode.HOOK_OVERRIDE,
                            'NNTP POST action blocked by hook',
                        )

                except SocketException:
                    # Connection Lost
                    self.close()

                    # Setup Response
                    response = NNTPResponse(
                        NNTPResponseCode.CONNECTION_LOST,
                        'Connection Lost',
                    )

                finally:
                    # Point our body to our content
                    response.body = weakref.proxy(content)

                    self.hooks.call(
                        'post_post',
                        content=weak_content,
                        response=weakref.proxy(response),
                        status=response in NNTPResponseCode.SUCCESS,
                    )

                if success_only:
                    if response in NNTPResponseCode.SUCCESS:
                        results.add(response)
                else:
                    # Store all response types
                    results.add(response)

        # Return our results
        return results

    def _post(self, article, update_headers=True):
        """
        Takes an NNTPArticle() and posts it to an NNTP Server if it can

        """

        if not self.connected:
            # Attempt to establish a connection
            if not self.connect():
                logger.error('Could not establish a connetion to NNTP Server.')
                return NNTPResponse(
                    NNTPResponseCode.NO_CONNECTION, 'No Connection',
                )

        if not self.can_post:
            # 480 Transfer permission denied
            return NNTPResponse(480, 'Transfer permission denied.')

        if not isinstance(article, NNTPArticle):
            # Not postable content
                return NNTPResponse(
                    NNTPResponseCode.INVALID_INPUT,
                    'You may only post an NNTPArticle.',
                )

        # get an iterator
        post_iter = article.post_iter(update_headers=update_headers)
        if not post_iter:
            return NNTPResponse(
                NNTPResponseCode.INVALID_INPUT,
                'NNTPArticle() is missing data (not postable).',
            )

        # send the POST keyword to the remote server
        response = self.send('POST')
        if response.code not in NNTPResponseCode.PENDING:
            # We got a 400 or 500 error, no good
            logger.error('Could not post content / %s' % response)
            return response

        # If we reach here we're read to go
        xfer_total_bytes = 0
        for chunk in post_iter:
            xfer_bytes = super(NNTPConnection, self).send(chunk)
            if xfer_bytes != len(chunk):
                # uh-oh
                return NNTPResponse(436, 'Transfer failed.')
            xfer_total_bytes += xfer_bytes

        # Send our EOD and capture our response
        response = self.send(NNTP_EOL + NNTP_EOD)

        return response

    def group(self, name):
        """
        Changes to a specific group

        This function returns a tuple in the format of:

            (count, head, tail, group_name)

        If an error occurs, then the following is returned:

            (None, None, None, group_name)

        """

        if not self.connected:
            # Attempt to establish a connection
            if not self.connect():
                logger.error('Could not establish a connetion to NNTP Server.')
                return None

        if isinstance(name, unicode):
            # content should be byte encoded
            name = name.encode(self.encoding)

        response = self.send('GROUP %s' % name)

        self.group_name = name
        self.group_count = 0
        self.group_head = 0
        self.group_tail = 0
        self.group_index = 0

        if response.code in NNTPResponseCode.SUCCESS:
            match = NNTP_GROUP_RESPONSE_RE.match(response.code_str)
            if match:
                # We matched!
                # return is count, first, end, name
                self.group_count = int(match.group('count'))
                self.group_head = int(match.group('low'))
                self.group_tail = int(match.group('high'))
                self.group_name = match.group('name')
                self.group_index = self.group_head

                logger.info('Using Group: %s.' % self.group_name)
                return (
                    self.group_count,
                    self.group_head,
                    self.group_tail,
                    self.group_name,
                )

        logger.warning('Bad Group: %s' % name)
        return (None, None, None, self.group_name)

    def groups(self, filters=None, lazy=True):
        """
        Retrieves a list of groups from the server and returns them in an easy
        to parse dictionary:

              {
                  group: 'alt.binaries.test',
                  head: 0,
                  tail: 100,
                  count: 100,
                  flags: ['y'],
              }

        If filter(s) are specified, then the list is filtered based on
        the specified entries.

        The filters can be in the form of a string or regular expression.
        If multiple strings are specified (in a tuple or list format) then
        they will all be checked for a match. If fiters are specified then
        at least 1 filter must match.

        To fetch just binaries you could type:
            groups(filters='alt.binaries')

        If lazy is set to true; the last returned fetch is used instead
        which greatly increases the speed when you need to make multiple
        calls to this function. By defaut the lazy flag is enabled.
        """

        if not self.connected:
            # Attempt to establish a connection
            if not self.connect():
                logger.error('Could not establish a connetion to NNTP Server.')
                return None

        # Filter Management
        if not filters:
            filters = tuple()

        elif isinstance(filters, basestring):
            filters = (filters,)

        elif isinstance(filters, re._pattern_type):
            filters = (filters,)

        elif not isinstance(filters, (tuple, list)):
            filters = list(filters)

        # Iterate over each item in the list and compile it now
        _filters = []
        for _filter in filters:
            if isinstance(_filter, re._pattern_type):
                _filters.append(_filter)

            elif isinstance(_filter, basestring):
                try:
                    _filter = r'^.*%s.*$' % re.escape(_filter)
                    _filters.append(
                        re.compile(_filter, flags=re.IGNORECASE),
                    )
                    logger.debug('Compiled group regex "%s"' % _filter)

                except:
                    logger.error(
                        'Invalid group regular expression: "%s"' % _filter,
                    )
            else:
                logger.error(
                    'Ignored group expression: "%s"' % _filter,
                )

        if self._grouplist is None or not lazy:
            # Send LIST ACTIVE command
            response = self.send('LIST ACTIVE', decoders=[
                CodecGroups(work_dir=self.work_dir),
            ])
            if not response.is_success(multiline=True):
                logger.error('Failed to interpret NNTP LIST ACTIVE response.')
                # could not retrieve list
                return None

            # Pop the group list off
            grouplist = response.decoded.pop()
            if not isinstance(grouplist, NNTPMetaContent):
                # could not retrieve list
                logger.error('Failed to fetch NNTP LIST ACTIVE content.')
                return None

            # Lazy Caching for speed
            self._grouplist = grouplist.content
            logger.info('Cached %d group(s)' % len(self._grouplist))

        if _filters:
            cleaned = []
            for x in self._grouplist:
                # Apply filter if nessisary
                if not next((True for f in _filters
                             if f.search(x['group']) is not None), False):
                    # Filters were specified and the matched
                    # group did not match what was specified so
                    # we can move onto the next entry
                    continue

                # if we reach here, then we store our group
                cleaned.append(x)

            logger.info('Retrieved %d (and filtered %d) groups(s)' % (
                len(self._grouplist), len(cleaned),
            ))
            return cleaned

        logger.info('Retrieved %d groups(s)' % (
            len(self._grouplist),
        ))
        return self._grouplist

    def tell(self):
        """
        Returns the current index
        """
        return self.group_index

    def seek_by_date(self, refdate, group=None):
        """
        Similar to the seek() function in the sense it changes the
        current index pointer.  However this one attempts to narrow
        into the index location by a date.

        This function returns the first message id to scan from
        based on the time specified.  If no articles are found
        or something bad happens, false is returned.

        refdate must be a datetime object.

        As a user you don't need to set the last 2 flags. In fact
        it might be more advisable not to unless you know what you're
        doing.  Just call the function by setting the date to what you
        want and let the function manage it's own recursive actions.

            from datetime import datetime
            from datetime import timedelta

            NNTPConnection n(username, password, localhost)
            n.connect(localhost)
            # look back 40 days from 'now'
            n.seek_by_date(datetime.now()-timedelta(days=40))

        Seeking takes on average 15 to 16 queries in a group containing
        a size of 33 million records. So it's pretty optimized in that
        reguard. Here is it's Big-O representation:

            O(log n + log**2 k)
        """

        if group is not None and group != self.group_name:
            # allow us to switch groups if nessisary
            if self.group(group)[0] is None:
                # Could not select group
                logger.error(
                    'DATESEEK Could not select group %s' % group)
                return None

        elif group is not None and self.group_name is None:
            logger.error(
                'DATESEEK XOVER failed due to no group selected.')
            return None

        if not self.connected:
            return None

        index = self._seek_by_date(refdate)
        if index < 0:
            logger.warning(
                'No entries found at (or after) date(%s) in group %s' % (
                    refdate.strftime('%Y.%m.%d %H:%M:%S'),
                    group,
                ),
            )
            self.group_index = self.group_head
            return self.group_head

        self.group_index = index
        logger.info('Matched index: %d' % (self.group_index))
        return self.group_index

    def _seek_by_date(self, refdate, head=None, tail=None):
        """

        This ideally shouldn't be called directly; users should
        call the seek_by_date() function instead

        refdate must be a datetime object.

        How it's done:
        Using recursion; we drill down to a specific location but
        since usenet 'sort-of' sorted by time. Once we get within
        X records (defined by a global variable) we need to scan
        within those results to match our time (or as close as we
        can get to it without going over. Since our content is
        stored in a btree; this is relatively speedy.

        """

        if head is None:
            # Initial Entry
            head = self.group_head

        if tail is None:
            # Initial Entry
            tail = self.group_tail

        logger.debug('Scanning for index at %s in %d article(s)' % (
            refdate.strftime('%Y-%m-%d %H:%M:%S'),
            tail - head,
        ))

        # initalize filter count
        filter_count = tail - head

        # Get the middle
        start = head + (filter_count / 2) - (NNTP_FETCH_BY_DATE_MAX_MISSES / 2)
        end = start + min(NNTP_FETCH_BY_DATE_MAX_MISSES, filter_count)

        response = self.xover(
            start=start,
            end=end - 1,
            sort=XoverGrouping.BY_TIME,
        )

        if response is None:
            # Nothing Retrieved
            return -1
            # if response.code != 423:
            #    # 423 means there were no more items to fetch
            #    # this is a common error that even this class produces
            #    # and therefore we do not need to create a verbose
            #    # message from it.
            #    logger.error('NNTP Server responded %s' % response)
            # return -1

        # Deal with our response
        if len(response):
            # Get string equivalent
            _refdate = refdate.strftime('%Y%m%d%H%M%S:0000000000')

            # Get keys
            # (this returns an blist.sortedset() object): Big O(n)
            # This is 'very' fast operation since we're not dealing with a
            # dict()
            _refkeys = response.keys()

            # logger.debug('total=%d, left=%s, right=%s, ref=%s' % (
            #    end-start,
            #    _refkeys[0],
            #    _refkeys[-1],
            #    _refdate,
            # ))

            #
            # Decisions
            #
            if end - start < NNTP_FETCH_BY_DATE_MAX_MISSES and \
               len(_refkeys) < NNTP_FETCH_BY_DATE_MAX_MISSES:
                # We're in great shape if we reach here
                # It means we've narrowed off our search to just
                # the small window for our match
                return _refkeys.bisect_left(_refdate) + start

            elif _refkeys[-1] < _refdate:
                # look closer to the tail (our right)
                logger.debug('%s > %s / JUMP RIGHT' % (
                    _refdate,
                    _refkeys[-1],
                ))

                if tail - end > 0:
                    result = self._seek_by_date(refdate, end - 1, tail)
                else:
                    result = tail

                while result < 0:
                    # SHIFT LEFT
                    logger.debug('SHIFT LEFT')
                    tail -= ((tail - end) / 2)
                    if tail - end > 0:
                        result = self._seek_by_date(refdate, end - 1, tail)
                    else:
                        result = tail
                        break

                return result

            elif _refkeys[0] >= _refdate:
                # look closer to the front
                logger.debug('%s < %s / JUMP LEFT' % (
                    _refdate,
                    _refkeys[0],
                ))

                if start - head > 1:
                    result = self._seek_by_date(refdate, head, start)
                else:
                    result = start

                while result < 0:
                    # SHIFT RIGHT
                    logger.debug('SHIFT RIGHT')
                    head += ((start - head) / 2)
                    if start - head > 1:
                        result = self._seek_by_date(refdate, head, start)
                    else:
                        result = start
                        break

                return result

        # We recursively scanned too far in one
        # direction; we need to
        return -1

    def seek(self, index, whence=None):
        """
        Sets a default nntp index
        TODO: if wence is set then the index is set with respect
              to the results.

        seek() always returns the current index similar to what tell()
        would return if it was called right afterwards
        """
        if index <= self.group_tail:
            if index >= self.group_head:
                self.group_index = index
            else:
                # Can't be less than the head
                self.group_index = self.group_head
        else:
            # Can't be greater than the tail
            self.group_index = self.group_tail

        return self.group_index

    def next(self, count=50000):
        """
        A wrapper to the xover but using a counter offset based
        on a set index value

        use the seek() function to over-ride the default
        index value (which is usually set to the head of
        group.

        """
        # Control Count gracefully
        count = min(count, self.group_tail - self.group_index)

        if count <= 0:
            # 423: Empty Range
            return NNTPResponse(423, 'Empty Range')

        # Fetch our results and adjust our pointer accordingly
        response = self.xover(self.group_index, self.group_index + count - 1)
        if response:
            self.group_index += count

        # Return results
        return response

    def prev(self, count=50000):
        """
        A wrapper to the xover but using a counter offset based
        on a set index value

        use the seek() function to over-ride the default
        index value (which is usually set to the head of
        group.

        """
        # Control Count gracefully
        count = min(
            count, self.group_tail - self.group_index - self.group_head)

        if count <= 0:
            # 423: Empty Range
            return NNTPResponse(423, 'Empty Range')

        # Fetch our results and adjust our pointer accordingly
        response = self.xover(self.group_index - count, self.group_index - 1)
        if response:
            self.group_index -= count

        # Return results
        return response

    def xover(self, group=None, start=None, end=None,
              sort=XoverGrouping.BY_POSTER_TIME):
        """
        xover
        Returns a NNTPRequest object

        RFC: http://tools.ietf.org/html/rfc2980
        list is a list of tuples, one for each article in the range delimited

        If the start or end time are set to `None` then they default to the
        low/high (head or tail) watermark respectively.

        by the start and end article numbers. Each tuple is of the form
        (article number, subject, poster, date, id, references, size, lines).

        If the start or end time is a datetime variable, then additional
        overhead is performed to find these locations in the group identified.

        Content is parsed as follows:
              {
                  id: u'the unique identifier',
                  article_no: 12345678,
                  poster: u'the poster's information',
                  date: datetime() object,
                  subject: u'a subject line in unicode',
                  size: 2135  // the message size in bytes
                  lines: 53   // the number of lines
                  group: u'alt.group.one'
                  xgroups : {
                      // references the Message-ID (id) per cross post
                       u'alt.group.two': 987654321,
                       u'alt.group.three': 12341234,
                  }
              }
        """

        if group is not None and group != self.group_name:
            # allow us to switch groups if nessisary
            if self.group(group)[0] is None:
                # Could not select group
                logger.error('XOVER Could not select group %s' % group)
                return None

        elif group is not None and self.group_name is None:
            logger.error('XOVER failed due to no group selected.')
            return None

        if end is None:
            end = self.group_tail

        elif isinstance(end, datetime):
            # Get the end by date
            end = self.seek_by_date(end)

        # Default Ranges (if not otherwise specified)
        if start is None:
            start = self.group_head

        elif isinstance(start, datetime):
            # Get the start by date
            start = self.seek_by_date(start)

        if start > end:
            # 423: Empty Range
            logger.error('Invalid XOVER start/end (%d/%d) range.' % (
                start,
                end,
            ))
            return None

        elif not (self.group_head and start >= self.group_head):
            # 423: Empty Range
            logger.error('Invalid XOVER head/start (%d/%d) range.' % (
                self.group_head,
                start,
            ))
            return None

        elif not (self.group_tail and end <= self.group_tail):
            # 423: Empty Range
            logger.error('Invalid XOVER tail/end (%d/%d) range.' % (
                self.group_tail,
                end,
            ))
            return None

        try:
            # pre hook
            _response = self.hooks.call(
                'pre_xover',
                group=group,
                start=start,
                end=end,
            )

            # Allow a response over-ride which forces a denied xover
            if next((r for r in _response
                     if r is not False), None) is not False:

                response = self.send(
                    'XOVER %d-%d' % (start, end),
                    decoders=[
                        CodecArticleIndex(
                            filters=self.filters,
                            sort=sort,
                            encoding=self.encoding,
                            work_dir=self.work_dir,
                        ),
                    ],
                    retries=NNTP_XOVER_RETRIES,
                )

            else:
                response = NNTPResponse(
                    NNTPResponseCode.HOOK_OVERRIDE,
                    'NNTP XOVER action blocked by hook',
                )

        finally:

            try:
                weak_response = weakref.proxy(response)

            except TypeError:
                # Some types just can't be converted into a weak reference
                # no problem...
                weak_response = response

            self.hooks.call(
                'post_xover',
                group=group,
                start=start,
                end=end,
                response=weak_response,
                status=response in NNTPResponseCode.SUCCESS_MULTILINE,
            )

        if response.code not in NNTPResponseCode.SUCCESS_MULTILINE:
            logger.error('Failed to interpret NNTP XOVER response.')
            return None

        try:
            # return our decoded content
            return response.decoded.pop().content

        except IndexError:
            # TODO: Debug; wtf is going on
            return None

    def stat(self, id, full=None, group=None):
        """
        A wrapper to _stat that allows us to manage hooks

        """

        if full is None:
            # default
            full = self.use_head

        if self.join_group and group is not None and group != self.group_name:
            # allow us to switch groups if nessisary
            if self.group(group)[0] is None:
                # Could not select group
                logger.error('Could not select group %s' % group)
                return None

        # Initialize a bound object
        response = None

        try:
            _response = self.hooks.call(
                'pre_stat',
                content=id,
                full=True,
                group=self.group_name,
            )

            # Allow a response over-ride which forces a denied stat
            if next((r for r in _response
                     if r is not False), None) is not False:

                response = self._stat(id=id, full=full)

            else:
                response = NNTPResponse(
                    NNTPResponseCode.HOOK_OVERRIDE,
                    'NNTP STAT action blocked by hook',
                )

        finally:
            # Prepare our status for our hook call
            status = False
            if isinstance(response, NNTPHeader):
                status = response.article_exists()

            try:
                weak_response = weakref.proxy(response)

            except TypeError:
                # Some types just can't be converted into a weak reference
                # no problem...
                weak_response = response

            self.hooks.call(
                'post_stat',
                content=id,
                full=full,
                group=self.group_name,
                response=weak_response,
                status=status,
            )

        # Return our response
        return response

    def _stat(self, id, full=None):
        """
        A Simple check to return True of False on whether an article
        exists or not. The function returns:
            False:  if the article does not exist.
            None:   if there was no way to determine the answer.

            Otherwise the function returns a dictionary containing the
            stats the news server had on the file in question.

            if Full is left to None, then the results will be based
            on the defaut use_head function.
        """

        if not full:
            response = self.send('STAT <%s>' % id)
            if response.is_success(multiline=False):
                # we're good to go, return what we do know so it fits
                results = NNTPHeader()
                results['Message-ID'] = id
                return results

        else:
            # Force decoders to just be the header
            response = self.send(
                'HEAD <%s>' % id,
                decoders=[
                    CodecHeader(
                        encoding=self.encoding,
                        work_dir=self.work_dir,
                    ),
                ],
            )

            if response.is_success(multiline=True):
                # Return our content
                return response.decoded.pop()

        if response.code in NNTPResponseCode.NO_ARTICLE:
            if self._backups:
                # Try our backup servers in the sequential order they were
                # added in; if they all fail; then we return None
                logger.warning(
                    'ARTICLE <%s> not found; checking backups.' % id,
                )
                return next((
                    b.article for b in self._backups
                    if b.stat(id=id, full=full, group=self.group_name)
                    not in (None, False)), False)

            logger.warning('ARTICLE <%s> not found.' % id)
            return False

        elif response.code in NNTPResponseCode.SERVER_ERROR:
            # We have a problem
            self.close()
            logger.error('NNTP Error %s' % response)
            if self._backups:
                # Try our backup servers in the sequential order they were
                # added in; if they all fail; then we return None
                return next((
                    b.article for b in self._backups
                    if b.stat(id=id, full=full, group=self.group_name)
                    not in (None, False)), None)
            return None

        # Return
        return True

    def get(self, id, work_dir=None, decoders=None, group=None, max_bytes=0):
        """
        A wrapper to the _get call allowing support for more then one type
        of object (oppose to just _get() which only accepts the message id

        This function returns the articles as a sortedset() containing the
        downloaded content based on what was passed in.

        max_bytes is set to the number of bytes you want to have received
        before you automatically abort the connection and flag the object as
        having only been partially complete  Use this option when you need to
        inspect the first bytes of a binary file. Set this to zero to download
        the entire thing (this is the default value)

        """
        if work_dir is None:
            # Default
            work_dir = self.work_dir

        if isinstance(id, basestring):
            # We're dealing a Message-ID (Article-ID)
            return self._get(
                id=id,
                work_dir=work_dir,
                decoders=decoders,
                group=group,
                max_bytes=max_bytes,
            )

        # A sorted list of all articles pulled down
        results = sortedset(key=lambda x: x.key())

        if isinstance(id, (set, tuple, sortedset, list)):
            # iterate over all items and append them to our resultset
            for entry in id:
                _results = self._get(
                    id=id.id,
                    work_dir=work_dir,
                    decoders=decoders,
                    group=group,
                    max_bytes=max_bytes,
                )
                if _results is not None:
                    # Append our results
                    results |= _results

        elif isinstance(id, NNTPArticle):
            # Support NNTPArticle Objects if they have an id defined
            if id.id:
                return self._get(
                    id=id.id,
                    work_dir=work_dir,
                    decoders=decoders,
                    group=group,
                    max_bytes=max_bytes,
                )

        elif isinstance(id, NNTPSegmentedPost):
            # Support NNTPSegmentedPost() Objects

            for part_no, _article in enumerate(id):
                # We need to download each article define
                if self.join_group:
                    for group in id.groups:
                        # Try each group
                        article = self._get(
                            id=_article,
                            work_dir=work_dir,
                            decoders=decoders,
                            group=group,
                            max_bytes=max_bytes,
                        )

                        if article:
                            # Found
                            break

                    # If we reach here we failed to fetch the item, so
                    # we'll try the next group
                    logger.warning(
                        'Failed to fetch segment #%.2d (%s)' % (
                            _article.no,
                            _article.filename,
                        ),
                    )

                    # mark our article invalid as part of it's response
                    article._is_valid = False

                else:
                    # Fetch our content
                    article = self._get(
                        id=_article,
                        work_dir=work_dir,
                        decoders=decoders,
                        max_bytes=max_bytes,
                    )

                if article:
                    if len(article) == 0:
                        logger.warning(
                            'No content found in segment #%.2d (%s)' % (
                                _article.no, _article.filename,
                            ),
                        )

                else:
                    # Uh oh, we failed to get anything; so just add
                    # our current article generated from the nzb file.
                    # Mark the object invalid and re-add it
                    logger.warning(
                        'Failed to retrieve segment (%s)' % (
                            id.filename,
                        ),
                    )

                    # mark our article invalid as part of it's response
                    article._is_valid = False

                # Store information from our nzb_article over top of
                # the contents in the new article we retrieved
                _article.load(article)

                # Store our article
                results.add(_article)

        elif isinstance(id, NNTPnzb):
            # We're dealing with an NZB File
            if not id.is_valid():
                return None

            # We iterate over each segment defined int our NZBFile and merge
            # them into 1 file. We do this until we've processed all the
            # segments and we return a list of articles
            for segpost_no, segpost in enumerate(id):
                for article in iter(segpost):
                    _results = self.get(article)
                    if _results is not None:
                        # Append our results
                        results |= _results

        else:
            # Unsupported
            return None

        # Return our results
        return results

    def _get_wrapper(self, id, work_dir, decoders=None, group=None,
                     max_bytes=0):
        """
        A wrapper to the __get() function so that we can capture it
        with our hooks

        """
        # Initialize a bound object
        response = None
        try:
            weak_content = weakref.proxy(id)

        except TypeError:
            # Some types just can't be converted into a weak reference
            # no problem...
            weak_content = id

        try:
            _response = self.hooks.call(
                'pre_get',
                work_dir=work_dir,
                group=group,
                max_bytes=max_bytes,
                content=weak_content,
            )

            # Allow a response over-ride which forces a denied stat
            if next((r for r in _response
                     if r is not False), None) is not False:

                response = self._get(
                    id=id,
                    work_dir=work_dir,
                    group=group,
                    max_bytes=max_bytes,
                )

            else:
                response = NNTPResponse(
                    NNTPResponseCode.HOOK_OVERRIDE,
                    'NNTP GET action blocked by hook',
                )

        finally:
            # Prepare our status for our hook call
            status = False
            if isinstance(response, NNTPArticle):
                status = response.header.article_exists()

            try:
                weak_response = weakref.proxy(response)

            except TypeError:
                # Some types just can't be converted into a weak reference
                # no problem...
                weak_response = response

            self.hooks.call(
                'post_get',
                content=id,
                group=self.group_name,
                response=weak_response,
                status=status,
            )

        # Return our response
        return response

    def _get(self, id, work_dir, decoders=None, group=None, max_bytes=0):
        """
        Download a specified message to the work_dir specified. This function
        returns an NNTPArticle() object if it can.

        None is returned if the content could not be retrieved due to an error
        or DMCA.

        This function will always attempt to retrieve missing content from any
        identified backup servers in the order they were saved.

        max_bytes is set to the number of bytes you want to have received
        before you automatically abort the connection and flag the object as
        having only been partially complete  Use this option when you need to
        inspect the first bytes of a binary file. Set this to zero to download
        the entire thing (this is the default value)

        """

        if self.join_group and group is not None and group != self.group_name:
            # allow us to switch groups if nessisary
            if self.group(group)[0] is None:
                # Could not select group
                logger.error('Could not select group %s' % group)
                return None

        if not isdir(work_dir) and not mkdir(work_dir):
            logger.error('Could not create directory %s' % work_dir)
            return None

        if decoders is None:
            # Prepare a default list of decoders
            decoders = list()
            if not self.use_body:
                # BODY calls pull down header information too
                decoders.append(
                    CodecHeader(encoding=self.encoding, work_dir=work_dir),
                )

            decoders.extend([
                # Yenc Encoder/Decoder
                CodecYenc(work_dir=work_dir, max_bytes=max_bytes),
                # UUEncoder/Decoder
                CodecUU(work_dir=work_dir, max_bytes=max_bytes),
            ])

        elif decoders is False:
            # Disable all decoders
            decoders = list()

        elif isinstance(decoders, CodecBase):
            decoders = [decoders, ]

        if self.use_body:
            # Body returns the contents past the header;  Hence there will be
            # no header information to parse if this is the option used.

            # This is faster, but does not allow for some extra checking
            # we can do.
            response = self.send('BODY <%s>' % id, decoders=decoders)
        else:
            # Article retrives the same content as Body plus the Header
            # too.
            response = self.send('ARTICLE <%s>' % id, decoders=decoders)

        if response.is_success(multiline=True):
            # we're good to go!
            pass

        elif response.code in NNTPResponseCode.NO_ARTICLE:
            logger.warning('ARTICLE <%s> not found.' % id)
            if self._backups:
                # Try our backup servers in the sequential order they were
                # added in; if they all fail; then we return None
                return next(
                    (b.article for b in self._backups
                        if b.get(id, work_dir, group) is not None), None)
            return None

        else:  # response.code in NNTPResponseCode.SERVER_ERROR:
            # Close our connection
            self.close()

            logger.error('NNTP Fetch <%s> / %s' % (id, response))

            if self._backups:
                # Try our backup servers in the sequential order they were
                # added in; if they all fail; then we return None
                return next(
                    (b.article for b in self._backups
                        if b.get(id, work_dir, group) is not None), None)
            return None

        # If we reach here, we have data we can work with; build our article
        # using the content retrieved.
        article = NNTPArticle(id=id, work_dir=work_dir)
        article.load(response)

        # Return the content retrieved
        return article

    def send(self, command, timeout=None, decoders=None, retries=0):
        """
        A Simple wrapper for sending NNTP commands to the server

        send() always returns the result of _recv() an internal function
        used to parse the content returned from the NNTP Server.

        article is only used if a list is passed in and the expected
        results are a multi-line response. It is populated with the
        data returned.
        """

        if not self.connected:
            # Attempt to establish a connection
            if not self.connect():
                logger.error('Could not establish a connetion to NNTP Server.')
                return NNTPResponse(
                    NNTPResponseCode.NO_CONNECTION,
                    'No Connection',
                )

        match = NNTP_AUTHINFO_CHECK.match(command)
        if not match:
            logger.debug('send(%s)' % command)

        else:
            logger.debug('send(AUTHINFO %s ********)' % (
                match.group('key'),
            ))

        total_retries = retries
        while True:
            # Soft reset in preparation for returned results
            self._soft_reset()

            if not super(NNTPConnection, self).send(command + EOL):
                break

            try:
                response = self._recv(timeout=0, decoders=decoders)

            except SocketException:
                # Connection Lost
                self.close()

                # Setup Response
                response = NNTPResponse(
                    NNTPResponseCode.CONNECTION_LOST,
                    'Connection Lost',
                )

            if retries > 0:
                # We have a retry left; depending on the severity of our return
                # code, we may try again
                if response.code in (
                        NNTPResponseCode.FETCH_ERROR,
                        NNTPResponseCode.BAD_RESPONSE,
                        NNTPResponseCode.NO_CONNECTION,
                        NNTPResponseCode.CONNECTION_LOST):
                    # If we reach here; our return code is considered
                    # recoverable;

                    # decrement our retry counter
                    retries -= 1

                    # Reset our decoders for re-use
                    for decoder in decoders:
                        decoder.reset()

                    logger.warning(
                        'Received NNTP error %d; retrying... (%d/%d)' % (
                            response.code,
                            total_retries - retries,
                            total_retries,
                        ))
                    continue

            return response

        return NNTPResponse(
            NNTPResponseCode.BAD_RESPONSE,
            'Invalid Command: "%s"' % command,
        )

    def _recv(self, decoders=None, timeout=None):
        """ Receive data, return #bytes, done, skip

        The response always returns the content in the format of
         (code, data)

        if response is set to an NNTPResponse object, it's populated
        with the data retrieved.
        """

        if not self.connected:
            logger.debug('_recv() %d: %s' % (
                NNTPResponseCode.NO_CONNECTION, 'No Connection',
            ))
            return NNTPResponse(
                NNTPResponseCode.NO_CONNECTION, 'No Connection',
            )

        if self.article_eod is True:
            # We've completed
            return NNTPResponse(self.last_resp_code, self.last_resp_str)

        # Pointers used to help identify the start and end
        # of the datablock read from the News Server
        head_ptr = 0

        # Pointer into the _buffer at the last new line
        # as we don't want to grab content from a line that hasn't
        # completely downloaded yet
        tail_ptr = 0

        # Tracks the size of the buffer
        total_bytes = 0

        # Our response object
        response = None

        # Our decoded result
        decoded = None

        # The maximum allowable bytes we can parse before we abort
        # this is calculated from the decoders
        max_bytes = 0

        if not decoders:
            decoders = []

        elif isinstance(decoders, CodecBase):
            decoders = [decoders, ]

        if len(decoders):
            # If at least 1 decoder is defined, we scan it for it's
            # max_bytes() value which tells us the maximum # of bytes we're
            # expecting before completing the transaction (early).  max_bytes()
            # is set from get() and _get() calls.

            # If we determine the end user has specified a max_bytes value,
            # then we adjust the total buffer read size.  This allows us to
            # force an earlier processing.  The minimum buffer size can never
            # be less than 2048
            max_bytes = max([d.max_bytes() for d in decoders])
            if max_bytes > 0:
                if max_bytes < NNTP_MIN_READ_BUFFER_SIZE:
                    self.MAX_BUFFER_SIZE = NNTP_MIN_READ_BUFFER_SIZE
                else:
                    self.MAX_BUFFER_SIZE = min(
                        [max_bytes, self.MAX_BUFFER_SIZE])

        logger.debug('Read Buffer set to %d bytes' % self.MAX_BUFFER_SIZE)

        #  We track the last codec activated using the codec_active
        #  variable.
        codec_active = None

        while not self.article_eod:
            # This loop really only takes effect if we aren't forced
            # to just wait for a server command line. Otherwise
            # we break out the first second we have data to process

            self._buffer.seek(0, SEEK_END)

            try:
                self._buffer.write(self.read(
                    max_bytes=self.MAX_BUFFER_SIZE - total_bytes,
                    timeout=timeout, retry_wait=None,
                ))

            except (SocketException, SignalCaughtException):
                logger.debug('_recv() Connection Lost')
                return NNTPResponse(
                    NNTPResponseCode.CONNECTION_LOST,
                    'Connection Lost',
                )

            # Some Stats (TODO)
            _bytes = self._buffer.tell() - total_bytes
            total_bytes += _bytes
            logger.debug('_recv() %d byte(s) read.' % (_bytes))

            # # DEBUG START
            # self._buffer.seek(head_ptr, SEEK_SET)
            # logger.debug('Characters "%s"' % \
            #     ", ".join(['0x%0x' % ord(b) for b in self._buffer.read()]))
            # # DEBUG END

            ##################################################################
            #                                                                #
            #  Step: 1: Get the status code from the NNTP Server if we don't #
            #           already have it.                                     #
            #                                                                #
            ##################################################################
            if not self.last_resp_code:

                # Reset our payload flag
                self.payload_gzipped = None

                # Seek to head of buffer
                self._buffer.seek(head_ptr, SEEK_SET)

                # Extract Header Response
                data = self._buffer.readline()

                # Only the first time do we remove the first entry
                # because this is our header
                match = NNTP_RESPONSE_RE.match(data.strip())
                if match:
                    # Generate Reponse code
                    self.last_resp_code = int(match.group('code'))
                    self.last_resp_str = match.group('desc')

                    # Adjust head ptr to end of line
                    head_ptr = self._buffer.tell()

                else:
                    if self.connected:
                        # Generate erroneous reponse code
                        self.last_resp_code = NNTPResponseCode.FETCH_ERROR
                        if len(data):
                            self.last_resp_str = 'Bad Response: (len=%d)' % (
                                len(data),
                            )
                        else:
                            self.last_resp_str = 'Bad Response: <nothing>'

                        # Drop connection
                        self.close()

                    else:
                        self.last_resp_code = NNTPResponseCode.CONNECTION_LOST
                        if len(data):
                            self.last_resp_str = \
                                'Connection Lost: (len=%d)' % (len(data))
                        else:
                            self.last_resp_str = 'Connection Lost: <nothing>'

                    # Toggle End of Article and Flush Results
                    self.article_eod = True

                    logger.debug('_recv() %d: %s' % (
                        self.last_resp_code, self.last_resp_str))

                    # Return our response
                    return NNTPResponse(
                        self.last_resp_code, self.last_resp_str,
                    )

                logger.debug('_recv() %d: %s' % (
                    self.last_resp_code, self.last_resp_str))

                # Detect Compression from NNTP response if possible
                if GZIP_COMPRESS_RE.search(self.last_resp_str):
                    logger.debug('_recv() Detected compressed payload.')
                    # GZIP response type; toggle our flag
                    self.payload_gzipped = True

                else:
                    # Uncompressed response type; toggle our flag
                    self.payload_gzipped = False

            # Non-Multiline responses mean were done now.
            if self.last_resp_code not in NNTPResponseCode.SUCCESS_MULTILINE:
                # We're done; flush results and move on
                self.article_eod = True

                # Safety: just clean anything remaining
                while self.can_read():
                    try:
                        self.read(timeout=timeout, retry_wait=None)

                    except (SocketException, SignalCaughtException):
                        # Connection lost
                        return NNTPResponse(
                            NNTPResponseCode.NO_CONNECTION,
                            'Connection Lost',
                        )

                # Return our response
                return NNTPResponse(self.last_resp_code, self.last_resp_str)

            # Store our can_read() flag
            can_read = self.can_read(3)

            if response is None:
                # Initialize our response object on first iteration
                # through loop
                response = NNTPResponse(
                    self.last_resp_code,
                    self.last_resp_str,
                    work_dir=self.work_dir,
                )

            # We have multi-line code to store fill our buffer before
            # proceeding.
            if can_read and total_bytes < self.MAX_BUFFER_SIZE:
                # Keep storing content until we've either reached the end
                # or filled our buffer
                # logger.debug('_recv() Data pending on server...')
                continue

            ##################################################################
            #                                                                #
            #  Step: 2: Check for END OF DATA (EOD) and END OF LINE (EOL).   #
            #           If we find any; trim them and mark flags accordingly #
            #                                                                #
            ##################################################################

            # Initialize our EOL Flag
            eol = False

            if not can_read and total_bytes > 0:
                # Offset
                offset = min(5, total_bytes)

                # The below accomplishes the same results as abs()
                # but does it at a much faster speed (screw looking pythonic)
                if offset > 0:
                    offset = -offset

                self._buffer.seek(offset, SEEK_END)

                # Now we read what we can from the buffer
                data = self._buffer.read()

                if offset == -5:
                    # Adjust offset if nessisary
                    offset += 1

                # Check for the End of Data
                eod_results = EOD_RE.search(data)
                if eod_results:
                    # We can truncate here to trim the EOD off
                    self._buffer.truncate(
                        total_bytes - len(eod_results.group(1)),
                    )

                    # Correct Total Length
                    total_bytes = self._buffer.seek(0, SEEK_END)

                    # Toggle flag so we can break out
                    self.article_eod = True

                    # Toggle our end of line flag incase it doesn't
                    # match below (this is okay)
                    eol = True

                    # Update Offset
                    offset = min(2, total_bytes)

                    # The below accomplishes the same results as abs()
                    # does but at a much faster speed (screw looking
                    # pythonic)
                    if offset > 0:
                        offset = -offset

                    self._buffer.seek(offset, SEEK_END)

                    # Now we read what we can from the buffer
                    data = self._buffer.read()

                # Check for end of line
                eol_results = EOL_RE.search(data)
                if eol_results:
                    # We can truncate here to trim the EOL off
                    self._buffer.truncate(
                        total_bytes - len(eol_results.group(1)),
                    )

                    # Correct Total Length
                    total_bytes = self._buffer.seek(0, SEEK_END)

                    # We have our end of line
                    eol = True

                    if self.last_resp_code in NNTPResponseCode.SUCCESS \
                       and not self.can_read():
                        # If we're still not seeing more data pending on the
                        # server; then we can safely assume we're done. This
                        # handles the situations where we retrieved a
                        # multi-line response code yet we didn't get one.
                        #
                        # the GROUP NNTP server command is known to return 211
                        # which just returns 1 line, yet the command LISTGROUP
                        # also returns 211 too and returns a listing
                        self.article_eod = True

            ##################################################################
            #                                                                #
            #  Step: 3: We now have to handle situations where we never      #
            #           found the EOD or EOL markers. This is a perfectly    #
            #           okay situation so long as our buffer is full.        #
            #                                                                #
            #           If our buffer is full, then we need to mark the tail #
            #           of our buffer because we don't want to handle the    #
            #           content that hasn't completely downloaded yet.       #
            #                                                                #
            ##################################################################

            # the tail_ptr is used to track the last officially completed line
            # as we don't want to process a line that hasn't completely
            # downloaded yet.
            tail_ptr = total_bytes

            if not self.payload_gzipped and \
               total_bytes >= self.MAX_BUFFER_SIZE and not eol:
                # We have a full buffer and there is most likely
                # a lot more content to still download; we need to find
                # the last new line

                # Number of characters to look back (chunk):
                chunk_size = self.READBACK_CHUNK_SIZE

                # Current Matched offset (<0 means no match yet)
                offset = -1

                # Our reference pointer
                ref_ptr = tail_ptr - head_ptr
                while offset < 0:
                    # Keep iterating backwards by chunk sizes
                    # until we get a match

                    # a few checks to make sure we don't look past
                    # our head_ptr
                    chunk_size = min(ref_ptr - head_ptr, chunk_size)
                    ref_ptr -= (chunk_size + 1)

                    if ref_ptr < head_ptr:
                        # Nothing more to look back at; we reached
                        # the head of our buffer. do nothing; just let
                        # process all of the data we received.
                        break

                    chunk_ptr = self._buffer.seek(ref_ptr, SEEK_SET)
                    offset = self._buffer.read(chunk_size).rfind('\n')
                    if offset >= 0:
                        # We found the new line; safe our offset and break
                        tail_ptr = offset + chunk_ptr
                        break

            if not self.payload_gzipped and (tail_ptr - head_ptr) > 0 \
                    and total_bytes < self.MAX_BUFFER_SIZE \
                    and tail_ptr < self.MAX_BUFFER_SIZE and self.can_read(1):
                # Astraweb is absolutely terrible for sending a little
                # bit more data a few seconds later. This is a final
                # call to try to handle these stalls just before the last
                # few bytes are sent.
                continue

            # Ensure we're pointing at the head of our buffer
            self._buffer.seek(head_ptr, SEEK_SET)

            # Compression Support
            if (tail_ptr - head_ptr) > 0:
                if self.payload_gzipped is True:

                    dc_obj = decompressobj()
                    try:
                        self._data.write(dc_obj.decompress(
                            self._buffer.read(tail_ptr - head_ptr),
                        ))
                        logger.debug("NNTP ZLIB decompression successful.")

                    except ZlibException:
                        # Decompression error; since compression is only used
                        # when retrieving server-side listings; it's best to
                        # just alert the end user and move along
                        logger.error(
                            '_recv() %d byte(s) ZLIB decompression failure.'
                            % (_bytes),
                        )

                        # Convert our response to that of an response Fetch
                        # Error
                        return NNTPResponse(
                            NNTPResponseCode.FETCH_ERROR,
                            'Fetch Error',
                        )
                else:
                    # No compression
                    self._data.write(self._buffer.read(tail_ptr - head_ptr))

                # Truncate our original buffer by striping out what was already
                # processed from it
                self._buffer = BytesIO(self._buffer.read())

                # Correct Total Length
                total_bytes = self._buffer.tell()

            ##################################################################
            #                                                                #
            #  Step: 4: This step processes the extra content founds from    #
            #           the NNTP stream based on the decoders passed in.     #
            #                                                                #
            ##################################################################

            self._data_len = self._data.seek(0, SEEK_END)
            d_head = self._data.seek(0, SEEK_SET)
            while d_head < self._data_len and self.connected:

                if codec_active is None:
                    # Scan our decoders (sequentially) and detect our match
                    # If we get a match, then we want to save it in the
                    # codec_active variable. We use this to track the data
                    # found for processing.

                    # Get our data
                    d_head = self._data.tell()
                    data = self._data.readline()

                    # This line scans the line of data we read and determines
                    # what kind of data it is (yEnc, Headers, etc) based on the
                    # decoders passed into _recv()
                    codec_active = next(
                        (d for d in decoders
                            if d.detect(data) is not None), None)
                    if codec_active:
                        logger.debug('Decoding using %s' % type(codec_active))

                # Based on previous check; we may actually have an active codec
                # now if we don't have one yet; well want to store the content
                # into our body and move along
                if codec_active is None:
                    # All data matched that no decoder took ownership off is
                    # saved into our body
                    response.body.write(data)

                    # Update d_head value
                    d_head = self._data.tell()

                    # Track lines processed
                    self.line_count += 1
                    continue

                # Adjust pointer for processing
                self._data.seek(d_head, SEEK_SET)

                # Begin decoding content
                decoded = codec_active.decode(self._data)

                # Adjust our pointer
                d_head = self._data.tell()

                # If we're at the end of our buffer; we can go ahead and clear
                # it
                if d_head >= self._data_len:
                    # Reset our data object once we're done parsing
                    self._data.truncate(0)
                    # Adjust pointer for processing
                    self._data.seek(0, SEEK_SET)

                # A little Decoder 101 for anyone reading my code; the below
                # identifies the possible return types from a Codec
                # (specifically the Decoder):
                #
                #     - NNTPContent:  We're done with the decoder, everything
                #                     was correctly processed. we're handled
                #                     and NNTPContent object.
                #
                #     - True:         We're still expecting more content
                #                     before we're finished, don't adjust
                #                     this from being the 'active' decoder!
                #
                #     - False:        Oh boy; we had a problem and we
                #                     couldn't deal with it. We're finished
                #                     with the decoder; the data is bad.
                #
                #     - None:         A graceful way of saying that we're done
                #                     with the decoder. Like an abort if you
                #                     will
                #
                if decoded is None:
                    # The Codec has completed and has nothing to return for
                    # storing. We gracefully move along at this point.
                    logger.debug(
                        'Decoding complete (no results) / %s' % codec_active,
                    )

                    # we're done; do nothing more
                    codec_active = None

                    continue

                elif decoded is True:
                    # We're expecting more data a long as the End of Data
                    # (EOD) flag hasn't been picked up.
                    if not self.article_eod:
                        logger.debug(
                            'Expecting more data to build results with...',
                        )
                        continue

                    # If we reach here, we've reached the end of the line
                    # a half (or possibly full block of data), but we just
                    # need to close off what we have. fall through and
                    # handle our codec; the HEAD call for example doesn't
                    # return an empty line after the the header like expected
                    # so it's normal to reach here.  Corrupted stuff still
                    # needs to be saved; and we can hope the par files can
                    # rebuild it if it is infact damaged.
                    if isinstance(codec_active, CodecBase):
                        # Store our decoded content (complete or not)
                        decoded = codec_active.decoded

                # If we got here, our content was good; we can safely
                # toggle our codec_active back to off since we're going to
                # be expecting more data now
                logger.debug('Decoding completed. %s' % codec_active)
                codec_active = None

                if not isinstance(decoded, NNTPContent):
                    # We ignore any other return type, Decoders should always
                    # return an NNTPContent type; anything else is considered
                    # moot
                    continue

                # Add to our NNTPContent() to our decoded set associated with
                # our NNTPResponse() object
                response.decoded.add(decoded)

                if not isinstance(decoded, NNTPMetaContent):
                    # Print a representative string into the body to identify
                    # the content parsed out (and decoded)

                    if max_bytes > 0:
                        # We're instructed to only retrieve 'some' content and
                        # abort for peaking purposes; so we're done now.
                        self.close()
                        return response

        # Track lines processed
        self.line_count += len(self.lines)

        logger.debug('Returning Response %s' % response)
        return response

    def close(self):
        """
        Drop the connection
        """

        # Reset our other flags
        self.group_name = None
        self.group_count = 0
        self.group_head = 0
        self.group_tail = 0
        self.group_index = 0

        # Reset Can Post Flag (if set)
        self.can_post = False

        if self.connected:
            logger.debug('Closing Connection')
            try:
                # Prevent Recursion by calling parent send()
                super(NNTPConnection, self).send('QUIT' + EOL)
            except:
                pass

        # SocketBase close() tries to grab the last
        # content on the socket before closing it
        return super(NNTPConnection, self).close()

    def _soft_reset(self):
        """
        Reset tracking items
        """
        self.article = {}
        self.article_eod = False
        self.article_fname = None

        # Reset our Buffers
        self._buffer.truncate(0)
        self._data.truncate(0)
        self._data_len = 0

        # Our response code (integer) returned by the NNTPServer on the last
        # request made
        self.last_resp_code = None

        # Our response string returned by the NNTPServer on the last request
        # made
        self.last_resp_str = ''

        # A flag set to True if the payload to follow the response is gzipped.
        # GZipped responses require a bit more CPU crunching but much less
        # bandwidth. This value is set to None if there simply wasn't a payload
        # at all (or an initialization value)
        self.payload_gzipped = None

        self.lines = []
        self.line_count = 0

        # Empty the receive buffer if anything is pending
        while self.can_read():
            try:
                self.read()

            except (SocketException, SignalCaughtException):
                pass

    def _hard_reset(self, wait=True):
        """
        Drop the connection and re-establish it
        """
        self.close()
        self.connect()

    def __del__(self):
        """
        Handle Deconstruction
        """
        self.close()

    def __str__(self):
        return '%s://%s@%s:%d' % (
            self.protocol, self.username, self.host, self.port)

    def __unicode__(self):
        return u'%s://%s@%s:%d' % (
            self.protocol, self.username, self.host, self.port)

    def __repr__(self):
        """
        Return a printable object
        """
        return '<NNTPConnection id=%d url="%s://%s@%s:%d" />' % (
            id(self),
            self.protocol,
            self.username,
            self.host,
            self.port,
        ) class NNTPHeader(NNTPMetaContent):
    """
    A Header representation of an NNTP Article
    """
    def __init__(self, work_dir=None, *args, **kwargs):
        super(NNTPHeader, self).__init__(
            work_dir=work_dir, sort_no=100, *args, **kwargs)

        # Initialize our header
        self.content = dict()

    def __setitem__(self, key, item):
        """
        Mimic Dictionary:  dict[key] = value
        """
        key = self.__fmt_key(key)
        self.content[key] = item

    def __getitem__(self, key):
        """
        Mimic Dictionary:  value = dict[key]
        """
        key = self.__fmt_key(key)
        return self.content[key]

    def post_iter(self):
        """
        Returns NNTP string as it would be required for posting
        """
        for k in HEADER_PRINT_SEQUENCE:
            if k in self.content:
                yield '%s: %s%s' % (k, self.content[k], NNTP_EOL)

        for k, v in self.content.iteritems():
            if k not in HEADER_PRINT_SEQUENCE:
                yield '%s: %s%s' % (k, v, NNTP_EOL)

        # Last entry
        yield NNTP_EOL

    def clear(self):
        """
        Mimic Dictionary:  dict.clear()
        """
        return self.content.clear()

    def copy(self):
        """
        Mimic Dictionary:  dict.copy()
        """
        return self.content.copy()

    def update(self, *args, **kwargs):
        """
        Mimic Dictionary:  dict.update(key)
        """
        return self.content.update(*args, **kwargs)

    def keys(self):
        """
        Mimic Dictionary:  dict.keys()
        """
        return self.content.keys()

    def iterkeys(self):
        """
        Mimic Dictionary:  dict.iterkeys()
        """
        return self.content.iterkeys()

    def itervalues(self):
        """
        Mimic Dictionary:  dict.itervalues()
        """
        return self.content.itervalues()

    def key(self):
        """
        Returns a key that can be used for sorting with:
            lambda x : x.key()
        """
        return '%.5d/Header/' % self.sort_no

    def values(self):
        """
        Mimic Dictionary:  dict.values()
        """
        return self.content.values()

    def items(self):
        """
        Mimic Dictionary:  dict.items()
        """
        return self.content.items()

    def iteritems(self):
        """
        Mimic Dictionary:  dict.items()
        """
        return self.content.iteritems()

    def pop(self, key, d=None):
        """
        Mimic Dictionary:  dict.pop(key, default)
        """
        key = self.__fmt_key(key)
        return self.content.pop(key, d)

    def str(self, delimiter=': ', eol='\r\n', count=0):
        """Returns a nice consistent message header enforcing header sequence.
        Anything not defined in the sequence is printed at the end.

        use count to specfy how may header lines to print; if you set this
        to zero, then all of the matched header lines are printed.

        This function attempts to honour print sequence allowing for a
        consistent output.
        """
        response = []
        for k in HEADER_PRINT_SEQUENCE:
            if k in self.content:
                response.append('%s%s%s' % (k, delimiter, self.content[k]))

        for k, v in self.content.iteritems():
            if k not in HEADER_PRINT_SEQUENCE:
                response.append('%s%s%s' % (k, delimiter, v))

        return eol.join(response)

    def __str__(self):
        """General str() default message format
        """
        return self.str()

    def __fmt_key(self, key):
        """Formats the hash key for more consistent hits; hence fetching the
        'Message-ID' key should still be fetched even if the user indexes
        with 'message-id'.
        """
        def _fmt(_k):
            return _k.group(1) + _k.group(2).upper()

        if not isinstance(key, basestring):
            # Handle invalid key entries types
            key = str(key)

        key = re.sub(
            # Flip -id to ID (short for Identifier)
            # Flip -crc to CRC (short for Cyclic Redundancy Check)
            r'([_-])((id|crc)([^a-z0-9]|$))',
            _fmt,
            re.sub(r'(^|\s|[_-])(\S)', _fmt, key.strip().lower()),
            flags=re.IGNORECASE,
        )
        if key in VALID_HEADER_ENTRIES or key.startswith(UNKNOWN_PREFIX):
            return key
        return UNKNOWN_PREFIX + key

    def article_exists(self):
        """
        Simply scans the header for keys that usually invalidate an article or
        anounce it's non-existance.

        If these keywords aren't found, we'll presume the article does in fact
        exist and therefore we'll return True

        """
        # If we reach here; we've decoded the file, the next thing
        # we need to do is check for some common entries that
        # would oherwise have made it so it's not valid
        return next((
            False for k in self.iterkeys()
            if NNTP_NUKED_MSG.match(k) is not None), True)

    def __len__(self):
        """
        Returns the number of header entries
        """
        return len(self.content)

    def __contains__(self, key):
        """
        support 'in' keyword
        """
        key = self.__fmt_key(key)
        return key in self.content

    def __delitem__(self, key):
        """
        allows the deletion of header keys
        """
        key = self.__fmt_key(key)
        del self.content[key]

    def __repr__(self):
        """
        Keep it only possible to track 1 header
        """
        return '<NNTPHeader sort=%d content="%s" />' % (
            self.sort_no, repr(self.content),
        ) class NNTPManager(object):
    """
    Used to manage multiple NNTPConnections via worker threads.

    The intent is to accept the same actions an NNTPConnection()
    would take but then re-assing the request to an available worker.

    NNTPManager() requires an NNTPSettings() object to work correctly

    """
    # Defines hook(s) allowing us to over-ride sections of the process.
    # This grants the user to ability to manipulate content prior to it
    # being posted and/or staged
    hooks = None

    def __init__(self, settings=None, hooks=None, *args, **kwargs):
        """
        Initialize the NNTPManager() based on the provided settings.
        it is presumed settings is a loaded NNTPSettings() object.
        """

        # A connection pool of NNTPConnections
        self._pool = []

        # A mapping of active worker threads
        self._workers = []

        # Keep track of the workers available for processing
        # we will use this value to determine if we need to spin
        # up another process or not.
        self._work_tracker = WorkTracker()

        # Queue Control
        self._work_queue = Queue()

        # Map signal
        gevent.signal(signal.SIGQUIT, gevent.kill)

        # Define our hooks (if any)
        self.hooks = HookManager()
        if hooks:
            self.hooks.add(hooks=hooks)

        if settings is None:
            # Use defaults
            settings = NNTPSettings()

        if not len(settings.nntp_servers):
            logger.warning("There were no NNTP Servers defined to load.")
            raise AttributeError('No NNTP Servers Defined')

        # Store our defined settings
        self._settings = settings

        return

    def hooks(self, hooks, reset=True):
        """
        Sets hooks into our connection object(s)

        """
        if reset:
            # Reset our loaded hooks
            self.hooks.reset()

        if isinstance(hooks, HookManager):
            # update our hooks
            for key in hooks.iterkeys():
                self.hooks.add(hooks[key])

        else:
            self.hooks.add(hooks)

    def spawn_workers(self, count=1):
        """
        Spawns X workers (but never more then the total allowed)
        """
        _count = 0
        while len(self._pool) < self._settings.nntp_processing['threads']:
            # First we build our connection object
            connection = NNTPConnection(**self._settings.nntp_servers[0])

            # Directly map the connection's hooks to the ones here in the
            # NNTPManager() object
            connection.hooks = self.hooks

            if len(self._settings.nntp_servers) > 1:
                # Append backup servers (if any defined)
                for idx in range(1, len(self._settings.nntp_servers)):
                    _connection = NNTPConnection(
                        **self._settings.nntp_servers[idx])

                    # Directly map the connection's hooks to the ones here in
                    # the NNTPManager() object
                    _connection.hooks = self.hooks

                    connection.append(_connection)

            # Append connection object to a pool
            self._pool.append(connection)

            logger.debug("Spawning worker...")
            g = Worker(
                connection=connection,
                work_queue=self._work_queue,
                work_tracker=self._work_tracker,
            )
            g.start()

            # Track our worker
            self._workers.append(g)

            _count += 1
            if _count >= count:
                # Stop spawning
                break

        if _count > 0:
            logger.info("Loaded %d new worker(s)." % (_count))
            return True

        return False

    def get_connection(self):
        """
        Grabs a connection from the thread pool and returns it by reference.
        It is NOT safe to us this function if you have concurrent thread going
        on at the same time.

        The function attempts to find a connection that has already been
        established first.  If it fails to then it just returns the
        first connection in the queue.

        """

        if len(self._pool):
            # Find the first connected connection
            connection = next(
                (c for c in self._pool if c.connected is True), None)

            if connection:
                return connection

            # Return the first entry if nothing is already connected
            return self._pool[0]

        # Otherwise there is nothing to return
        return None

    def close(self):
        """
        closes out any open threads and cleans up NNTPManager
        gracefully.
        """
        while not self._work_queue.empty():
            try:
                self._work_queue.get_nowait()
            except EmptyQueueException:
                # Nothing available for us
                break

        for worker in self._workers:
            # Toggle Exit
            worker._exit.set()
            self._work_queue.put(StopIteration)

        for entry in self._pool:
            entry.close()

        for worker in self._workers:
            logger.info("Waiting for workers to exit.")
            worker.join()

        del self._pool
        del self._workers
        self._workers = []
        self._pool = []

    def put(self, request):
        """
        Handles the adding to the worker queue

        """

        # Determine if we need to spin a worker or not
        self._work_tracker.lock.acquire(blocking=True)

        if len(self._work_tracker.available) == 0:
            if len(self._work_tracker) < self._settings\
                    .nntp_processing['threads']:
                # Spin up more work
                self.spawn_workers(count=1)

        # Append to Queue for processing
        self._work_queue.put(request)

        # Release our lock
        self._work_tracker.lock.release()

    def group(self, name, block=True):
        """
        Queue's an NNTPRequest for processing and returns a call
        to GROUP (fetching details on it specifically)

        If block is not set to true, then it is up to the calling
        application to monitor the request until it's complete.

        Since the Request Object is inherited from a gevent.Event()
        object, one can easily check the status with the ready()
        call or, wait() if they want to block until content is ready.

        See http://www.gevent.org/gevent.event.html#module-gevent.event
        for more details.

        To remain thread-safe; it's recommended that you do not change
        any of the response contents or articles contents prior to
        it's flag being set (marking completion)

        """
        # Push request to the queue
        request = NNTPConnectionRequest(actions=[
            # Append list of NNTPConnection requests in a list
            # ('function, (*args), (**kwargs) )
            ('group', (name, ), {}),
        ])

        # Append to Queue for processing
        self.put(request)

        # We'll know when our request has been handled because the
        # request is included in the response.
        if block:
            request.wait()

            # Simplify things by returning just the response object
            # instead of the request
            return request.response[0]

        # We aren't blocking, so just return the request object
        return request

    def groups(self, filters=None, lazy=True, block=True):
        """
        Queue's an NNTPRequest for processing and returns the
        NNTP Group lists.

        If block is not set to true, then it is up to the calling
        application to monitor the request until it's complete.

        Since the Request Object is inherited from a gevent.Event()
        object, one can easily check the status with the ready()
        call or, wait() if they want to block until content is ready.

        See http://www.gevent.org/gevent.event.html#module-gevent.event
        for more details.

        To remain thread-safe; it's recommended that you do not change
        any of the response contents or articles contents prior to
        it's flag being set (marking completion)

        """
        # Push request to the queue
        request = NNTPConnectionRequest(actions=[
            # Append list of NNTPConnection requests in a list
            # ('function, (*args), (**kwargs) )
            ('groups', list(), {'filters': filters, 'lazy': lazy}),
        ])

        # Append to Queue for processing
        self.put(request)

        # We'll know when our request has been handled because the
        # request is included in the response.
        if block:
            request.wait()

            # Simplify things by returning just the response object
            # instead of the request
            return request.response[0]

        # We aren't blocking, so just return the request object
        return request

    def stat(self, id, full=None, group=None, block=True):
        """
        Queue's an NNTPRequest for processing and returns it's
        response if block is set to True.

        If block is not set to true, then it is up to the calling
        application to monitor the request until it's complete.

        Since the Request Object is inherited from a gevent.Event()
        object, one can easily check the status with the ready()
        call or, wait() if they want to block until content is ready.

        See http://www.gevent.org/gevent.event.html#module-gevent.event
        for more details.

        To remain thread-safe; it's recommended that you do not change
        any of the response contents or articles contents prior to
        it's flag being set (marking completion)

        """
        # Push request to the queue
        request = NNTPConnectionRequest(actions=[
            # Append list of NNTPConnection requests in a list
            # ('function, (*args), (**kwargs) )
            ('stat', (id, ), {'group': group, 'full': full}),
        ])

        # Append to Queue for processing
        self.put(request)

        # We'll know when our request has been handled because the
        # request is included in the response.
        if block:
            request.wait()

            # Simplify things by returning just the response object
            # instead of the request
            return request.response[0]

        # We aren't blocking, so just return the request object
        return request

    def post(self, payload, update_headers=True, success_only=False,
             block=True):
        """
        Queue's an NNTPRequest for processing and returns it's
        response if block is set to True.

        If block is not set to true, then it is up to the calling
        application to monitor the request until it's complete.

        Since the Request Object is inherited from a gevent.Event()
        object, one can easily check the status with the ready()
        call or, wait() if they want to block until content is ready.

        See http://www.gevent.org/gevent.event.html#module-gevent.event
        for more details.

        To remain thread-safe; it's recommended that you do not change
        any of the response contents or articles contents prior to
        it's flag being set (marking completion)

        """
        # A list of results
        requests = []

        if isinstance(payload, NNTPnzb):
            # We're dealing with an NZB-File
            if not payload.is_valid():
                return None

            # Pre-Spawn workers based on the number of segments found in our
            # NZB-File.
            self.spawn_workers(payload.segcount())

            # We iterate over each segment defined int our NZBFile and merge
            # them into 1 file. We do this until we've processed all the
            # segments and we return a list of articles
            for segpost in payload:
                for article in segpost:

                    # Push request to the queue
                    request = NNTPConnectionRequest(actions=[(
                        # Append list of NNTPConnection requests in a list
                        # ('function, (*args), (**kwargs) )
                        'post', (article, ), {
                            'update_headers': update_headers,
                            'success_only': success_only,
                        }),
                    ])

                    # Append to Queue for processing
                    self.put(request)

                    # Store our request
                    requests.append(request)

        elif isinstance(payload, NNTPSegmentedPost):
            # Pre-Spawn workers based on the number of segments we find.
            self.spawn_workers(len(payload))

            # We iterate over each segment defined int our NZBFile and merge
            # them into 1 file. We do this until we've processed all the
            # segments and we return a list of articles
            for article in payload:

                # Push request to the queue
                request = NNTPConnectionRequest(actions=[(
                    # Append list of NNTPConnection requests in a list
                    # ('function, (*args), (**kwargs) )
                    'post', (article, ), {
                        'update_headers': update_headers,
                        'success_only': success_only,
                    }),
                ])

                # Append to Queue for processing
                self.put(request)

                # Store our request
                requests.append(request)

        elif isinstance(payload, NNTPArticle):
            # We're dealing with a single Article

            # Push request to the queue
            request = NNTPConnectionRequest(actions=[(
                # Append list of NNTPConnection requests in a list
                # ('function, (*args), (**kwargs) )
                'post', (payload, ), {
                    'update_headers': update_headers,
                    'success_only': success_only,
                }),
            ])

            # Append to Queue for processing
            self.put(request)

            # Store our request
            requests.append(request)

        # We'll know when our request has been handled because the
        # request is included in the response.
        if not block:
            # We aren't blocking, so just return the request objects
            return requests

        # Block indefinitely on all pending requests
        [req.wait() for req in iter(requests)]

        # Simplify things by returning just the response object
        # instead of the request
        responses = [req.response[0] for req in iter(requests)]

        if isinstance(payload, NNTPArticle):
            # A single Article, we can just go ahead and return a single
            # response
            return responses[0]

        # Return our responses
        return responses

    def get(self, id, work_dir, decoders=None, group=None, max_bytes=0,
            block=True, force=False):
        """
        Queue's an NNTPRequest for processing and returns it's
        response if block is set to True.

        If block is not set to true, then it is up to the calling
        application to monitor the request until it's complete.

        Since the Request Object is inherited from a gevent.Event()
        object, one can easily check the status with the ready()
        call or, wait() if they want to block until content is ready.

        See http://www.gevent.org/gevent.event.html#module-gevent.event
        for more details.

        To remain thread-safe; it's recommended that you do not change
        any of the response contents or articles contents prior to
        it's flag being set (marking completion)

        max_bytes is set to the number of bytes you want to have received
        before you automatically abort the connection and flag the object as
        having only been partially complete  Use this option when you need to
        inspect the first bytes of a binary file. Set this to zero to download
        the entire thing (this is the default value)

        The force flag when set to true forces the download of content even
        if it has previously already been retrieved.
        """

        # A list of results
        requests = []

        if isinstance(id, NNTPnzb):
            # We're dealing with an NZB-File
            if not id.is_valid():
                return None

            # Pre-Spawn workers based on the number of segments found in our
            # NZB-File.
            self.spawn_workers(id.segcount())

            # We iterate over each segment defined int our NZBFile and merge
            # them into 1 file. We do this until we've processed all the
            # segments and we return a list of articles
            for segpost_no, segpost in enumerate(id):
                for article_no, article in enumerate(segpost):

                    # Push request to the queue
                    request = NNTPConnectionRequest(actions=[
                        # Append list of NNTPConnection requests in a list
                        # ('function, (*args), (**kwargs) )
                        ('get', (article, work_dir), {
                            'decoders': decoders,
                            'group': group,
                            'max_bytes': max_bytes,
                        }),
                    ])

                    # Append to Queue for processing
                    self.put(request)

                    # Store our request
                    requests.append(request)

        elif isinstance(id, NNTPSegmentedPost):
            # Pre-Spawn workers based on the number of segments we find.
            self.spawn_workers(len(id))

            # We iterate over each segment defined int our NZBFile and merge
            # them into 1 file. We do this until we've processed all the
            # segments and we return a list of articles
            for article_no, article in enumerate(id):

                # Push request to the queue
                request = NNTPConnectionRequest(actions=[
                    # Append list of NNTPConnection requests in a list
                    # ('function, (*args), (**kwargs) )
                    ('get', (article, work_dir), {
                        'decoders': decoders,
                        'group': group,
                        'max_bytes': max_bytes,
                    }),
                ])

                # Append to Queue for processing
                self.put(request)

                # Store our request
                requests.append(request)

        elif isinstance(id, NNTPArticle):
            # We're dealing with a single Article

            # Push request to the queue
            request = NNTPConnectionRequest(actions=[
                # Append list of NNTPConnection requests in a list
                # ('function, (*args), (**kwargs) )
                ('get', (id, work_dir), {
                    'decoders': decoders,
                    'group': group,
                    'max_bytes': max_bytes,
                }),
            ])

            # Append to Queue for processing
            self.put(request)

            # Store our request
            requests.append(request)

        if isinstance(id, basestring):
            # We're dealing a Message-ID (Article-ID)

            # Push request to the queue
            request = NNTPConnectionRequest(actions=[
                # Append list of NNTPConnection requests in a list
                # ('function, (*args), (**kwargs) )
                ('get', (id, work_dir), {
                    'decoders': decoders,
                    'group': group,
                    'max_bytes': max_bytes,
                }),
            ])

            # Append to Queue for processing
            self.put(request)

            # Store our request
            requests.append(request)

        # We'll know when our request has been handled because the
        # request is included in the response.
        if not block:
            # We aren't blocking, so just return the request objects
            return requests

        # Block indefinitely on all pending requests
        [req.wait() for req in iter(requests)]

        # Simplify things by returning just the response object
        # instead of the request
        responses = [req.response[0] for req in iter(requests)]

        if isinstance(id, NNTPnzb):
            # NZB-File; Assign our Article Responses with their respected
            # locations in the NZB-File. This is done in addition to returning
            # our response objects
            resp_iter = iter(responses)

            # Iterate over our NZB-File (SegmentedPost) Entries
            for segpost in iter(id):
                # Iterate over our Segmented Post Entries
                for article in iter(segpost):
                    # For each segment in our list
                    resp = resp_iter.next()

                    # This should always equate since our response list
                    # was generated from our post
                    assert(article.id == resp.id)

                    # Load our response back to our NNTPnzb object
                    article.load(resp)

        elif isinstance(id, NNTPSegmentedPost):
            # Support NNTPSegmentedPost() Objects
            resp_iter = iter(responses)

            # Iterate over our Segmented Post Entries
            for article in iter(segpost):
                # For each segment in our list
                resp = resp_iter.next()

                # This should always equate since our response list
                # was generated from our post
                assert(article.id == resp.id)

                # Load our response back to our NNTPnzb object
                article.load(resp)

        elif isinstance(id, NNTPArticle):
            # This should always equate since our response list
            # was generated from our post
            assert(id.id == responses[0].id)

            # Load our response back to our NNTPnzb object
            article.load(responses[0])

            # Return our single article
            return responses[0]

        elif isinstance(id, basestring):
            # Message-ID: Just return our response directly
            return responses[0]

        # Return our responses
        return responses

    def xover(self, group, start=None, end=None,
              sort=XoverGrouping.BY_POSTER_TIME, block=True):
        """

        If the start or end time are set to `None` then they default to the
        low/high (head or tail) watermark respectively.

        overhead is performed to find these locations in the group identified.

        If the start or end time is a datetime variable, then additional
        overhead is performed to find these locations in the group identified.

        Scans Usenet Index and returns the following:
              {
                  id: u'the unique identifier',
                  article_no: 12345678,
                  poster: u'the poster's information',
                  date: datetime() object,
                  subject: u'a subject line in unicode',
                  size: 2135  // the message size in bytes
                  lines: 53   // the number of lines
                  group: u'alt.group.one'
                  xgroups : {
                      // references the Message-ID (id) per cross post
                       u'alt.group.two': 987654321,
                       u'alt.group.three': 12341234,
                  }
              }

        """
        # Push request to the queue
        request = NNTPConnectionRequest(actions=[
            # Append list of NNTPConnection requests in a list
            # ('function, (*args), (**kwargs) )
            ('xover', tuple(), {
                'start': start,
                'end': end,
                'group': group,
                'sort': sort,
            }),
        ])

        # Append to Queue for processing
        self.put(request)

        # We'll know when our request has been handled because the
        # request is included in the response.
        if block:
            request.wait()

            # Simplify things by returning just the response object
            # instead of the request
            return request.response[0]

        # We aren't blocking, so just return the request object
        return request

    def seek_by_date(self, refdate, group=None, block=True):
        """
        Returns a pointer in the selected group identified
        by the date specified.

        If block is not set to true, then it is up to the calling
        application to monitor the request until it's complete.

        Since the Request Object is inherited from a gevent.Event()
        object, one can easily check the status with the ready()
        call or, wait() if they want to block until content is ready.

        See http://www.gevent.org/gevent.event.html#module-gevent.event
        for more details.

        To remain thread-safe; it's recommended that you do not change
        any of the response contents or articles contents prior to
        it's flag being set (marking completion)

        """
        # Push request to the queue
        request = NNTPConnectionRequest(actions=[
            # Append list of NNTPConnection requests in a list
            # ('function, (*args), (**kwargs) )
            ('seek_by_date', (refdate, ), {'group': group, }),
        ])

        # Append to Queue for processing
        self.put(request)

        # We'll know when our request has been handled because the
        # request is included in the response.
        if block:
            request.wait()

            # Simplify things by returning just the response object
            # instead of the request
            return request.response[0]

        # We aren't blocking, so just return the request object
        return request

    def __del__(self):
        """
        Gracefully clean up any lingering connections
        """
        self.close() def bytes_to_strsize(byteval):
    """
    This function returns the string size equivalent (as a string) that best
    represents the integer (in bytes) passed in.

    """
    unit = 'B'
    try:
        byteval = float(byteval)

    except(ValueError, TypeError):
        return None

    if byteval >= 1024.0:
        byteval = byteval / 1024.0
        unit = 'KB'
        if byteval >= 1024.0:
            byteval = byteval / 1024.0
            unit = 'MB'
            if byteval >= 1024.0:
                byteval = byteval / 1024.0
                unit = 'GB'
                if byteval >= 1024.0:
                    byteval = byteval / 1024.0
                    unit = 'TB'

    return '%.2f%s' % (round(byteval, 2), unit) def mkdir(name, perm=0775):
    """
    A more contained wrapper to directory management
    """
    attempt = 3
    if isdir(name):
        return True

    while attempt > 0:
        try:
            makedirs(name, perm)
            logger.debug('Created directory: %s' % name)
            return True

        except OSError, e:
            if e[0] == errno.EEXIST:
                # directory exists; this is okay
                return isdir(name)

            logger.debug('Created directory %s exception: %s' % (
                name, e,
            ))

        # racing condition; just try again
        attempt -= 1

    # To many attempts... fail
    # ... fall through...
    return False def rm(path, *args, **kwargs):
    """
    A  rmtree reimplimentation. More importantly, Microsoft Windows
    can't recursively delete a directory if it contains read-only files
    with rmtree().  This rm() attempts to handle this. Symbolic links
    are not supported

    See http://stackoverflow.com/a/2656405 for a partial basis of
    why this rewrite exists.  The on_error refernced in this function didn't
    really satisfy my needs, so a rewrite of the rmtree() function was
    required.

    """
    if not exists(path):
        # Nothing more to do
        return True

    # We track our recursion level so we don't go in some unexitable loop
    recursion_count = kwargs.get('__recursion_count', 0)
    if recursion_count == 0:
        # We're on our first iteration
        logger.debug('Attempting to remove %s' % path)

    elif recursion_count >= RM_RECURSION_LIMIT:
        # Recursive Limit reached
        return False

    if not access(path, W_OK):
        # Is the error an access error ?
        chmod(path, S_IRWXU)

    if islink(path):
        # symlinks to directories are forbidden (taken from shutils.rmtree()
        # see: https://bugs.python.org/issue1669

        # We'll attempt to treat the link as a file and NOT a directory. If we
        # fail then we're done
        try:
            unlink(path)
            logger.debug('Removed link %s' % path)
            return True

        except:
            logger.warning(
                'Failed to remove link %s' % path)
            return False

    if isdir(path):
        names = []
        try:
            names = listdir(path)

        except _OSError:
            # Failed
            return False

        for name in names:
            fullname = join(path, name)
            try:
                mode = lstat(fullname).st_mode

            except _OSError:
                mode = 0

            if S_ISDIR(mode):
                if not access(fullname, W_OK):
                    # Make sure we can access 'this' path
                    chmod(fullname, S_IRWXU)

                try:
                    # Attempt to remove directory
                    rmdir(fullname)

                except _OSError:
                    # It's not empty, so recursively enter it
                    if not rm(
                            fullname, __recursion_level=recursion_count + 1):
                        return False
            else:
                # We're dealing with file/link
                try:
                    unlink(fullname)
                    logger.debug('Removed file %s' % fullname)

                except:
                    logger.warning('Failed to remove file %s' % fullname)
                    return False

        try:
            rmdir(path)
            logger.debug('Removed file %s' % path)

        except _OSError:
            logger.warning('Failed to remove file %s' % path)
            return False

    else:
        # We're dealing with file/link
        try:
            unlink(path)
            logger.debug('Removed file %s' % path)

        except:
            logger.warning('Failed to remove file %s' % path)
            return False

    return True def hook(name=None, priority=Hook.priority):
    """
    @hook decorator allows you to map functions you've defined to be called
    at specific times throughout the life of an action performed by newsreap.

    if you don't specify a name then the function name you wrap is used.

    The priority field allows you to set your function to be called before
    or after others that might also be configured to run for a specific
    type of hook.
        @hook(name="new_name", priority=100)
        def your_function(*args, **kwargs):
            ...

    The lower the priority, the further to the front of the line of execution
    it moves to. The default value is 1000
    """
    if callable(name):
        def callable_hook(func):
            # Acquire our potentially existing entries as we don't want to
            # replace them
            entries = getattr(func, Hook.hook_id, dict())

            # Store our new entry contents
            entries[func.__name__] = {
                'priority': Hook.priority,
            }

            # Save our entries back to the function
            setattr(func, Hook.hook_id, entries)

            # Storing meta information into our object allows us to reference
            # it later on by external wrappers such as the HookManager
            setattr(func, Hook.module_id, '{module}.{function}'.format(
                module=splitext(basename(__file__))[0],
                function=func.__name__,
            ))

            return func
        return callable_hook(name)

    else:
        def noncallable_hook(func):
            # Acquire our potentially existing entries as we don't want to
            # replace them
            entries = getattr(func, Hook.hook_id, dict())

            # Store our new entry contents
            entries[func.__name__ if name is None else name] = {
                'priority': priority,
            }

            # Save our entries back to the function
            setattr(func, Hook.hook_id, entries)

            # Storing meta information into our object allows us to reference
            # it later on by external wrappers such as the HookManager
            setattr(func, Hook.module_id, '{module}.{function}'.format(
                module=splitext(basename(__file__))[0],
                function=func.__name__,
            ))

            return func

        return noncallable_hook NEWSREAP_ENGINE = '%s.engine' % NEWSREAP_LOGGER
import weakref
import sys
import logging
from collections import deque
from tqdm import tqdm
from os import getcwd
from os.path import isfile
from os.path import join
from os.path import abspath
from os.path import expanduser
from os.path import basename
from os.path import dirname
from os.path import splitext
from StringIO import StringIO
from .NNTPGroup import NNTPGroup
from .NNTPArticle import MESSAGE_ID_RE
from .NNTPArticle import NNTPArticle
from .NNTPSegmentedPost import NNTPSegmentedPost
from .NNTPnzb import NNTPnzb
from .NNTPGetDatabase import NNTPGetDatabase
from .NNTPConnection import NNTPConnection
from .NNTPHeader import NNTPHeader
from .NNTPManager import NNTPManager
from .Utils import bytes_to_strsize
from .Utils import mkdir
from .Utils import rm
from .decorators import hook
from newsreap.Logging import NEWSREAP_ENGINE
        self.tmp_path = None

        # A link to either an NNTPManager or NNTPConnection object
        # This only referenced for the upload and verify stages
        self.connection = connection
        if not self.connection:
            self.connection = NNTPManager()

        # Setup our HookManager for managing hooks
        self.hooks = self.connection.hooks

        if hooks:
            # load our hooks
            self.hooks.add(hooks, ('.', self.default_hook_path))

        # The path to our database for managing our staged content
        self.db_path = None

        # The sqlalchemy engine reference
        self.engine = None

        # our Database object
        self._db = None

        # A weak reference to our returned results so that we can print
        # and or access our results from.  This gets cleared on each
        # subsiquent call to load.
        self.results = None

        # Used for monitoring our transfer speeds
        self.xfer_rate = deque()

        # Our tqdm status bar reference
        self.xfer_tqdm = None

        # Track our errors if and/or when they occur
        self.err_stream = StringIO()

        # Add our internal hooks
        self.hooks.add(self.transfer_count)
        self.hooks.add(self.transfer_rates)

    @hook(name='socket_read')
    def transfer_rates(self, xfer_bytes, xfer_time, **kwargs):
        """
        calculate and monitor our transfer speeds

        """

        self.xfer_rate.append(xfer_bytes / xfer_time)
        if len(self.xfer_rate) > self.xfer_rate_max_queue_size:
            # bump oldest off of our list
            self.xfer_rate.popleft()

        # Calculate our average speed
        xfer_avg = sum(self.xfer_rate) / float(len(self.xfer_rate))

        # Print current speed and average
        #sys.stdout.write("xfer={xfers / avg={avg}s\n".format(
        #    xfer=bytes_to_strsize(self.xfer_rate[-1] / 1000.0),
        #    avg=bytes_to_strsize(xfer_avg / 1000.0),
        #))

        if self.xfer_tqdm is not None:
            self.xfer_tqdm.update(xfer_bytes)

        # Flush our buffer
        # sys.stdout.flush()

    @hook(name='post_stat')
    @hook(name='post_get')
    def transfer_count(self, status, **kwargs):
        """
        used to track downloading content

        """
        if status is True:
            sys.stdout.write('.')
        else:
            sys.stdout.write('x')
        sys.stdout.flush()

    def load(self, source, hooks=True, groups=None, path=None,
             *args, **kwargs):
        """
        Takes a Message-ID or NZB-File (aource) and retrieves the content

        If hooks is set to True, it will continue using whatever hooks are
        already in place.  Otherwise you can define your own hooks here

        The path becomes our base_path if specified, otherwise it is
        determined based on the source passed in.  If the source is a
        Message-ID, then the path is the current working directory we're
        in.

        """
        # Ensure we're not loaded
        self._loaded = False

        # Reset a few variables that may not get reset later
        self.nzb = None
        self.name = None
        self.path = None
        self.tmp_path = None
        self.db_path = None
        self.engine = None
        self._db = None
        self.results = None

        # Reset our transfer rate queue
        self.xfer_rate.clear()

        # Destory our tqdm object (if present)
        self.xfer_tqdm = None

        # Start by defining our base path
        self.base_path = path

        if groups is not None:
            # Update our groups if new ones were specified

            self.groups = NNTPGroup.split(groups)