class NNTPHeader(NNTPMetaContent):
    """
    A Header representation of an NNTP Article
    """
    def __init__(self, work_dir=None, *args, **kwargs):
        super(NNTPHeader, self).__init__(
            work_dir=work_dir, sort_no=100, *args, **kwargs)

        # Initialize our header
        self.content = dict()

    def __setitem__(self, key, item):
        """
        Mimic Dictionary:  dict[key] = value
        """
        key = self.__fmt_key(key)
        self.content[key] = item

    def __getitem__(self, key):
        """
        Mimic Dictionary:  value = dict[key]
        """
        key = self.__fmt_key(key)
        return self.content[key]

    def post_iter(self):
        """
        Returns NNTP string as it would be required for posting
        """
        for k in HEADER_PRINT_SEQUENCE:
            if k in self.content:
                yield '%s: %s%s' % (k, self.content[k], NNTP_EOL)

        for k, v in self.content.iteritems():
            if k not in HEADER_PRINT_SEQUENCE:
                yield '%s: %s%s' % (k, v, NNTP_EOL)

        # Last entry
        yield NNTP_EOL

    def clear(self):
        """
        Mimic Dictionary:  dict.clear()
        """
        return self.content.clear()

    def copy(self):
        """
        Mimic Dictionary:  dict.copy()
        """
        return self.content.copy()

    def update(self, *args, **kwargs):
        """
        Mimic Dictionary:  dict.update(key)
        """
        return self.content.update(*args, **kwargs)

    def keys(self):
        """
        Mimic Dictionary:  dict.keys()
        """
        return self.content.keys()

    def iterkeys(self):
        """
        Mimic Dictionary:  dict.iterkeys()
        """
        return self.content.iterkeys()

    def itervalues(self):
        """
        Mimic Dictionary:  dict.itervalues()
        """
        return self.content.itervalues()

    def key(self):
        """
        Returns a key that can be used for sorting with:
            lambda x : x.key()
        """
        return '%.5d/Header/' % self.sort_no

    def values(self):
        """
        Mimic Dictionary:  dict.values()
        """
        return self.content.values()

    def items(self):
        """
        Mimic Dictionary:  dict.items()
        """
        return self.content.items()

    def iteritems(self):
        """
        Mimic Dictionary:  dict.items()
        """
        return self.content.iteritems()

    def pop(self, key, d=None):
        """
        Mimic Dictionary:  dict.pop(key, default)
        """
        key = self.__fmt_key(key)
        return self.content.pop(key, d)

    def str(self, delimiter=': ', eol='\r\n', count=0):
        """Returns a nice consistent message header enforcing header sequence.
        Anything not defined in the sequence is printed at the end.

        use count to specfy how may header lines to print; if you set this
        to zero, then all of the matched header lines are printed.

        This function attempts to honour print sequence allowing for a
        consistent output.
        """
        response = []
        for k in HEADER_PRINT_SEQUENCE:
            if k in self.content:
                response.append('%s%s%s' % (k, delimiter, self.content[k]))

        for k, v in self.content.iteritems():
            if k not in HEADER_PRINT_SEQUENCE:
                response.append('%s%s%s' % (k, delimiter, v))

        return eol.join(response)

    def __str__(self):
        """General str() default message format
        """
        return self.str()

    def __fmt_key(self, key):
        """Formats the hash key for more consistent hits; hence fetching the
        'Message-ID' key should still be fetched even if the user indexes
        with 'message-id'.
        """
        def _fmt(_k):
            return _k.group(1) + _k.group(2).upper()

        if not isinstance(key, basestring):
            # Handle invalid key entries types
            key = str(key)

        key = re.sub(
            # Flip -id to ID (short for Identifier)
            # Flip -crc to CRC (short for Cyclic Redundancy Check)
            r'([_-])((id|crc)([^a-z0-9]|$))',
            _fmt,
            re.sub(r'(^|\s|[_-])(\S)', _fmt, key.strip().lower()),
            flags=re.IGNORECASE,
        )
        if key in VALID_HEADER_ENTRIES or key.startswith(UNKNOWN_PREFIX):
            return key
        return UNKNOWN_PREFIX + key

    def article_exists(self):
        """
        Simply scans the header for keys that usually invalidate an article or
        anounce it's non-existance.

        If these keywords aren't found, we'll presume the article does in fact
        exist and therefore we'll return True

        """
        # If we reach here; we've decoded the file, the next thing
        # we need to do is check for some common entries that
        # would oherwise have made it so it's not valid
        return next((
            False for k in self.iterkeys()
            if NNTP_NUKED_MSG.match(k) is not None), True)

    def __len__(self):
        """
        Returns the number of header entries
        """
        return len(self.content)

    def __contains__(self, key):
        """
        support 'in' keyword
        """
        key = self.__fmt_key(key)
        return key in self.content

    def __delitem__(self, key):
        """
        allows the deletion of header keys
        """
        key = self.__fmt_key(key)
        del self.content[key]

    def __repr__(self):
        """
        Keep it only possible to track 1 header
        """
        return '<NNTPHeader sort=%d content="%s" />' % (
            self.sort_no, repr(self.content),
        ) class NNTPArticle(object):
    """
    An object for maintaining retrieved article content. The primary
    reason for this object is it makes it easier to sort the results
    in memory tracking both the content downloaded (using the
    NNTPConnection.get() call along with the file stored locally on disk
    that it's responsible for.

    This function does it's best to behave like a stream. But provides
    some functions to make manipulating and merging with other articles
    easier to do.

    Articles by default assume a roll of 'attached'.  This means that
    the files written to disk are removed the second the object is
    destroyed.  This is intentional!  You can call detach() at any
    time you want but now you are responsible for cleaning up the
    filename.

    """

    def __init__(self, id=None, subject=None, poster=None, groups=None,
                 work_dir=None, body=None, codecs=None, *args, **kwargs):
        """
        Initialize NNTP Article

        """

        # The Article Message-ID
        self.id = id if isinstance(id, basestring) else ''

        # The NNTP Group Index #
        self.no = 1000
        try:
            self.no = int(kwargs.get(u'no', self.no))
        except:
            # Just use the default
            pass

        # The Subject
        self.subject = subject
        if self.subject is None:
            self.subject = DEFAULT_NNTP_SUBJECT

        # The Poster
        self.poster = poster
        if self.poster is None:
            self.poster = DEFAULT_NNTP_POSTER

        if work_dir is None:
            self.work_dir = DEFAULT_TMP_DIR
        else:
            self.work_dir = abspath(expanduser(work_dir))

        # Contains a list of decoded content; it's effectively the articles
        # attachments
        self.decoded = sortedset(key=lambda x: x.key())

        # The group(s) associated with our article
        self.groups = NNTPGroup.split(groups)

        # A hash of header entries
        self.header = NNTPHeader(work_dir=self.work_dir)

        if isinstance(body, NNTPAsciiContent):
            self.body = body
        else:
            # Our body contains non-decoded content
            self.body = NNTPAsciiContent(work_dir=self.work_dir)

        if isinstance(body, basestring) and len(body) > 0:
            # Store our body content
            self.body.write(body)

        # Load our Codecs
        self._codecs = codecs

        # Provide default codecs if required
        if not self._codecs:
            self._codecs = [CodecYenc(), ]

        elif isinstance(self._codecs, CodecBase):
            self._codecs = [self._codecs, ]

    def load(self, response):
        """
        Loads an article by it's NNTPResponse or from another NNTPArticle

        """
        if isinstance(response, NNTPResponse):
            # Our body contains non-decoded content
            self.body = response.body

            # Store decoded content
            self.decoded = response.decoded

            # Store Header
            self.header = next(
                (d for d in self.decoded if isinstance(d, NNTPHeader)), None)

            # Our groups associated with the post (if we know it)
            self.groups = set()

            if self.header is not None:
                # Remove Header from decoded list
                self.decoded.remove(self.header)

                if u'Newsgroups' in self.header:
                    # Parse our groups out of the header
                    self.groups = NNTPGroup.split(self.header[u'Newsgroups'])

        elif isinstance(response, NNTPArticle):
            # We basically save everything except the work-dir since
            # our new work dir could be a new location

            # The NNTP Message-ID
            self.id = response.id

            # The NNTP Group Index
            self.no = response.no

            # Store the subject
            self.subject = response.subject

            # Store the poster
            self.poster = response.poster

            # The NNTP Groups
            self.groups = response.groups

            # Store the header from the other side
            self.header = response.header

            # Our body contains non-decoded content
            self.body = response.body

            # Store decoded content
            self.decoded = response.decoded

        else:
            # Unsupported
            return False

        return True

    def post_iter(self, update_headers=True):
        """
        Returns NNTP string as it would be required for posting
        """

        if not self.id:
            # Generate ourselves a Message-ID
            self.msgid()

        if not len(self.body) and not len(self.decoded):
            # Nothing to post
            logger.error(
                'Post Denied / Article %s has no content to post.' %
                self.msgid())
            return None

        if not len(self.groups):
            # Nothing to post
            logger.error(
                'Post Denied / Article %s has no groups defined.' %
                self.msgid())
            return None

        try:
            if not self.subject.strip():
                logger.error(
                    'Post Denied / Article %s has no subject.' %
                    self.msgid())
                return None

        except AttributeError:
            logger.error(
                'Post Denied / Article %s has no subject.' % self.msgid())
            return None

        try:
            if not self.poster.strip():
                logger.error(
                    'Post Denied / Article %s has no poster.' % self.msgid())
                return None

        except AttributeError:
            logger.error(
                'Post Denied /Article %s has no poster.' % self.msgid())
            return None

        if update_headers:
            self.update_headers()

        args = [
            iter(self.header.post_iter()),
        ]

        if len(self.body):
            args.append(iter(self.body.post_iter()))
            args.append(NNTP_EOL)

        if len(self.decoded):
            for entry in self.decoded:
                args.append(iter(entry.post_iter()))
                args.append(NNTP_EOL)

        return chain(*args)

    def encode(self, encoders):
        """
        A wrapper to the encoding of content. The function returns None if
        a problem occurs, otherwise the function returns an NNTPArtice()
        object.

        The power of this function comes from the fact you can pass in
        multiple encoders to have them all fire after one another.
        """
        if len(self) == 0:
            # Nothing to encode
            return None

        objs = sortedset(key=lambda x: x.key())
        for content in self:
            obj = content.encode(encoders)
            if obj is None:
                return None

            # Successful, add our object to our new list
            objs.add(obj)

        # If we reach here we encoded our entire article
        # Create a copy of our article
        article = self.copy(include_attachments=False)

        # In our new copy; store our new encoded content
        article.decoded = objs

        # Return our article
        return article

    def split(self, size=81920, mem_buf=1048576, body_mirror=0):
        """
        Split returns a set of NNTPArticle() objects containing the split
        version of the data it already represents.

        Even if the object can't be split any further given the parameters, a
        set of at least 1 entry will always be returned.  None is returned if
        an error occurs. None is also returned if split() is called while there
        is more then one NNTPContent objects since it makes the situation
        Ambiguous.

        The body_mirror flag has a series of meanings; since we start with a
        single post before calling this (which goes on and splits the post) we
        have to decide if we want the contents of our message body to appear
        in every post. So here is how it breaks down.

        If you set it to False, None then the body content is ignored
        and nothing is done with it.

        If you set it to True, then the body is copied to every single part.

        If you set it to -X (negative #) then the body is stored in the
        part generated as if you called the python slice expression:
            parts[-X].body <-- store body here

        If you set it to X (positive #), then the body is stored in
        that index value similar to above:
            parts[X].body <-- store body here

        IndexErrors are automatially and silently handled.  If you access
        a part that is out of bounds then the bounds are limited to fit
        automtatically.  For instance, if you specified -100 as the
        part and there were only 4 parts generated, (-4 is the maximum
        it could have been) then -4 is used in place of the -100.

        consiquently if you passed in 100 (a positive value). since
        (positive) indexes are measured started at 0 (zero), this means a
        with a list of 4 items the largest value can be a 3.  Therefore
        the index of 3 is used instead.
        """
        if len(self) > 1:
            # Ambiguous
            return None

        if len(self) == 0:
            # Nothing to split
            return None

        content = next((
            c for c in self.decoded if isinstance(c, NNTPContent)), None)

        if not content:
            # Well this isn't good; we have decoded entries that are not
            # of type NNTPContent.
            return None

        # ensure our size is set to some value
        size = strsize_to_bytes(size)
        if size is None:
            # You can't call split() with a size value set at None
            return None

        # Size needs to consider our message body
        # The length of our headers are not applicable
        if len(self.body):
            size -= len(self.body) + len(NNTP_EOL)

        if size < 0:
            # we can't perform the split
            return None

        # Split our content
        new_content = content.split(size=size, mem_buf=mem_buf)

        if new_content is None:
            # something bad happened
            return None

        # If we get here, we have content to work with.  We need to generate
        # a list of articles based on our existing one.
        articles = sortedset(key=lambda x: x.key())

        for no, c in enumerate(new_content):
            a = NNTPArticle(
                subject=self.subject,
                poster=self.poster,
                groups=self.groups,
                # Increment our index #
                no=self.no + no,
                work_dir=self.work_dir,
            )

            # Set our header to be a copy of what we already have
            a.header = deepcopy(self.header)

            # Store our NNTPContent() object
            a.add(c)

            if body_mirror is True:
                # body is mirrored to everything
                a.body = self.body

            # Store our Article
            articles.add(a)

        # Now we transfer over our body if nessisary
        if isinstance(body_mirror, int):
            if body_mirror > -1:
                try:
                    articles[body_mirror].body = self.body

                except IndexError:
                    # Store at last entry
                    articles[-1].body = self.body
            else:
                # Negative Number
                try:
                    articles[body_mirror].body = self.body

                except IndexError:
                    # Store at first entry
                    articles[0].body = self.body

        # Return our articles
        return articles

    def append(self, articles):
        """
        Appends one article onto another. This is the reverse of split()
        articles must be an NNTPArticle object, or a set-of NNTPArticle()
        objects.

        """
        if isinstance(articles, NNTPArticle):
            # Convert our entry into a list object
            articles = [articles, ]

        # Change to an iterator
        articles = iter(articles)
        if len(self) > 1:
            # Ambiguous; we can't append content if we have
            # more then one possible thing to append to
            return False

        for article in articles:
            if len(article) == 0:
                # No attachments; move along silently
                continue

            elif len(article) > 1:
                # To many attachments in article
                return False

            if len(self.decoded) == 0:
                # Store a copy of our decoded object
                content = article[0].copy()
                if content is None:
                    # We failed to create a copy
                    return False

                # Add our content to our object
                self.decoded.add(content)
                continue

            # If we reach here, we have 1 entry to append our content to
            if not self.decoded[0].append(article[0]):
                return False

        # Return our success
        return True

    def copy(self, include_attachments=True):
        """
        Creates a copy of the article returning one

        """

        article = NNTPArticle(
            subject=self.subject,
            poster=self.poster,
            groups=self.groups,
        )

        article.id = self.id
        article.no = self.no
        article.work_dir = self.work_dir
        article.header = self.header.copy()
        article.body = self.body.copy()

        if include_attachments:
            for content in self.decoded:
                obj = content.copy()
                if obj is None:
                    return None

                # Add our copied NNTP object
                article.decoded.add(obj)

        # Return our copy
        return article

    def is_valid(self):
        """
        Iterates over article content an returns True if all of it is valid
        based on it's crc32/md5 and/or whatever mechanism is used to determine
        a NNTPContents validity
        """
        if len(self.decoded) == 0:
            # No articles means no validity
            return False

        return next((
            False for c in self.decoded if c.is_valid() is False), True)

    def files(self):
        """
        Returns a list of the files within article
        """
        return [x.path() for x in self.decoded]

    def key(self):
        """
        Returns a key that can be used for sorting with:
            lambda x : x.key()
        """
        return '%.5d%s' % (self.no, self.id)

    def detach(self):
        """
        Detach the article stored on disk from being further managed by this
        class
        """
        for a in self.decoded:
            if isinstance(a, NNTPBinaryContent):
                a.detach()
        return

    def attach(self):
        """
        Detach the article stored on disk from being further managed by this
        class
        """
        for a in self.decoded:
            if isinstance(a, NNTPBinaryContent):
                a.attach()
        return

    def add(self, content):
        """
        Used for adding content to the self.decoded class
        """
        if isinstance(content, basestring):
            # A mime object we can use to detect the type of file
            m = Mime()

            if isfile(content):
                # file relative to current path; get our mime response
                mr = m.from_bestguess(content)

                # Our NNTPContent object will depend on whether or not we're
                # dealing with an ascii file or binary
                instance = NNTPBinaryContent \
                    if mr.is_binary() else NNTPAsciiContent

                content = instance(
                    filepath=content,
                    work_dir=self.work_dir,
                )

            else:
                # Not relative; is the file relative to the work_dir?
                with pushd(self.work_dir, create_if_missing=True):
                    if isfile(content):

                        # file relative to current path; get our mime response
                        mr = m.from_bestguess(content)

                        # Our NNTPContent object will depend on whether or not
                        # we're dealing with an ascii file or binary
                        instance = NNTPBinaryContent \
                            if mr.is_binary() else NNTPAsciiContent

                        # load our file
                        content = instance(
                            filepath=content,
                            work_dir=self.work_dir,
                        )

                    else:
                        # we're dealing with a new file; just save what we have
                        content = NNTPBinaryContent(
                            filepath=content,
                            work_dir=self.work_dir,
                        )

        if not isinstance(content, NNTPContent):
            return False

        # duplicates are ignored in a blist and therefore
        # we just capture the length of our list before
        # and after so that we can properly return a True/False
        # value
        _bcnt = len(self.decoded)
        self.decoded.add(content)

        return len(self.decoded) > _bcnt

    def msgid(self, host=None, reset=False):
        """
        Returns a message ID; if one hasn't been generated yet, it is
        based on the content in the article and used

        If reset is set to True, then a new ID is generated
        """
        if not reset and self.id:
            return self.id

        if not host:
            # Generate a 32-bit string we can use
            host = random_str(32)

        if len(self.decoded):
            partno = self.decoded[0].part
        else:
            partno = 1

        m = hashlib.sha1()
        reference = datetime.utcnow()
        m.update(datetime.utcnow().strftime('%Y%m%d%H%M%S%f'))
        key = m.hexdigest()

        # If we reach here an ID hasn't been generated yet; generate one
        self.id = '%s%x.%x!%x%x%x%x%x@%s%s%s' % (
            key[0:5],
            reference.second,
            reference.microsecond,
            reference.year,
            reference.month,
            reference.day,
            reference.minute,
            partno,
            reference.hour,
            host,
            key[5:10],
        )

        # Then return it
        return self.id

    def save(self, filepath=None, copy=False):
        """
        Saves all of the NNTPContent into the directory specified by
        the filepath; if no filepath is specified, then content is written
        into it's work_dir
        """

        if filepath:
            if not isdir(filepath) and not mkdir(filepath):
                # We failed
                return False

        for attachment in self.decoded:
            if not attachment.save(filepath=filepath, copy=copy):
                return False

        return True

    def deobsfucate(self, filebase='', codecs=None):
        """
        Using the information we have in the article generate a filename to
        the best of our ability.

        filebase: provide a fallback filename base (the part of the file
                  before the extension) to build on if we can't detect
                  the file on our own.

        codecs:   The codec(s) you wish to use to assist in the deobsfucation
        """

        if len(self.decoded) != 1:
            # There is ambiguity if there is not one attachment
            return None

        if filebase is None:
            # Safety
            filebase = ''

        # The detected article filename and extension
        a_name = ''
        a_fext = ''
        a_mime = None

        # The detected attachment filename and extension
        d_name = ''
        d_fext = ''
        d_mime = None

        # Create our Mime object since we'll be using it a lot
        m = Mime()

        if codecs is None:
            codecs = self._codecs

        elif isinstance(codecs, CodecBase):
            codecs = [codecs, ]

        # Use our Codec(s) to extract our Yenc Subject
        matched = None
        for c in codecs:
            # Use our Defined Codec(s) to extract content from our subject
            matched = c.parse_article(
                id=self.id,
                article_no=self.no,
                subject=self.subject,
                poster=self.poster,
            )

            if matched:
                # We succesfully got a filename from our subject line
                a_fname = matched.get('fname', '').strip()
                if a_fname:
                    # always allow the name to over-ride the detected filename
                    # if we actually have a real name we can assciated with it
                    # by
                    a_fext = m.extension_from_filename(a_fname)
                    a_name = a_fname[:-len(a_fext)]
                    a_mime = m.from_filename(a_fname)
                    if a_mime and a_mime.type() == DEFAULT_MIME_TYPE:
                        a_mime = None

                # We can break out
                break

        ####################################
        # Now we want to scan our attachment
        ####################################

        # Our mime object for our attachment
        d_mime = self.decoded[0].mime()
        d_fname = self.decoded[0].filename if self.decoded[0].filename \
            else basename(self.decoded[0].path())
        d_mime = m.from_bestguess(d_fname)
        if d_mime and d_mime.type() == DEFAULT_MIME_TYPE:
            d_mime = None

        d_fext = m.extension_from_filename(d_fname)
        d_name = d_fname[:-len(d_fext)]

        # At this point we have to make a decision to go with the filename
        # pulled from our content object, or go with the filename detected
        # from the article.

        _name = filebase
        _fext = None

        # Using the mime types, we decide which one is more likely the name
        if a_mime is None:
            # Attachment was not parsable
            if d_mime is not None:
                # However we did detect a file through this method
                _fext = d_fext
                _name = filebase if filebase else d_name

        elif d_mime is not None:
            # Attachment is parsable and so is our attachment
            if d_mime == a_mime:
                # Same type
                _fext = d_fext if d_fext else a_fext
                _name = _name if _name else d_name

            else:
                # We need to choose one over the other
                if len(d_fext):
                    # we have a filesize associated with our content which
                    # means there is a good chance we parsed our mime typefrom
                    # here
                    _fext = d_fext if d_fext else d_mime.extension()
                else:
                    # Use whatever we parsed
                    _fext = a_fext if a_fext else a_mime.extension()

                _name = _name if _name else d_name

        else:
            # Attachment is parsable and our Article isn't, easy-peasy
            # just use the content
            _fext = a_fext if a_fext else a_mime.extension()
            _name = filebase if filebase else a_name

        if not _fext:
            # Last resort
            _fext = d_fext if d_fext else a_fext
            if not _fext:
                return None

        if not _name:
            # Last resort
            _name = d_name if d_name else a_name
            if not _name:
                return None

        # Return our assembled identifier
        return "{0}{1}".format(_name, _fext)

    def size(self):
        """
        return the total size of our decoded content; we factor in the body if
        it exists.
        """
        if len(self.body):
            return sum(len(d) for d in self.decoded) + \
                len(self.body) + len(NNTP_EOL)
        return sum(len(d) for d in self.decoded)

    def strsize(self):
        """
        return the total size of our decoded content in a human readable
        string.
        """
        return bytes_to_strsize(self.size())

    def update_headers(self):
        """
        Updates our header information based on specified content
        """
        # Our Newsgroups can be of type NNTPGroup, but it can also just
        # be a plain string 'alt.binaries.test', etc. to support both
        # types we need to do a loop inside of our join below
        self.header['Newsgroups'] = ','.join([str(x) for x in self.groups])
        self.header['Subject'] = self.subject.strip()
        self.header['From'] = self.poster.strip()
        self.header['X-Newsposter'] = '%s v%s' % (__title__, __version__)
        self.header['Message-ID'] = '<%s>' % self.msgid()

    def __iter__(self):
        """
        Grants usage of the next()
        """
        # Ensure our stream is open with read
        return iter(self.decoded)

    def __len__(self):
        """
        Returns the number of decoded content entries found
        """
        return len(self.decoded)

    def __lt__(self, other):
        """
        Handles less than for storing in btrees
        """
        return self.key() < other.key()

    def __eq__(self, other):
        """
        Handles equality

        """
        return self.__dict__ == other.__dict__

    def __getitem__(self, index):
        """
        Support accessing NNTPContent objects by index
        """
        return self.decoded[index]

    def __str__(self):
        """
        Return a printable version of the article
        """
        return '%s' % self.id

    def __unicode__(self):
        """
        Return a printable version of the article
        """
        return u'%s' % self.id

    def __repr__(self):
        """
        Return an unambigious version of the object
        """

        return '<NNTPArticle Message-ID="%s" attachments="%d" />' % (
            self.id,
            len(self.decoded),
        ) class NNTPResponse(object):
    """
    This is used with the NNTPManager class; specificially the query()
    function.

    you feed NNTPManager.query() NNTPRequest() objects and get NNTPResponse()
    objects in return.
    """

    def __init__(self, code=None, code_str=None, work_dir=None,
                 *args, **kwargs):
        """
        Initializes a request object and the 'action' must be a function
        name that exists in the NNTPConnection(), you can optionally specify
        it the args and kwargs too.
        """

        # The response information is placed here
        self.code = code
        if self.code is None:
            self.code = 0

        self.code_str = code_str
        if self.code_str is None:
            self.code_str = ''

        # Track the time our response object was created; this is
        # useful for multi-threading since we know when our response
        # was generated
        self.created = datetime.now()

        # Our body contains non-decoded content
        self.body = NNTPAsciiContent(work_dir=work_dir)

        # Contains a list of decoded content
        self.decoded = sortedset(key=lambda x: x.key())

        # For iterating over decoded items
        self._iter = None

    def is_success(self, multiline=None):
        """
        Returns True if the code falls in the success category

        If multline is set to None, then we're just checking for a success
        flag. Both (both multi-lined and non success messages are checked
        here). This is the defalt option

        If multline is set to False, then we're just checking for a success
        flag associated with only a non-multi lined response.

        If multline is set to True, then we're just checking for a success
        flag associated with only multi lined response.
        """

        if multiline is None:
            # any 200 reponse is good
            return self.code/200 == 1

        elif multiline is True:
            # Check multiline only
            if self.code in NNTPResponseCode.SUCCESS_MULTILINE:
                return True

        else:
            # Check non-multiline only
            if self.code in NNTPResponseCode.SUCCESS:
                return True

        # No match
        return False

    def detach(self):
        """
        Detach the article stored on disk from being further managed by this
        class
        """
        for entry in self.decoded:
            if isinstance(entry, NNTPContent):
                entry.detach()
        return


    def key(self):
        """
        Returns a key that can be used for sorting with:
            lambda x : x.key()
        """
        return self.body.key()

    def next(self):
        """
        Python 2 support
        Support stream type functions and iterations
        """
        if not self._iter:
            self._iter = iter(self.decoded)

        return next(self._iter)

    def __next__(self):
        """
        Python 3 support
        Support iterating through list
        """
        if not self._iter:
            self._iter = iter(self.decoded)

        return next(self._iter)

    def __iter__(self):
        """
        Mimic iter()
        """
        return iter(self.decoded)

    def __len__(self):
        """
        Returns the length of the article
        """
        length = 0
        for a in self.decoded:
            length += len(a)
        return length

    def __contains__(self, code):
        """
        Enables usage of the 'in' keyword.  This allows us to check
        if a code is set by it's value or by a set/tuple

        """

        if isinstance(code, (set, tuple, list)):
            # find if one item in set matches us
            return self.code in code

        elif isinstance(code, int):
            return code == self.code

    def __str__(self):
        """
        Returns the response information returned from the NNTP Server
        """
        if not self.code:
            return ''
        elif not self.code_str:
            return '%d' % self.code

        return '%d: %s' % (self.code, self.code_str)

    def __repr__(self):
        """
        Return an unambigious version of the object
        """
        return '<NNTPResponse id="%s" />' % id(self) class NNTPResponseCode(object):
    """
    A Simple lookup table that makes the error codes returned by the NNTP
    server a bit more human readable.  These codes are based on:
       - http://tools.ietf.org/html/rfc3977
       - http://tools.ietf.org/html/rfc2980
    """

    # Custom Responce Codes
    NO_ARTICLE_DMCA = 499
    NO_ARTICLE_NUKED = 498

    # There are some codes that aren't defined here which were evented
    # A Custom error code to handle situations where what was
    # received didn't line up to what we should have received
    BAD_RESPONSE = 504
    NO_CONNECTION = 505
    CONNECTION_LOST = 506
    FETCH_ERROR = 507
    INVALID_GROUP = 508
    INVALID_INPUT = 509
    HOOK_OVERRIDE = 510

    # to also work with the NNTP wrapper to make life easy to
    # reference back (simply check you're return code against this
    # list.  If it's not here, then we have a bug and should add it!

    # 200 Service available, posting allowed
    # 201 Service available, posting prohibited
    # 203 Streaming is OK
    # 205 Connection closing
    # 235 Article transferred OK
    # 238 No such article found, please send it to me
    # 239 Article transferred OK
    # 240 Article received OK
    # 250 Authorization accepted
    # 281 Authentication accepted
    # 290 features updated
    SUCCESS = \
        (111, 200, 201, 203, 205, 211, 223, 235, 238, 239, 240, 250, 281, 290)

    # A Success message that will be followed with data
    # This is done when calling NEWSGROUPS, XOVER, etc

    # 100 Help text follows (multi-line)
    # 101 Capability list follows (multi-line)
    # 211 list of article numbers follow (multi-line)
    # 215 Information follows (multi-line)
    # 218 tin-style index follows (multi-line)
    # 224 Overview information follows (multi-line)
    # 225 Headers follow (multi-line)
    # 231 List of new newsgroups follows (multi-line)
    # 230 list of new articles by message-id follows (multi-line)
    # 282 list of groups and descriptions follows (multi-line)
    # 288 Binary data to follow (multi-line)
    SUCCESS_MULTILINE = (
        100, 101, 211, 215, 218, 220, 221, 222, 224, 225, 230, 231, 282, 288)

    # Pending is a state the NNTP server will enter
    # when it's waiting for 'you' to continue to doing something.
    # it's partially a success message, but is awaiting the next
    # set of data from you to proceed

    # 335 Send article to be transferred
    # 340 Send article to be posted
    # 350 Continue with authorization sequence
    # 381 More authentication information required
    PENDING = (335, 381, 340, 350)

    # Posting Failures
    # 400 Service temporarily unavailable
    # 400 not accepting articles
    # 435 Article not wanted
    # 435 Duplicate
    # 437 Article rejected; don't send again
    # 438 Already have it, please don't send it to me
    # 450 Authorization required for this command
    # 480 Transfer permission denied
    # 480 Authentication required
    ACTION_DENIED = (400, 435, 437, 438, 440, 450, 480)

    # 431 Try sending it again later
    # 436 Transfer not possible; try again later
    # 436 Transfer failed
    # 436 Retry later
    # 439 Article transfer failed
    # 441 Posting failed
    # 452 Authentication rejected
    # 482 Authentication rejected
    ACTION_FAILED = (431, 436, 439, 441, 452, 482)

    # 420 No current article selected
    # 420 No article with that number
    # 421 No next article in this group
    # 422 No previous article in this group
    # 423 No article with that number
    # 423 No such article in this group
    # 423 Empty range
    # 430 No article with that message-id
    # 430 No Such Article Found
    NO_ARTICLE = (
        420, 421, 422, 423, 430, 435, NO_ARTICLE_DMCA, NO_ARTICLE_NUKED,
    )

    # 411 No such newsgroup
    # 412 No newsgroup selected
    # 412 Not currently in newsgroup
    # 418 no tin-style index is available for this news group
    # 481 Groups and descriptions unavailable
    NO_GROUP = (411, 412, 418, 481)

    # 500 Command not understood
    # 501 Syntax Error
    # 502 Service permanently unavailable
    # 503 Data item not stored
    # 503 Overview by message-id unsupported
    # 503 program error, function not performed
    ERROR = (
        500, 501, 502, 503, BAD_RESPONSE, NO_CONNECTION, INVALID_GROUP,
        CONNECTION_LOST,
    ) class NNTPContent(object):
    """
    An object for maintaining retrieved article content. There can only
    be 1 article, however that 1 article can have a lot of content
    found within it.

    This identifies the content found internally within an article.

    This function does it's best to behave like a stream. But provides
    some functions to make manipulating and merging with other articles
    easier to do.

    Articles by default assume a roll of 'attached'.  This means that
    the files written to disk are removed the second the object is
    destroyed.  This is intentional!  You can call detach() at any
    time you want but now you are responsible for cleaning up the
    filename.

    """

    def __init__(self, filepath=None, part=None, total_parts=None,
                 begin=None, end=None, total_size=None, work_dir=None,
                 sort_no=10000, unique=False, *args, **kwargs):
        """
        Initialize NNTP Content

        If a filepath is specified, it can be either a stream (already opened
        file or ByteIO or StringIO class is fine too), or it can be a path to
        a filename which will be open in 'wb' mode.

        if unique is set to True, then if the file passed in already exists,
        a temporary file is used instead. But the filename info isn't lost.

        Set the unique to True when you don't want to accidently over-write
        or alter a file you may already be working with.
        """

        # The sort is used with sorting; different filetypes/content types
        # should be processed before otheres.

        # For example, the NNTPHeader() and NNTPMetaContent() area always kept
        # at the head where as NNTPAsciiContent() and NNTPBinaryContent() are
        # kept at the back.

        # The lower sort no is always processe first; but default we choose
        # a rather large sort value. Grouped content should share the same sort
        # value so that they sort their content together.
        self.sort_no = sort_no

        # A _unique string is used to ensure our file is always different
        # from another
        self._unique = False
        if unique:
            # Convert to string for indexing speed in key() calls since
            # the index is already in string format
            self._unique = str(id(self))

        # Default filename
        self.filename = ''

        # The filepath is automatically set up when the temporary file is
        # created
        self.filepath = None

        # used to track the filemode (saves on time from opening and closing
        # un-nessisarily).  These flags are set during an open and a close
        self.filemode = None

        # Prepare temporary folder we intend to use by default
        if work_dir:
            self.work_dir = abspath(expanduser(work_dir))
        else:
            self.work_dir = DEFAULT_TMP_DIR

        # A Stream object
        self.stream = None

        # Detached prevents the article from cleaning up all of
        # the data it otherwise tracks (such as the article stored
        # on disk)
        #
        # If set to None, then it hasn't been initalized yet
        self._detached = None

        # Dirty flag is set to true if a write is made; all data
        # is considered dirty until flush() is called (forcing
        # contents out of cache and onto disk)
        self._dirty = False

        # A flag that can be toggled if the data stored is
        # corrupted in some way. Such as through CRC Failing
        # or part construction (a part missing perhaps, etc)
        # if all is good, then we just leave the flag as is
        self._is_valid = False

        # Store part
        self.part = 1
        if part is not None:
            try:
                self.part = int(part)

            except (ValueError, TypeError):
                raise AttributeError(
                    "Invalid part specified (%s)." %
                    str(part),
                )

        # Tracks parts (most used for posting/encoding)
        self.total_parts = 1
        if total_parts is not None:
            try:
                self.total_parts = int(total_parts)

            except (ValueError, TypeError):
                raise AttributeError(
                    "Invalid total_parts specified (%s)." %
                    str(total_parts),
                )

            if self.total_parts < self.part:
                raise AttributeError(
                    "Invalid parts/total_parts specified (%s/%s)." % (
                        str(part), str(total_parts),
                    )
                )
        else:
            self.total_parts = self.part

        # Used for tracking the indexes (head/tail) that make
        # up the block of data this NNTPContent object represents.
        # this is only used if split() is called
        self._begin = 0
        if begin is not None:
            try:
                self._begin = int(begin)

            except (ValueError, TypeError):
                raise AttributeError(
                    "Invalid begin specified (%s)." % str(begin),
                )

        self._end = None
        if end is not None:
            try:
                self._end = int(end)

            except (ValueError, TypeError):
                self._end = None

        # Tracks total_size (used for posting when we have parts of files
        self._total_size = None
        if total_size is not None:
            try:
                self._total_size = int(total_size)

            except (ValueError, TypeError):
                raise AttributeError(
                    "Invalid total_size specified (%s)." %
                    str(total_size),
                )

        # Set blocksize as a variable for those who want to tweak it
        # later on.  This controls the maximum amount of content read
        # from a file in one chunk (thus how much memory will be occupied
        # when called).  Setting this too high will utilize a lot of
        # memory if concurrent calls are made but will be faster if your
        # system can handle it.
        self._block_size = BLOCK_SIZE

        # Reserved for split() function which populates any children
        # spawned from this object with a pointer back to this as
        # object as a reference.
        self._parent = None

        # TODO: all md5, crc32, len() calls etc should all cache their results
        # here and retrieve from here if present. If a write() or load() is
        # made, the cache should be destroyed. The cache is only populated on
        # demand.
        self._lazy_cache = {}

        # NNTPContent supports directory storing too. This is toggle in the
        # event we're dealing with a directory
        self._isdir = False

        if not filepath:
            # Will use load() or open() which causes temp file
            # to be created
            return

        elif isinstance(filepath, file):

            # store stream
            self.stream = filepath

            # Always detach streams
            self._detached = True

            if hasattr('name', self.stream):
                self.filepath = abspath(self.stream.name)
                self.filename = basename(filepath)

            if hasattr('mode', self.stream):
                # Store our mode
                self.filemode = self.stream.mode
        else:
            if self._unique is False and isfile(filepath):
                self.load(filepath, sort_no=sort_no)

            elif isdir(filepath):
                # Toggle Flag
                self._isdir = True
                # Directories are always detached
                self._detached = True
                # Store our dirname
                self.filename = basename(filepath)
                # Store our path
                self.filepath = abspath(expanduser(filepath))

            else:
                # Store our filename
                self.filename = basename(filepath)

    def getvalue(self):
        """
        This is mostly just used for unit testing, but it
        greatly makes life easier anyway.

        Effectively we put the pointer at the head of our
        file and read back the entire chunk into memory
        and return it.
        """

        if not self.open(mode=NNTPFileMode.BINARY_RO, eof=False):
            # Error
            return None

        # Head of data
        self.stream.seek(0L, SEEK_SET)

        return self.read()

    def can_post(self):
        """
        Similar to is_valid() except only returns true if the item is
        postable. The big difference is is_valid() is toggle after something
        has been downloaded, where as can_post() is a field checked prior
        to posting to an NNTP Server.

        Content is only postable in ascii form, so binaries and directories
        will always return a False here.
        """
        return False

    def is_valid(self):
        """
        A simple function that returns whether the article is valid or not

        The function returns True if it is valid, and False if it isn't
        and None if there isn't enough information to make a valid guess.

        The basic version (no overloading) just returns what the flag
        was set to.

        A directory can never be valid because it isn't content per say
        but a container to content.
        """
        return self._is_valid and not self._isdir

    def open(self, filepath=None, mode=None, eof=False):
        """
        Opens a filepath specified and re-attaches to it.
        You can also pass in an already open stream which
        causes it to operate in a detached state

        default open mode is BINARY_RW

        if eof is set to True, then after the file is opened, the
        pointer is placed at the end of the file (oppose to
        the head)
        """

        if not mode:
            # Read and write
            mode = NNTPFileMode.BINARY_RW

        if self.stream is not None:
            if self.filemode is not None and self.filemode == mode:
                # ensure we're at the head of the file
                if not eof:
                    self.stream.seek(0L, SEEK_SET)
                else:
                    self.stream.seek(0L, SEEK_END)

                return weakref.ref(self.stream)

        if not filepath and self.filepath:
            # Update filepath
            filepath = self.filepath

        elif not filepath:
            if not isdir(self.work_dir):
                # create directory
                mkdir(self.work_dir)

            # Create a Temporary File
            fileno, self.filepath = mkstemp(dir=self.work_dir)
            try:
                self.stream = fdopen(fileno, mode)
                if self._detached is None:
                    self._detached = False

                # save the last mode the file was opened as
                self.filemode = mode

                logger.debug(
                    'Opened %s (mode=%s)' %
                    (self.filepath, mode),
                )

            except (IOError, OSError) as e:
                logger.error(
                    'Could not open %s (mode=%s)' %
                    (self.filepath, mode),
                )
                logger.debug(
                    'fdopen({0}, {1}, wd={2}) exception ({3})'.format(
                        fileno, mode, self.work_dir, str(e)))
                return False

            return weakref.ref(self.stream)

        if isinstance(filepath, basestring):

            # expand our path to be absolute
            filepath = abspath(expanduser(filepath))

            # Create our stream
            try:
                self.stream = open(filepath, mode)

                self.filepath = filepath
                if self._detached is None:
                    self._detached = True

                # save the last mode the file was opened as
                self.filemode = mode

                logger.debug(
                    # D flag for Detached
                    'Opened %s (mode=%s) (flag=D)' %
                    (self.filepath, mode),
                )

            except (IOError, OSError) as e:
                logger.error(
                    'Could not open %s (mode=%s) (flag=D)' %
                    (self.filepath, mode),
                )
                logger.debug(
                    'open({0}, {1}, wd={2}) exception ({3})'.format(
                        filepath, mode, self.work_dir, str(e)))
                return False

        elif hasattr(filepath, 'seek'):
            # assume we're dealing with an already open stream and therefore
            # we work in a detached state
            self.stream = filepath
            self.filepath = filepath.get('name')
            self.filemode = filepath.get('mode')

            if self.filepath:
                self.filename = basename(self.filepath)
            else:
                self.filename = ''

            # You can never have an attached file without a filepath
            self._detached = False

            # Reset dirty flag
            self._dirty = False

        else:
            logger.error(
                'Could not open object %s' % (type(self.filepath))
            )
            return False

        if not eof:
            # Ensure we're at the head of the file
            self.stream.seek(0L, SEEK_SET)

        else:
            # Ensure we're at the end of the file
            self.stream.seek(0L, SEEK_END)

        return weakref.ref(self.stream)

    def encode(self, encoder):
        """
        A wrapper to the encoding of content. The function returns None if
        a problem occurs, otherwise the function returns an NNTPContent()
        object.

        The power of this function comes from the fact you can pass in
        multiple encoders to have them all fire after one another.
        """
        # Python does not allow recursive inclusion; since NNTPContent is
        # included via the codec paths we test if it's an encoder by just
        # looking for the encode() function
        if not isinstance(encoder, object):
            return None

        if not isinstance(encoder, (list, tuple, sortedset, list)):
            # work with a tuple for now
            encoder = (encoder, )

        # Content object we chain to
        content = self

        for _enc in encoder:
            # Support Type initializations
            if isinstance(_enc, type):
                enc = _enc()
            else:
                enc = _enc

            if hasattr(enc, 'encode') and \
               isinstance(enc.encode, MethodType):
                # We're dealing with a stream based encoder
                content = enc.encode(self)
                if content is None:
                    return None

            else:
                # We don't support this
                return None

        return content

    def load(self, filepath, sort_no=10000):
        """
        This causes the function to point to the file specified and acts in a
        detached manner to it.

        Loaded files are 'always' detached; if the file already existed in the
        filesystem it would be silly to tie it to the scope of this object.

        If the filepath identifie is an NNTPContent() object, then it is copied
        into an 'attached' NNTPContent() object.

        If the filepath identified is a set, sortedset or list of
        NNTPContent() objects, a new single 'attached' file will be generated
        based on the contains passed in.  NNTPContent() objects are
        automatically combined/merged based on their ordering by automating
        the use of the .append() function of this class for the caller.

        """

        if self.stream is not None:
            if self.filemode == NNTPFileMode.BINARY_WO_TRUNCATE:
                # Truncate remaining portions of the file
                self.truncate()

            # Close any existing open file
            self.close()

        if self._detached is False and self.filepath:
            # We're changing so it's better we unlink this (but only if we're
            # attached to it)
            rm(self.filepath)

        # Support directories but initialize field to false
        self._isdir = False

        # Set Detached since we don't want to obstruct our newly loaded
        # file in any way
        self._detached = True

        # Reset Valid Flag
        self._is_valid = False

        # Reset Unique Flag
        self._unique = False

        if isinstance(filepath, NNTPContent):
            # Support NNTPContent object copying; by simply storing
            # the object in a list, we are able to catch it in the
            # next check
            self.part = filepath.part
            filepath = [filepath]

        if isinstance(filepath, (tuple, set, sortedset, list)):
            # Perform merge if we detected a set of NNTPContent objects
            count = 0
            for content in filepath:
                if isinstance(content, NNTPContent):
                    self.append(content)
                    count += 1

            if count == 0 or len(filepath) != count:
                # Return True if we iterated over everything
                return False

            # update our filepath to be that of the file that was actually
            # created
            filepath = self.filepath

            # Our file is not detached in this state
            self._detached = False

        elif isdir(filepath):
            # Toggle our flag and fall through as we support directories
            self._isdir = True

        elif not isfile(filepath):
            # we can't load the file so reset some common variables
            self.filepath = None
            self.filename = ''

            return False

        # Assign new file
        self.filepath = filepath

        # Set Flag
        self._is_valid = True

        # Store our filename
        self.filename = basename(filepath)

        return True

    def copy(self):
        """
        copy is very close to save(); this is especially the case since
        the save() function has a copy parameter.

        the difference is copy actually duplicates/clones the content
        object as a completely new file that is not detached and therefore
        this copy (by default) is destroyed when the object goes out
        of scope.

        This is useful for handling downloads; one might want to create
        a copy of the original object and build onto it or alter it. With
        a copy to work with, you don't have to worry about obstructing the
        original file.

        copy() is just a clean way to simplify your code by wrapping
        the save() function with error handling.  Unlike save()
        which returns True if it was successful and False if it failed),
        copy() returns a new NNTPContent object.

        The function returns None if there was a failure

        """

        with pushd(self.work_dir, create_if_missing=True):
            # mkstemp used to genrate an unused temporary file
            _, filepath = mkstemp(dir=self.work_dir)
            try:
                # Remove the created file to silence any warnings
                # from the save() call coming next
                unlink(filepath)
            except:
                pass

        if not self.save(filepath, copy=True):
            return None

        # Initialize a new object
        obj = NNTPContent(
            filepath, part=self.part, total_parts=self.total_parts,
            begin=self._begin, end=self._end, total_size=self._total_size,
            work_dir=self.work_dir, sort_no=self.sort_no, unique=False,
        )

        # Save our official filename
        obj.filename = self.filename

        # Ensure our copy is attached
        obj.attach()

        return obj

    def save(self, filepath=None, copy=False):
        """
        This function writes the content to disk using the filename
        specified.

        If copy is False, then the content detaches itself from internal
        management and is moved to the new path specified by filepath.
        If copy is True, then content is written to the new location
        and remains in it's current detached state (whatever it was).

        If no filepath is specified, then the detected filename and
        work_dir specified during the objects initialization is used instead.

        The function returns True if it successfully saved the file and
        False otherwise. If you never passed in a filepath, then the path
        that was last loaded is saved to instead and the file is automatically
        detached from the Object.
        """
        if filepath:
            if not isfile(filepath):
                # If the file wasn't found relative to where we are, we'll try
                # again but relative to the work dir
                with pushd(self.work_dir, create_if_missing=True):
                    if isfile(filepath):
                        # Ensure we've expanded the file path
                        filepath = abspath(filepath)
                    else:
                        # Attempt to expand path
                        filepath = abspath(expanduser(filepath))
            else:
                # Ensure we've expanded the file path
                filepath = abspath(filepath)

        if filepath is None:
            if self.filename:
                filepath = join(self.work_dir, self.filename)
            else:
                filepath = join(self.work_dir, basename(self.filepath))

        elif isdir(filepath):
            if self.filename:
                filepath = join(filepath, basename(self.filename))
            else:
                filepath = join(self.work_dir, basename(self.filepath))

        if isfile(filepath):
            if self.path() != filepath:
                try:
                    unlink(filepath)
                    logger.warning('%s already existed (removed).' % (
                            filepath,
                        )
                    )
                except:
                    logger.error(
                        '%s already existed (and could not be removed).' % (
                            filepath,
                        )
                    )
                    return False

        # else: treat it as full path and filename included
        if not isdir(dirname(filepath)):
            # Attempt to pre-create save path
            if not mkdir(dirname(filepath)):
                return False

        if self.stream:
            # close the file if it's open
            self.close()

        # Function Wrapping
        if copy:
            action = _copy
            action_str = "copy"

        else:
            action = _move
            action_str = "move"

        if self.path() != filepath:
            try:
                action(self.filepath, filepath)

                logger.debug('%s(%s, %s)' % (
                    action_str, self.filepath, filepath,
                ))

            except ShutilError, e:
                logger.debug('%s(%s, %s) exception %s' % (
                    action_str, self.filepath, filepath, str(e),
                ))
                return False

        if not copy:
            # If we reach here, we want to treat moves() as an official
            # way of saving/writing the file to it's final destination
            # and therefore we can update our object

            # Detach File
            self._detached = True
            # Update filepath
            self.filepath = filepath
            # Update filename
            self.filename = basename(filepath)

        return True

    def path(self):
        """
        Always returns a filepath of the file, if one hasn't been created yet
        then one is automatically generated and returned.
        """
        if not self.filepath:
            # Create a Temporary File
            _, self.filepath = mkstemp(dir=self.work_dir)

        return self.filepath

    def split(self, size=81920, mem_buf=1048576):
        """Returns a set of NNTPContent() objects containing the split version
        of this object based on the criteria specified.

        Even if the object can't be split any further given the parameters, a
        set of at least 1 entry will always be returned.  None is returned if
        an error occurs.

        """
        # File Length
        file_size = len(self)
        if file_size == 0:
            # Object can not be split
            return None

        if not isinstance(size, int):
            # try to support other types
            size = strsize_to_bytes(size)

        if not size or size < 0:
            return None

        if not isinstance(mem_buf, int):
            # try to support other types
            mem_buf = strsize_to_bytes(mem_buf)

        if not mem_buf or mem_buf < 0:
            return None

        # Initialize Part #
        part = 0

        # Initialize Total Part #
        total_parts, partial = divmod(len(self), size)
        if partial:
            total_parts += 1

        # A lists of NNTPContent() objects to return
        objs = sortedset(key=lambda x: x.key())

        if not self.open(mode=NNTPFileMode.BINARY_RO):
            return None

        # Calculate the total length of our data
        total_size = len(self)

        # File length of our first object
        f_length = 0

        # Total Bytes Read
        total_bytes = 0

        # Initialize dummy value
        obj = None

        # Now read our chunks as per our memory restrictions
        while True:

            if total_bytes == 0:
                # Read memory chunk
                data = BytesIO(self.stream.read(mem_buf-total_bytes))

                # Retrieve length of our buffer
                total_bytes = data.seek(0L, SEEK_END)

                # Reset our pointer to the head of our data stream
                data.seek(0L, SEEK_SET)

                if total_bytes == 0:
                    if f_length > 0:
                        # Store our last object before we wrap up
                        obj.close()
                        objs.add(obj)
                    # Return our list of NNTPContent() objects
                    return objs

            while total_bytes > 0 and size-f_length > 0:
                # Store content up to our alotted amount
                block_size = size-f_length
                if total_bytes < block_size:
                    block_size = total_bytes

                if f_length == 0:

                    # Create a new object
                    obj = NNTPContent(
                        filepath=self.filename,
                        part=part+1,
                        total_parts=total_parts,
                        begin=(part*size),
                        end=((part*size)+size),
                        total_size=total_size,
                        work_dir=self.work_dir,
                        sort_no=self.sort_no,
                    )

                    # Increment our part
                    part += 1

                    # Create a pointer to the parent
                    obj._parent = weakref.proxy(self)

                    # Open the new file
                    obj.open(mode=NNTPFileMode.BINARY_WO_TRUNCATE)

                try:
                    obj.write(data.read(block_size))

                except IOError, e:
                    if e[0] is errno.ENOSPC:
                        # most probably a disk space issue
                        logger.error(
                            'Ran out of disk space while writing %s.' %
                            (obj.filepath),
                        )
                    else:
                        # most probably a disk space issue
                        logger.error(
                            'An I/O error '
                            '(%d) occured while writing %s to disk.' %
                            (e[0], obj.filepath),
                        )

                    # Tidy
                    self.close()

                    # Return None
                    return None

                f_length += block_size
                total_bytes -= block_size

            if f_length == size:
                # We're done
                obj.close()
                objs.add(obj)

                # File length reset
                f_length = 0

        # code will never reach here
        return None

    def write(self, data, eof=True):
        """
        Writes data to stream

        eof is only considered if the file wasn't open prior to the write()
        call. Otherwise the pointer remains where it last was. If set to True
        and the file wasn't previously open, the pointer is automatically
        placed at the end of the stream.

        """
        if self.stream is None:
            # open the file if it's not already open
            self.open(mode=NNTPFileMode.BINARY_RW, eof=eof)

        response = self.stream.write(data)

        if not self._dirty:
            # Set dirty flag
            self._dirty = True

            # We can't trust self._end anymore now because content was
            # written to the file.
            self._end = None

        return response

    def read(self, n=-1):
        """
        read up to n bytes from the stream
        """
        if self.stream is None:
            # open the file if it's not already open
            self.open(mode=NNTPFileMode.BINARY_RO, eof=False)

        return self.stream.read(n)

    def close(self):
        """
        Closes the file but retains any attachment to it.
        """
        if self.stream is not None:
            try:
                self.stream.close()
                if self.filepath:
                    logger.debug('Closed %s' % (self.filepath))
                else:
                    logger.debug('Closed stream.')
            except:
                pass

            self.stream = None
            self.filemode = None

            # A closed file can't be dirty as content is
            # flushed to disk at this point; therefore reset the flag
            # back to False
            self._dirty = False

        return

    def append(self, content):
        """
        This function takes a content object (or list of content objects) and
        appends them to `this` object.

        """
        if isinstance(content, NNTPContent):
            content = [content]

        if not self.open(mode=NNTPFileMode.BINARY_WO, eof=True):
            return False

        for entry in content:
            if isinstance(entry, NNTPContent):
                # Just append the current content
                if not entry.open(mode=NNTPFileMode.BINARY_RO, eof=False):
                    logger.debug('Error handling content: %s' % entry)
                    continue

                logger.debug('Appending content %s' % entry)

                while True:
                    buf = entry.stream.read(self._block_size)
                    if not buf:
                        # Set dirty flag
                        self._dirty = True
                        break
                    self.stream.write(buf)

                entry.close()

        return True

    def begin(self):
        """
        Returns the beginning ptr; this is nessisary when building encoded
        parts to be posted on an NNTPServer
        """
        if self._begin:
            return self._begin
        return 0

    def end(self):
        """
        Returns the end ptr; this is nessisary when building encoded
        parts to be posted on an NNTPServer
        """
        if self._end is None:
            self._end = self._begin + len(self)
        return self._end

    def total_size(self):
        """
        Returns the total size of the entire object (all parts included).
        This result is the same as len() if there is only 1 part to
        the entire object
        """
        if self.total_parts <= 1:
            return len(self)

        return self.end() - self.begin()

    def is_attached(self):
        """
        Simply returns whether or not the file is attached to the object or
        not.  Files that are attached are destroyed when the object goes
        out of scope.

        """
        return not self._detached

    def detach(self):
        """
        Detach the article stored on disk from being further managed by this
        class
        """
        self._detached = True
        return

    def attach(self):
        """
        Attach the file and it's data directly to this objects life expectancy
        """
        self._detached = False
        return

    def remove(self):
        """
        Gracefully remove the file (attached or not)
        """
        if self.stream is not None:
            self.close()

        if self.filepath:
            return rm(self.filepath)

        return False

    def key(self):
        """
        Returns a key that can be used for sorting with:
            lambda x : x.key()
        """
        if self.part is not None:
            result = '%.5d/%s/%.5d' % (self.sort_no, self.filename, self.part)
        else:
            result = '%.5d/%s//' % (self.sort_no, self.filename)

        if self._unique is not False:
            return result + self._unique

        return result

    def post_iter(self, block_size=BLOCK_SIZE):
        """
        Returns NNTP string as it would be required for posting to an
        NNTP Server
        """
        if not block_size:
            block_size = self._block_size

        if self.open(mode=NNTPFileMode.BINARY_RO):
            while 1:
                data = self.stream.read(block_size)
                if not data:
                    break
                yield data
            self.close()

    def crc32(self):
        """
        A little bit old-fashioned, but some encodings like yEnc require that
        a crc32 value be used.  This calculates it based on the file
        """
        # block size defined as 2**16
        block_size = 65536

        # The mask to apply to all CRC checking
        BIN_MASK = 0xffffffffL

        # Initialize
        _crc = 0

        if self.open(mode=NNTPFileMode.BINARY_RO):
            for chunk in iter(lambda: self.stream.read(block_size), b''):
                _crc = crc32(chunk, _crc)

            return format(_crc & BIN_MASK, '08x')

        return None

    def mime(self):
        """
        Returns the mime of the object
            Source: https://github.com/ahupp/python-magic
        """

        # Initialize our Mime object
        m = Mime()

        if self.open(mode=NNTPFileMode.BINARY_RO):
            mr = m.from_content(self.stream.read(128))
        else:
            mr = None

        if mr is None or mr.type() == DEFAULT_MIME_TYPE:
            # Try one more time by the filename
            mr = m.from_filename(
                self.filename if self.filename else self.filepath)

        # Return our type
        return mr

    def md5(self):
        """
        Simply return the md5 hash value associated with the content file.

        If the file can't be accessed, then None is returned.
        """
        md5 = hashlib.md5()
        if self.open(mode=NNTPFileMode.BINARY_RO):
            for chunk in \
                    iter(lambda: self.stream.read(128*md5.block_size), b''):
                md5.update(chunk)
            return md5.hexdigest()
        return None

    def sha1(self):
        """
        Simply return the sha1 hash value associated with the content file.

        If the file can't be accessed, then None is returned.
        """
        sha1 = hashlib.sha1()
        if self.open(mode=NNTPFileMode.BINARY_RO):
            for chunk in \
                    iter(lambda: self.stream.read(128*sha1.block_size), b''):
                sha1.update(chunk)
            return sha1.hexdigest()
        return None

    def sha256(self):
        """
        Simply return the sha256 hash value associated with the content file.

        If the file can't be accessed, then None is returned.
        """
        sha256 = hashlib.sha256()
        if self.open(mode=NNTPFileMode.BINARY_RO):
            for chunk in \
                    iter(lambda: self.stream.read(128*sha256.block_size), b''):
                sha256.update(chunk)
            return sha256.hexdigest()
        return None

    def tell(self):
        """
        Allows reference to our object from within a Codec()

        """
        if not self.filepath:
            # If there is no filepath, then we're probably dealing with a
            # stream in memory like a StringIO or BytesIO stream.
            if self.stream:
                # Advance to the end of the file
                return self.stream.tell()

        else:
            if self.stream and self._dirty is True:
                self.stream.flush()
                self._dirty = False

        if not self.stream:
            if not self.open(mode=NNTPFileMode.BINARY_RO):
                return None

        return self.stream.tell()

    def readline(self, *args, **kwargs):
        """
        Returns a single line from the stream
        """
        if not self.stream:
            return ''
        return self.stream.readline(*args, **kwargs)

    def next(self):
        """
        Python 2 support
        Support stream type functions and iterations
        """
        data = self.stream.read(self._block_size)
        if not data:
            self.close()
            raise StopIteration()

        return data

    def __next__(self):
        """
        Python 3 support
        Support stream type functions and iterations
        """
        data = self.stream.read(self._block_size)
        if not data:
            self.close()
            raise StopIteration()

        return data

    def __iter__(self):
        """
        Grants usage of the next()
        """

        # Ensure our stream is open with read
        self.open(mode=NNTPFileMode.BINARY_RO)
        return self

    def __len__(self):
        """
        Returns the length of the content
        """
        if not self.filepath:
            # If there is no filepath, then we're probably dealing with a
            # stream in memory like a StringIO or BytesIO stream.
            if self.stream:
                # Advance to the end of the file
                ptr = self.stream.tell()
                # Advance to the end of the file and get our length
                length = self.stream.seek(0L, SEEK_END)
                if length != ptr:
                    # Return our pointer
                    self.stream.seek(ptr, SEEK_SET)
            else:
                # No Stream or Filepath; nothing has been initialized
                # yet at all so just return 0
                length = 0
        else:
            if self.stream and self._dirty is True:
                self.stream.flush()
                self._dirty = False

            # Get the size
            length = getsize(self.filepath)

        return length

    def hexdump(self, max_bytes=128):
        """
        Returns a hex dump of characters up to the defined max_bytes
        If max_bytes is 0 then all content is dumped

        """

        if not self.open(mode=NNTPFileMode.BINARY_RO, eof=False):
            # Error
            return None

        # Head of data
        self.stream.seek(0L, SEEK_SET)

        if not max_bytes:
            return hexdump(self.read())
        return hexdump(self.read(max_bytes))

    def __del__(self):
        """
        Gracefully remove the file retrieved as it was removed
        from scope for a good reason. We can easily avoid
        having this step called by calling the detach() function
        """
        if self.stream is not None:
            self.close()

        if not self._detached and self.filepath:
            # We need to do some cleanup
            rm(self.filepath)

    def __lt__(self, other):
        """
        Support Less Than (<) operator for sorting
        """
        return self.key() < self.key()

    def __cmp__(self, content):
        """
        Support comparative checks
        """
        return cmp(self.key(), content.key())

    def __str__(self):
        """
        Return a printable version of the file being read
        """
        if self.part is not None:
            return '%s.%.5d' % (self.filename, self.part)

        return self.filename

    def __enter__(self):
        """
        supports use of the 'with' clause.  You can use the expression:

        with myobj as fp:
            # write and/or read content here

        """
        # Open our file and return a pointer to it
        if self.open():
            return self

        # Throw an exception
        raise IOError(errno.EIO, 'Could not open NNTPContent', self.path())

    def __exit__(self, type, value, traceback):
        """
        our exit function executed from the 'with' clause
        """
        # Close our file (if open)
        self.close()

    def __repr__(self):
        """
        Return a printable version of the file being read
        """
        if self.part is not None:
            return \
                '<NNTPContent sort=%d filename="%s" part=%d/%d len=%s />' % (
                    self.sort_no,
                    self.filename,
                    self.part,
                    self.total_parts,
                    bytes_to_strsize(len(self))
                )
        else:
            return '<NNTPContent sort=%d filename="%s" len=%s />' % (
                self.sort_no,
                self.filename,
                bytes_to_strsize(len(self)),
            ) class NNTPSegmentedPost(object):
    """
    An object for maintaining retrieved nzb content. Large files need
    to be split across multiple Articles in order to be posted.

    When combined into one, They create a SegmentedPost

    Similarily; this file makes posting easy by allowing you to add files
    to it in order to prepare an object worthy of posting.

    Every file you wish to post to an NNTP Server should be done through
    this file for the most power/control

    """

    def __init__(self, filename, subject=DEFAULT_NNTP_SUBJECT,
                 poster=DEFAULT_NNTP_POSTER, groups=None,
                 utc=None, work_dir=None, sort_no=None, codecs=None,
                 *args, **kwargs):
        """Initialize NNTP Segmented File

        Args:
            filename (str): Either a path of an existing filename or if
                            you're initializing a new file from scratch this
                            should be the name of the filename being
                            assembled.
        Kwargs:
            subject (str): The subject to associate with the file. This is
                            only required if you're posting the content (to
                            Usenet).
            poster (str): The poster to associate with the file. This is
                           only required if you're posting the content (to
                           Usenet).
            group (str, set): This can be either a string identifying a single
                               Usenet group, or it can be a set() of strings
                               identifying all of the Usenet groups you want
                               to cross-post to. This is only required if you
                               intend on posting the content (to Usenet).
            utc (int, datetime): Should always be in UTC format and can be
                                either defined as the number of seconds from
                                epoch (usually produced from gmtime()) or it
                                can be a datetime() object. This identifies the
                                date to associate with the file.  It's mostly
                                used for sorting.  If no time is specified
                                then it defaults to datetime.utcnow(). If a
                                valid `filename` was specified then the utc
                                defaults to the time associated with the files
                                modification date.
            codecs (CodecBase): The codec to use as our deobsfucation engine
                                You can specify as many Codecs as you want in
                                an interable form.

        Returns:
            Nothing

        Raises:
            AttributeError() if you don't follow group isn't a set() of str(),
                              None or str().

        """
        # The Filename
        self.filename = filename

        # These fields get populated when reading in an nzb file
        self.poster = poster
        self.utc = utc
        self.subject = subject

        # The sort order which helps with file ordering, if set to None
        # or a common value, then it has no bearing on anything.  Its
        # main goal is to provide a method of sorting multiple SegmentedPosts
        self.sort_no = sort_no

        # When writing files, this is the memory buffer to fill when
        # dealing with very large files. If this is set to None, then
        # no buffer is used. Default 500K
        self.mem_buffer = 512000

        if not isinstance(self.utc, datetime):
            # Convert into datetime
            try:
                self.utc = datetime.fromtimestamp(int(self.utc))

            except (TypeError, ValueError):
                # Used None, uninitialized, or Used bad value
                # Default timezone to 'now' but make it consistent
                # with the world, use the UTC as a common source
                self.utc = datetime.utcnow()

        # Load our Codecs
        self._codecs = codecs

        # Provide default codecs if required
        if not self._codecs:
            self._codecs = [CodecYenc(), ]

        elif isinstance(self._codecs, CodecBase):
            self._codecs = [self._codecs, ]

        # A sorted set of articles
        self.articles = sortedset(key=lambda x: x.key())

        if work_dir is None:
            self.work_dir = DEFAULT_TMP_DIR
        else:
            self.work_dir = abspath(expanduser(work_dir))

        # The group(s) associatd with our article(s)
        self.groups = NNTPGroup.split(groups)

        if self.filename:
            # attempt to add our filename
            self.add(filename)

    def pop(self, index=0):
        """
        Pops an Article at the specified index out of the segment table
        """
        return self.articles.pop(index)

    def add(self, content):
        """
        Add an NNTPArticle()
        """

        if isinstance(content, basestring):
            # A mime object we can use to detect the type of file
            m = Mime()

            if isfile(content):
                # file relative to current path; get our mime response
                mr = m.from_bestguess(content)

                # Our NNTPContent object will depend on whether or not we're
                # dealing with an ascii file or binary
                instance = NNTPBinaryContent \
                    if mr.is_binary() else NNTPAsciiContent

                content = instance(
                    filepath=content,
                    work_dir=self.work_dir,
                )

            else:
                with pushd(self.work_dir, create_if_missing=True):
                    # Not relative; is the file relative to the work_dir?
                    if isfile(content):
                        # Create a content object from the data
                        # This isn't always the best route because no part #'s
                        # are assigned this way; but if it's only a single file
                        # the user is working with; this way is much easier.

                        # file relative to current path; get our mime response
                        mr = m.from_bestguess(content)

                        # Our NNTPContent object will depend on whether or not
                        # we're dealing with an ascii file or binary
                        instance = NNTPBinaryContent \
                            if mr.is_binary() else NNTPAsciiContent

                        content = instance(
                            filepath=content,
                            work_dir=self.work_dir,
                        )

                    else:
                        # Nothing to add
                        return False

                    # At this point we fall through and the next set of
                    # if checks will catch our new content object we created

        # duplicates are ignored in a blist and therefore
        # we just capture the length of our list before
        # and after so that we can properly return a True/False
        # value
        _bcnt = len(self.articles)

        if isinstance(content, NNTPContent):
            # Create an Article and store our content
            article = NNTPArticle(
                subject=self.subject,
                poster=self.poster,
                groups=self.groups,
                work_dir=self.work_dir,
            )

            # Add the content to our article
            article.add(content)

            # Store the article in our collection
            self.articles.add(article)

        elif isinstance(content, NNTPArticle):
            # Add our article
            self.articles.add(content)

        return len(self.articles) > _bcnt

    def apply_template(self, custom=None, relative=None, strftime=True):
        """
        Iterates over all of the articles defined in the segment and replaces
        any defined {{directive}} with their translated value. All tokens must
        be wrapped in {{ }} braces.

        There are some built in directives, but you can alternatively provide
        your own as well.  The built in directives are:

            {{filename}}        : Translates to the filename associated with
                                    the 'first' attachment inside of the
                                    article.

            {{filesize}}        : Translates to the size of the 'first'
                                    attachment inside of the article

            {{filenameXXX}}     : Handles situations where you know there are
                                    multiple attachments and you want to
                                    exclusively index the filename of one of
                                    these entries. XXX is a numeric
                                    represetation starting a 001 and can range
                                    to 999

            {{filesizeXXX}}     : Handles situations where you know there are
                                    multiple attachments and you want to
                                    exclusively index the filesize of one of
                                    these entries. XXX is a numeric
                                    represetation starting a 001 and can range
                                    to 999

            {{totalsize}}       : Translates to the total size of all
                                    accumulated attachments in the article.

            {{index}}           : Translates to NNTPArticles index as
                                    represented inside of the SegmentedPost

            {{count}}           : Translates to total number of NNTPArticles
                                    defined inside of the SegmentedPost

        You can add additionally use your own custom dictionary to which you
        just provide it keys and their translated values.  If you provide any
        custom dictionary, it will always be applied 'before' the built in
        ones.  This allows you to over-ride them if you like.

        All braces are replaced, so in the event braces are detected that
        could not be translated, then they are replaced with 'nothing'

        if strftime is set to True, then the contents of the subject is also
        passed through datetime.strftime() as a last task. this allows custom
        templating to even insert datetime directives if required knowing they
        will be masked shortly thereafter.

        The time strftime() bases itself off is relative to the time specified
        (the actual variable 'relative').  If 'relative' is not specified then
        'now()' is used instead

        The following information was taken from:
                https://docs.python.org/2/library/datetime.html

        It will give you an idea of what datetime objects can be set to. Note
        that datetime.stftime() entries are 'NOT' wrapped in {{ }} objects.

        %a  Weekday as locale’s abbreviated name.
                Sun, Mon, …, Sat (en_US)
                So, Mo, …, Sa (de_DE)

        %A  Weekday as locale’s full name.
                Sunday, Monday, …, Saturday (en_US)
                Sonntag, Montag, …, Samstag (de_DE)

        %w  Weekday as a decimal number, where 0 is Sunday and 6 is Saturday.
                0, 1, …, 6

        %d  Day of the month as a zero-padded decimal number.
                01, 02, …, 31

        %b  Month as locale’s abbreviated name.
                Jan, Feb, …, Dec (en_US)
                Jan, Feb, …, Dez (de_DE)

        %B  Month as locale’s full name.
                January, February, …, December (en_US);
                Januar, Februar, …, Dezember (de_DE)

        %m  Month as a zero-padded decimal number.
                01, 02, …, 12

        %y  Year without century as a zero-padded decimal number.
                00, 01, …, 99

        %Y  Year with century as a decimal number.
                1970, 1988, 2001, 2013

        %H  Hour (24-hour clock) as a zero-padded decimal number.
                00, 01, …, 23

        %I  Hour (12-hour clock) as a zero-padded decimal number.
                01, 02, …, 12

        %p  Locale's equivalent of either AM or PM.
                AM, PM (en_US)
                am, pm (de_DE)

        %M  Minute as a zero-padded decimal number.
                00, 01, …, 59

        %S  Second as a zero-padded decimal number.
                00, 01, …, 59

        %f  Microsecond as a decimal number, zero-padded on the left.
                000000, 000001, …, 999999

        %z  UTC offset in the form +HHMM or -HHMM (empty string if the the
            object is naive).
                (empty), +0000, -0400, +1030

        %Z  Time zone name (empty string if the object is naive).
                (empty), UTC, EST, CST

        %j  Day of the year as a zero-padded decimal number.
                001, 002, …, 366

        %U  Week number of the year (Sunday as the first day of the week) as a
            zero padded decimal number. All days in a new year preceding the
            first Sunday are considered to be in week 0.
                00, 01, …, 53

        %W  Week number of the year (Monday as the first day of the week) as a
            decimal number. All days in a new year preceding the first Monday
            are considered to be in week 0.
                00, 01, …, 53

        %c  Locale’s appropriate date and time representation.
                Tue Aug 16 21:30:00 1988 (en_US);
                Di 16 Aug 21:30:00 1988 (de_DE)

        %x  Locale’s appropriate date representation.
                08/16/88 (None)
                08/16/1988 (en_US)
                16.08.1988 (de_DE)

        %X  Locale’s appropriate time representation.
                21:30:00 (en_US)
                21:30:00 (de_DE)

        %%  A literal '%' character.
                %
        """
        if custom is None:
            custom = {}

        for index, article in enumerate(self.articles):

            # Make a copy of our subject
            subject = self.subject \
                if isinstance(self.subject, basestring) else ''

            # Make a copy of our poster
            poster = self.poster \
                if isinstance(self.poster, basestring) else ''

            if custom:
                # Apply our custom object if one is defined; first escape
                # content
                cmask = {re.escape(str(key)): str(value)
                         for (key, value) in custom.items()}

                # Build ourselves a translation map
                cmask_r = re.compile(
                    r'(' + '|'.join(cmask.keys()) + r')',
                    re.IGNORECASE,
                )

                # Apply our custom masks
                subject = cmask_r.sub(
                    lambda x: cmask[re.escape(x.group())], subject)
                poster = cmask_r.sub(
                    lambda x: cmask[re.escape(x.group())], poster)

            # Initialize ourselves a master translation table
            mask = {
                re.escape('{{count}}'): str(len(self)),
                re.escape('{{index}}'): str(index+1),
                re.escape('{{totalsize}}'): str(article.size()),
            }

            # Our first item can be referenced as
            #   {{filename001}} or {{filename}}
            #
            # Similarily the first time (only) can be reference as
            #   {{filesize001}} or {{filesize}}
            #
            try:
                mask[re.escape('{{filename}}')] = article[0].filename
                mask[re.escape('{{filesize}}')] = str(len(article[0]))

            except IndexError:
                mask[re.escape('{{filename}}')] = ''
                mask[re.escape('{{filesize}}')] = ''

            # for each file, create a filenameNo and filesizeNo
            for no, content in enumerate(article):
                try:
                    mask[re.escape('{{filename%.3d}}' % (no+1))] = \
                        article[0].filename
                    mask[re.escape('{{filesize%.3d}}' % (no+1))] = \
                        str(len(article[0]))

                except IndexError:
                    # No files found
                    mask[re.escape('{{filename%.3d}}' % (no+1))] = ''
                    mask[re.escape('{{filesize%.3d}}' % (no+1))] = ''

            # Build ourselves a translation map
            mask_r = re.compile(
                r'(' + '|'.join(mask.keys()) + r')',
                re.IGNORECASE,
            )

            # Apply our common masks
            subject = mask_r.sub(lambda x: mask[re.escape(x.group())], subject)
            poster = mask_r.sub(lambda x: mask[re.escape(x.group())], poster)

            # Final Tidy
            subject = re.sub('{{[^}]*}}', '', subject)
            poster = re.sub('{{[^}]*}}', '', poster)

            if strftime:
                # Time reference
                _relative = relative
                if _relative is None:
                    # A reference time
                    _relative = datetime.now()

                # apply time support
                subject = _relative.strftime(subject)
                poster = _relative.strftime(poster)

            # Store our subject
            article.subject = subject
            # Store our poster
            article.poster = poster

        return True

    def is_valid(self):
        """
        Iterates over article content an returns True if all of it is valid
        based on it's crc32/md5 and/or whatever mechanism is used to determine
        a NNTPContents validity
        """
        if len(self.articles) == 0:
            # No articles means no validity
            return False

        return next((False for c in self.articles
                     if c.is_valid() is False), True)

    def split(self, size=81920, mem_buf=1048576):
        """
        If there is one Article() and one (valid) NNTPContent() object within
        it, this object will split the Article() into several and break apart
        the contents of the file into several pieces.

        This function returns True if this action was successful, otherwise
        this function returns False.  Newly split content is 'not' in a
        detached form meaning content is removed if the object goes out of
        scope
        """
        if len(self.articles) != 1:
            # Not possible to split a post that is already split
            return False

        articles = self.articles[0].split(size=size, mem_buf=mem_buf)
        if articles is None:
            return False

        # Otherwise store our goods
        self.articles = articles
        return True

    def encode(self, encoders):
        """
        Iterates over all articles and encodes them based on the specified
        encoders.
        """

        # Prepare a new article set
        articles = sortedset(key=lambda x: x.key())
        for article in self.articles:
            result = article.encode(encoders)
            if not result:
                return False

            articles.add(result)

        # Store our new article set
        self.articles = articles

        return True

    def join(self):
        """
        If there are more then one Article() objects containing one valid()
        NNTPContent() object, they can be joined together into one single
        Article() object.

        This function returns True if this action was successful, otherwise
        this function returns False.  Newly joined content is 'not' in a
        detached form meaning content is removed if the object goes out of
        scope
        """

        if len(self.articles) == 1:
            # Nothing to do; no need to fail
            return True

        if len(self.articles) < 1:
            # Not possible to join less than 1 article
            return False

        # acquire the first entry; this will be what we build from
        articles = iter(self.articles)

        # Create a copy of our first object
        head_article = articles.next().copy()

        # Iterate over all our remaining article entries and stack their
        # content onto our head_article
        for content in articles:

            # Append our content
            if not head_article.append(content):

                # Clean up our copy
                del head_article

                # We failed
                return False

        # Reset with a new sorted set of articles
        self.articles.clear()

        # Add our single head_article entry as our primary entry
        self.articles.add(head_article)

        # We're done!
        return True

    def deobsfucate(self, filebase='', codecs=None):
        """
        Using the article information we have, attempt to generate the
        filename to the best of our ability
        """
        if not len(self.articles) > 0:
            # we need at least 1 article
            return None

        if filebase is None:
            # Safety
            filebase = ''

        if codecs is None:
            codecs = self._codecs

        elif isinstance(codecs, CodecBase):
            codecs = [codecs, ]

        # Create our Mime object
        m = Mime()

        # Initialize our objects
        _name = filebase
        _mime = m.from_filename(self.filename)
        _fext = None

        if _mime and _mime.type() != DEFAULT_MIME_TYPE:
            # Store our potential extension
            _fext = _mime.extension()

        if not _name and self.filename:
            # create a filebase
            _name = splitext(basename(self.filename))[0]

        # Because our articles could amount to a lot of crap, we can only hope
        # that one of them provides us with a valid file we can work with.
        # our map allows us to pick and choose what looks good amongst
        # everything else.

        # A sorted set of matches; index 0 is our index our element was found
        # we want this to prevail when match hunting
        ext_map = dict()

        for no, article in enumerate(self.articles):
            # Detect our article _fname
            _fname = article.deobsfucate(filebase=_name, codecs=codecs)
            # Detect our type
            mr = m.from_filename(_fname)
            if mr:
                # Store our tuple if we can
                ext_map[mr.type()] = (no, _fname, mr)

        # Now comes the hard part... if we're lucky we'll have just 2 types at
        # the most (one being DEFAULT_MIME_TYPE).  We can ignore these entries
        # unless they're all we have to pick from.  Otherwise hopefully we
        # have an option #2, that will be our official extension and filename

        # Initialize our findings
        match = None
        for _, potential in ext_map.iteritems():
            if potential[2].type() == DEFAULT_MIME_TYPE or _mime is None:
                if match is None:
                    # We allow this match if we have to
                    match = potential

            elif _mime and _mime.type() == potential[2].type():
                # Our detected mime type is exactly what we expected

                # Store our match
                match = potential

                if not _name and _mime and _mime.type() != DEFAULT_MIME_TYPE:
                    # Update our basename
                    _name = splitext(match[1])[0]

                # Safely break because we can't match any better than this
                break

            elif _mime and _mime.type() == DEFAULT_MIME_TYPE:
                # Update our candidate
                match = potential

        if not match:
            return None

        # use the extension gathered from our match and apply it to our base
        # if we can
        if not _fext:
            _fext = match[2].extension()
            # Return our matched filename
            return '%s%s' % (_name, _fext)

        # Return your file
        return match[1]

    def save(self, filepath=None, copy=False):
        """
        Saves all of the NNTPContent into the directory specified by
        the filepath; if no filepath is specified, then content is written
        into it's work_dir
        """

        for article in self.articles:
            if not article.save(filepath=filepath, copy=copy):
                return False

        return True

    def files(self):
        """
        Returns a list of the files within article
        """
        _files = []
        for article in self.articles:
            _files.extend(article.files())
        return _files

    def size(self):
        """
        return the total size of our articles
        """
        return sum(a.size() for a in self.articles)

    def strsize(self):
        """
        return the total size of our articles
        """
        return bytes_to_strsize(self.size())

    def key(self):
        """
        Returns a key that can be used for sorting with:
            lambda x : x.key()
        """
        return '%s' % self.filename

    def __iter__(self):
        """
        Grants usage of the next()
        """
        # Ensure our stream is open with read
        return iter(self.articles)

    def __len__(self):
        """
        Return the length of the articles
        """
        return len(self.articles)

    def __lt__(self, other):
        """
        Handles less than for storing in btrees
        """
        if self.sort_no is not None:
            # Compare based on a sort number, undefined sorting always
            # trumps those with sorting defined.
            if other.sort_no is not None:
                if self.sort_no == other.sort_no:
                    return str(self.filename) < str(other.filename)

                # Compare our sorting values
                return str(self.sort_no) < str(other.sort_no)

            # If no other sort_no then we are not less than it; those without
            # a sort_no should always trump those with one
            return False

        elif other.sort_no is not None:
            # We don't have a sort_no, and the other does; we're less than it
            # based on our set rules.
            return True

        # If we reach here, neither comparison objects have a sort_no defined.
        # as a result, we just base our match on the filename specified
        return str(self.filename) < str(other.filename)

    def __eq__(self, other):
        """
        Handles equality

        """
        if self.sort_no:
            # Compare based on a sort number, undefined sorting always
            # trumps those with sorting defined.
            if other.sort_no:
                if self.sort_no == other.sort_no:
                    return self.__dict__ == other.__dict__

            # Any other posibility is False
            return False

        elif other.sort_no:
            # they have a sort and we don't. there is no equality in that
            return False

        return self.__dict__ == other.__dict__

    def __getitem__(self, index):
        """
        Support accessing NNTPArticle objects by index
        """
        return self.articles[index]

    def __str__(self):
        """
        Return a printable version of the article
        """
        return '%s' % self.filename

    def __unicode__(self):
        """
        Return a printable version of the article
        """
        return u'%s' % self.filename

    def __repr__(self):
        """
        Return an unambigious version of the object
        """
        if not self.sort_no:
            return '<NNTPSegmentedPost filename="%s" articles=%d />' % (
                self.filename,
                len(self.articles),
            )

        return '<NNTPSegmentedPost sort="%d" filename="%s" articles=%d />' % (
            self.sort_no,
            self.filename,
            len(self.articles),
        ) class NNTPMetaContent(NNTPContent):
    """
    An NNTP Meta representation
    """
    def __init__(self, filepath=None, part=0, work_dir=None, sort_no=2500,
                 *args, **kwargs):

        super(NNTPMetaContent, self).__init__(
            filepath=filepath,
            part=part,
            work_dir=work_dir, sort_no=sort_no, *args, **kwargs)

        self.content = list()
        self._iter = None

    def key(self):
        """
        Returns a key that can be used for sorting with:
            lambda x : x.key()
        """
        return '%.5d/MetaContent/%d' % (self.sort_no, id(self))

    def next(self):
        """
        Python 2 support
        Support stream type functions and iterations
        """
        if not self._iter:
            self._iter = iter(self.content)

        return next(self._iter)

    def __next__(self):
        """
        Python 3 support
        Support iterating through list
        """
        if not self._iter:
            self._iter = iter(self.content)

        return next(self._iter)

    def __iter__(self):
        """
        Mimic iter()
        """
        return iter(self.content)

    def __delitem__(self, key):
        """
        Mimic del action
        """
        del self.content[key]

    def __contains__(self, item):
        """
        Mimic 'in' directive
        """
        return item in self.content

    def __len__(self):
        """
        Support len() call
        """
        return len(self.content)

    def __repr__(self):
        """
        Keep it only possible to track 1 header
        """
        return '<NNTPMetaContent sort=%d content="%s" />' % (
            self.sort_no, repr(self.content),
        ) class NNTPFilterBase(object):
    """
    A class that can be applied to an NNTPConnection to additionaly apply a
    pre-determined number of filters to the content as it's fetched
    """

    _scorelist = {}

    def __init__(self, paths=None):
        """
        Initialize Filter
        """
        # the regex_map contains the pre-compied lookups of all the
        # compiled regular expressions
        self._regex_map = {}

        # The hash contains all of the blacklist, whitelist and scorelist
        # precompiled; it's populated when a group is first accessed
        self._regex_hash = {}

        self._blacklist = {}
        self._whitelist = {}

        # Scoring is done before applying filters;  It's a way of marking your
        # favorite poster, keywords, and bumping up a score on them.

        # By default everything consists of a score of zero (0) unless you
        # otherwise want to adjust it. You can then later use the filters to
        # adjust the score.
        self._scorelist = {}

        # Tracks a list of loaded files
        self.loaded_files = []

        if paths:
            # Load our content if path(s) were specified
            self.load(paths)


    def load(self, paths, reset=True):
        """
        Input can be either a directory or file to scan.  If a directory is
        passed in, it is scanned for .nrf files (newsreap filter) files and
        loads their content.

        If a list is passed in, then the contents of the list are expected
        to be either directories and/or files. to which all are loaded.

        If the paths is None or contains nothing, then nothing is loaded.

        If reset is set to true, then existing content is erased in
        preparation for the new content.
        """

        if reset:
            self._regex_map = {}
            self._regex_hash = {}
            self._blacklist = {}
            self._whitelist = {}
            self._scorelist = {}
            self.loaded_files = []

        if not paths:
            # Nothing more to do
            return

        if isinstance(paths, basestring):
            # Convert to list
            paths = [paths, ]

        for path in paths:
            _paths = []
            if isdir(path):
                # Search for newsreap filters in the path specified
                for entry in listdir(path):
                    if isdir(join(path, entry)):
                        continue

                    if not FILTER_FILE_RE.match(entry):
                        continue
                    _paths.append(join(path, entry))
                # Load our content
                self.load(paths=_paths, reset=False)

            elif isfile(path):
                # Parse the file
                lineno = 0
                logger.info('Loading filters from %s' % (path))
                with open(path) as f:
                    for line in f:
                        # Increment line no
                        lineno += 1

                        if not len(line.rstrip()):
                            # Blank line; ignore these
                            continue

                        # Parseable?
                        result = FILTER_LINE_RE.match(line)
                        if not result:
                            logger.warning(
                                "%s[:%d] not parseable." % (
                                    basename(path), lineno,
                            ))
                            continue

                        if result.group('comment') is not None:
                            # Ignore commented lines
                            continue

                        try:
                            code = result.group('code').upper()

                        except AttributeError:
                            # None Type; Assign Default
                            code = '%s%s' % (
                                FilterListCode.SCORELIST,
                                FilterMatchCode.BY_SUBJECT,
                            )

                        if code[0] not in FILTER_LIST_CODES:
                            logger.warning(
                                "%s[:%d] not has an invalid code (%s)." % (
                                    basename(path), lineno, str(code),
                            ))
                            continue

                        # Store List Code
                        l_code = code[0]

                        op = result.group('op')
                        if op is None or op == '=':
                            op = FilterDirectives.SCORE

                        elif op == '>':
                            op = FilterDirectives.SCORE_GREATER_THAN

                        elif op == '<':
                            op = FilterDirectives.SCORE_LESS_THAN

                        else:
                            logger.warning(
                                "%s[:%d] contains an invalid operation code (%s) (expecting < > or =)." % (
                                    basename(path), lineno, str(op),
                            ))
                            continue
                        if l_code == FilterListCode.SCORELIST and \
                           op != FilterDirectives.SCORE:
                            logger.warning(
                                "%s[:%d] contains an invalid operation code (%s) (not expecting one)." % (
                                    basename(path), lineno, str(op),
                            ))
                            continue

                        score = result.group('score')
                        if score is not None:
                            sign = result.group('sign')
                            if not sign:
                                sign = '+'

                            elif sign not in ('-', '+'):
                                logger.warning(
                                    "%s[:%d] not has an invalid sign (%s) (expecting +/-)." % (
                                        basename(path), lineno, str(sign),
                                ))
                                continue

                            # Apply sign and convert score to integer
                            if sign == '-':
                                score = -int(score)
                            else:
                                score = int(score)

                        elif l_code == FilterListCode.SCORELIST:
                            # A score is manditory
                            logger.warning(
                                "%s[:%d] (scorelist) not has no score associated with it." % (
                                    basename(path), lineno,
                            ))
                            continue

                        try:
                            group = result.group('group').lower()

                        except AttributeError:
                            # None Type; Assign Default
                            group = '.*'

                        _filter = {
                            FilterDirectives.DESCRIPTION: \
                                'file="%s", line="%d"' % (path, lineno),
                        }

                        if score is not None:
                            _filter[op] = score

                        regex = result.group('regex')
                        if regex is not None:
                            if code[1] == 'S':
                                _filter[FilterDirectives.SUBJECT_MATCHES_REGEX] = regex

                            elif code[1] == 'P':
                                _filter[FilterDirectives.POSTER_MATCHES_REGEX] = regex

                        try:
                            if l_code == FilterListCode.BLACKLIST:
                                self.blacklist_append(group, _filter)

                            elif l_code == FilterListCode.WHITELIST:
                                self.whitelist_append(group, _filter)

                            elif l_code == FilterListCode.SCORELIST:
                                self.scorelist_append(group, _filter)

                        except ValueError, e:
                            # Internal Parse Error Occured
                            logger.warning(
                                "%s[:%d] %s." % (
                                    basename(path), lineno, str(e),
                            ))
                            continue

            else:
                logger.warning(
                    'Unsupported type <%s> passed as filter' % type(path),
                )



    def scorelist_append(self, group_regex, entry):
        """
        Add a scoring entry to list
        """
        if not isinstance(group_regex, basestring):
            raise ValueError(
                "Invalid group regex specified; (expecting a string)",
            )

        # Prepare filter count
        entry[FilterDirectives.FILTERS] = \
            len([l for l in entry if l[0] == '_'])

        if entry[FilterDirectives.FILTERS] <= 0:
            raise ValueError(
                "There was no filters specified.",
            )

        # Compile Subject and/or Poster if necessary
        if FilterDirectives.SUBJECT_MATCHES_REGEX in entry:
            if isinstance(entry[FilterDirectives.SUBJECT_MATCHES_REGEX],
                    basestring):
                try:
                    entry[FilterDirectives.SUBJECT_MATCHES_REGEX] = \
                        re.compile(
                            entry[FilterDirectives.SUBJECT_MATCHES_REGEX],
                            re.IGNORECASE,
                        )
                except:
                    raise ValueError(
                        "Invalid subject regex '%s'" % \
                        entry[FilterDirectives.SUBJECT_MATCHES_REGEX],
                    )

        if FilterDirectives.POSTER_MATCHES_REGEX in entry:
            if isinstance(entry[FilterDirectives.POSTER_MATCHES_REGEX],
                    basestring):
                try:
                    entry[FilterDirectives.POSTER_MATCHES_REGEX] = \
                        re.compile(
                            entry[FilterDirectives.POSTER_MATCHES_REGEX],
                            re.IGNORECASE,
                        )
                except:
                    raise ValueError(
                        "Invalid poster regex '%s'" % \
                        entry[FilterDirectives.POSTER_MATCHES_REGEX],
                    )

        if FilterDirectives.SCORE not in entry:
            raise ValueError(
                "There was no score specified.",
            )

        if group_regex not in self._regex_map:
            try:
                # pre-compile regular expression for both hashing
                # and a way of preforming some early validation
                self._regex_map[group_regex] = re.compile(group_regex)

            except:
                raise ValueError(
                    "Invalid group regex '%s'" % group_regex,
                )

        if group_regex not in self._scorelist:
            self._scorelist[group_regex] = list()
        self._scorelist[group_regex].append(entry)

        if len(self._regex_hash):
            # Force a rehash next time we apply our list
            self._regex_hash = {}


    def blacklist_append(self, group_regex, entry):
        """
        Add a blacklist entry to list
        """
        if not isinstance(group_regex, basestring):
            raise ValueError(
                "Invalid group regex specified; (expecting a string)",
            )

        # Prepare filter count
        entry[FilterDirectives.FILTERS] = \
            len([l for l in entry if l[0] == '_'])

        if entry[FilterDirectives.FILTERS] <= 0:
            raise ValueError(
                "There was no filters specified.",
            )

        # Compile Subject and/or Poster if necessary
        if FilterDirectives.SUBJECT_MATCHES_REGEX in entry:
            if isinstance(entry[FilterDirectives.SUBJECT_MATCHES_REGEX],
                    basestring):
                try:
                    entry[FilterDirectives.SUBJECT_MATCHES_REGEX] = \
                        re.compile(
                            entry[FilterDirectives.SUBJECT_MATCHES_REGEX],
                            re.IGNORECASE,
                        )
                except:
                    raise ValueError(
                        "Invalid subject regex '%s'" % \
                        entry[FilterDirectives.SUBJECT_MATCHES_REGEX],
                    )

        if FilterDirectives.POSTER_MATCHES_REGEX in entry:
            if isinstance(entry[FilterDirectives.POSTER_MATCHES_REGEX],
                    basestring):
                try:
                    entry[FilterDirectives.POSTER_MATCHES_REGEX] = \
                        re.compile(
                            entry[FilterDirectives.POSTER_MATCHES_REGEX],
                            re.IGNORECASE,
                        )
                except:
                    raise ValueError(
                        "Invalid poster regex '%s'" % \
                        entry[FilterDirectives.POSTER_MATCHES_REGEX],
                    )

        if group_regex not in self._regex_map:
            try:
                # pre-compile regular expression for both hashing
                # and a way of preforming some early validation
                self._regex_map[group_regex] = re.compile(group_regex)

            except:
                raise ValueError(
                    "Invalid group regex '%s'" % group_regex,
                )

        if group_regex not in self._blacklist:
            self._blacklist[group_regex] = list()
        self._blacklist[group_regex].append(entry)

        if len(self._regex_hash):
            # Force a rehash next time we apply our list
            self._regex_hash = {}


    def whitelist_append(self, group_regex, entry):
        """
        Add a whitelist entry to list
        """
        if not isinstance(group_regex, basestring):
            raise ValueError(
                "Invalid group regex specified; (expecting a string)",
            )

        # Prepare filter count
        entry[FilterDirectives.FILTERS] = \
            len([l for l in entry if l[0] == '_'])

        if entry[FilterDirectives.FILTERS] <= 0:
            raise ValueError(
                "There was no filters specified.",
            )

        # Compile Subject and/or Poster if necessary
        if FilterDirectives.SUBJECT_MATCHES_REGEX in entry:
            if isinstance(entry[FilterDirectives.SUBJECT_MATCHES_REGEX],
                    basestring):
                try:
                    entry[FilterDirectives.SUBJECT_MATCHES_REGEX] = \
                        re.compile(
                            entry[FilterDirectives.SUBJECT_MATCHES_REGEX],
                            re.IGNORECASE,
                        )
                except:
                    raise ValueError(
                        "Invalid subject regex '%s'" % \
                        entry[FilterDirectives.SUBJECT_MATCHES_REGEX],
                    )

        if FilterDirectives.POSTER_MATCHES_REGEX in entry:
            if isinstance(entry[FilterDirectives.POSTER_MATCHES_REGEX],
                    basestring):
                try:
                    entry[FilterDirectives.POSTER_MATCHES_REGEX] = \
                        re.compile(
                            entry[FilterDirectives.POSTER_MATCHES_REGEX],
                            re.IGNORECASE,
                        )
                except:
                    raise ValueError(
                        "Invalid poster regex '%s'" % \
                        entry[FilterDirectives.POSTER_MATCHES_REGEX],
                    )

        if group_regex not in self._regex_map:
            try:
                # pre-compile regular expression for both hashing
                # and a way of preforming some early validation
                self._regex_map[group_regex] = re.compile(group_regex)

            except:
                raise ValueError(
                    "Invalid group regex '%s'" % group_regex,
                )

        if group_regex not in self._whitelist:
            self._whitelist[group_regex] = list()
        self._whitelist[group_regex].append(entry)

        if len(self._regex_hash):
            # Force a rehash next time we apply our list
            self._regex_hash = {}


    def lazy_re_fetch(self, group):
        """
        This function takes a group and builds it's matching
        hash entries based on filters, whitelist, etc
        """
        if group not in self._regex_hash:
            # Store the group in a hash
            self._regex_hash[group] = {
                FilterListCode.BLACKLIST: [],
                FilterListCode.WHITELIST: [],
                FilterListCode.SCORELIST: [],
            }

            # Build Groups Black List
            for _k, _v in self._blacklist.iteritems():
                k = self._regex_map.get(_k, re.compile(_k))
                if k.match(group):
                    # Keys are prefixed with (underscore); a count makes it
                    # easier later to apply multiple filters and not trying
                    # to pre-determine ahead of time if they've all matched
                    for v in _v:
                        if FilterDirectives.FILTERS not in v:
                            v[FilterDirectives.FILTERS] = \
                                    len([l for l in v if l[0] == '_'])

                        self._regex_hash[group][FilterListCode.BLACKLIST].append(v)

            # Build Groups White List
            for _k, _v in self._whitelist.iteritems():
                k = self._regex_map.get(_k, re.compile(_k))
                if k.match(group):
                    # Keys are prefixed with (underscore); a count makes it
                    # easier later to apply multiple filters and not trying
                    # to pre-determine ahead of time if they've all matched
                    for v in _v:
                        if FilterDirectives.FILTERS not in v:
                            v[FilterDirectives.FILTERS] = \
                                    len([l for l in v if l[0] == '_'])

                        self._regex_hash[group][FilterListCode.WHITELIST].append(v)

            # Build Groups Score List
            for _k, _v in self._scorelist.iteritems():
                k = self._regex_map.get(_k, re.compile(_k))
                if k.match(group):
                    # Keys are prefixed with (underscore); a count makes it
                    # easier later to apply multiple filters and not trying
                    # to pre-determine ahead of time if they've all matched
                    for v in _v:
                        if FilterDirectives.FILTERS not in v:
                            v[FilterDirectives.FILTERS] = \
                                    len([l for l in v if l[0] == '_'])

                        self._regex_hash[group][FilterListCode.SCORELIST].append(v)

        return self._regex_hash[group]


    def _match(self, entry, poster, date, subject, size, lines, xgroups,
               *args, **kwargs):
        """
        The generic function called to scan whitelists, blacklists,
        and scorelists which is defined by 'typelist',
        """

        # Acquire a match count
        matches = entry.get(FilterDirectives.FILTERS, 0)
        if not matches:
            # Probably spit a warning out here (TODO)
            return False

        # Acquire Score
        score = kwargs.get(FilterDirectives.SCORE, 0)
        if FilterDirectives.SCORE_GREATER_THAN in entry:
            if score > entry[FilterDirectives.SCORE_GREATER_THAN]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.SCORE_LESS_THAN in entry:
            if score < entry[FilterDirectives.SCORE_LESS_THAN]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.SUBJECT_MATCHES_REGEX in entry:
            if entry[FilterDirectives.SUBJECT_MATCHES_REGEX].match(subject):
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.SUBJECT_STARTS_WITH in entry:
            if entry[FilterDirectives.SUBJECT_STARTS_WITH] == \
               subject[0:len(entry[FilterDirectives.SUBJECT_STARTS_WITH])]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.SUBJECT_ENDS_WITH in entry:
            if entry[FilterDirectives.SUBJECT_ENDS_WITH] == \
               subject[-len(entry[FilterDirectives.SUBJECT_ENDS_WITH]):]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.POSTER_MATCHES_REGEX in entry:
            if entry[FilterDirectives.POSTER_MATCHES_REGEX].match(poster):
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.POSTER_STARTS_WITH in entry:
            if entry[FilterDirectives.POSTER_STARTS_WITH] == \
               poster[0:len(entry[FilterDirectives.POSTER_STARTS_WITH])]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.POSTER_ENDS_WITH in entry:
            if entry[FilterDirectives.POSTER_ENDS_WITH] == \
               poster[-len(entry[FilterDirectives.POSTER_ENDS_WITH]):]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.LINE_COUNT_GREATER_THAN in entry:
            if lines > entry[FilterDirectives.LINE_COUNT_GREATER_THAN]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.LINE_COUNT_LESS_THAN in entry:
            if lines < entry[FilterDirectives.LINE_COUNT_LESS_THAN]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.SIZE_GREATER_THAN in entry:
            if size > entry[FilterDirectives.SIZE_GREATER_THAN]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.SIZE_LESS_THAN in entry:
            if size < entry[FilterDirectives.SIZE_LESS_THAN]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.DATE_GREATER_THAN in entry:
            if date > entry[FilterDirectives.DATE_GREATER_THAN]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.DATE_LESS_THAN in entry:
            if date < entry[FilterDirectives.DATE_LESS_THAN]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.XPOST_COUNT_GREATER_THAN in entry:
            if len(xgroups) > entry[FilterDirectives.XPOST_COUNT_GREATER_THAN]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        if FilterDirectives.XPOST_COUNT_LESS_THAN in entry:
            if len(xgroups) < entry[FilterDirectives.XPOST_COUNT_LESS_THAN]:
                matches -= 1
                if matches <= 0:
                    return True
            else:
                # No match
                return False

        return matches <= 0


    def whitelist(self, group, poster, date, subject, size, lines, xgroups,
                  *args, **kwargs):
        """
        Returns true if entry matches a filter identifed on the whitelist
        """

        whitelist = self.lazy_re_fetch(group)[FilterListCode.WHITELIST]
        if not whitelist:
            # No whitelist entries to apply
            return False

        for entry in iter(whitelist):
            if self._match(entry, poster, date, subject, size, lines, xgroups,
                           *args, **kwargs):
                # We can stop on our first match
                return True

        # no match
        return False


    def blacklist(self, group, poster, date, subject, size, lines, xgroups,
                  *args, **kwargs):
        """
        Returns true if entry matches a filter identifed on the blacklist
        """

        blacklist = self.lazy_re_fetch(group)[FilterListCode.BLACKLIST]
        if not blacklist:
            # No blacklist entries to apply
            return False

        for entry in iter(blacklist):
            if self._match(entry, poster, date, subject, size, lines, xgroups,
                           *args, **kwargs):
                # We can stop on our first match
                return True

        # no match
        return False


    def score(self, group, poster, date, subject, size, lines, xgroups,
              *args, **kwargs):
        """
        Scores content based on what is passed in; zero (0) is always returned
        since that is the default score to assign to anything if nothing
        was matched otherwise the calculated score is returned.
        """

        scorelist = self.lazy_re_fetch(group)[FilterListCode.SCORELIST]
        if not scorelist:
            # No scores applied
            return 0

        # Our score counter
        score = 0

        for entry in iter(scorelist):

            new_score = entry.get(FilterDirectives.SCORE)
            if not new_score:
                # No score defined
                continue

            if self._match(entry, poster, date, subject, size, lines, xgroups,
                           *args, **kwargs):

                # (no stopping); apend score and keep going
                score += new_score

        # not all matches
        return score class NNTPIOStream(object):
    """
    Defines the supported NNTP Input/Output Streams

    """
    # RFC-3977 Standard NNTP Stream - No compression at all
    RFC3977 = 'rfc3977'

    # GZip RFC-3977 (COMPRESS Keyword used)
    RFC3977_GZIP = 'gzip.rfc3977' NNTP_SUPPORTED_IO_STREAMS = (
    # The sequence here is important since it also plays a roll
    # in the enumeration stored in referenced databases potentially
    # used outside of this class. Always add new types to the end.
    NNTPIOStream.RFC3977,
    NNTPIOStream.RFC3977_GZIP,
) NNTP_DEFAULT_ENCODING = 'ISO-8859-1' class SocketBase(object):
    """
       Abstract class with all the functionalist needed for a socket manager.
       methods raising an exception must be implemented in derived classes.

       Arguments to initialize a SocketBase:

            host            the host to connect/listen on

            port            int (default=0)

                            - Port to bind (slave)
                            - 0 means an ephemeral port is chosen
                              (random > 32768)

            timeout         int (default=None)

                            - connection timeout

            secure          Use encryption when managing the connection
                            Expects a True or False, but you can also
                            Specify the encryption Cypher to use here
                            too.

            verify_cert     Attempt to verify the SSL certificate or fail
                            if we can't.  This is used to help detect against
                            the man-in-the-middle attacks.

    """
    def __init__(self, host=None, port=0, bindaddr=None, bindport=0,
                 secure=False, verify_cert=True, hooks=None, *args, **kwargs):

        try:
            self.port = int(port)
        except:
            self.port = 0

        self.host = host
        self.bindaddr = bindaddr
        self.bindport = bindport

        self.connected = False
        self.secure = secure
        self.verify_cert = verify_cert

        if self.secure is None:
            # a little qwirky, but allow users to set secure to
            # None and have it treated the same way as False
            self.secure = False

        self.socket = None

        # Track the current index of the secure protocol singleton to use
        self.secure_protocol_idx = 0

        if self.secure not in (True, False):
            # If self.secure identifies an actual protocol, we want to use it
            # The below simply looks for it's existance, and if it isn't
            # present then we return None
            self.secure_protocol_idx = next(
                (i for i, v in enumerate(SECURE_PROTOCOL_PRIORITY)
                    if v[0] == self.secure), None)

            if self.secure_protocol_idx is None:
                # Protocol specified was not found and/or supported; alert the
                # user with a loud bang; we're done here.
                raise AttributeError("Invalid secure protocol specified.")

        # A spot we can store our peer certificate; this is only used
        # if we're dealing with a secure connection
        self.peer_certificate = None

        # CA stands for Certificate Authority (for those reading this code)
        # This is the master list of servers that you trust for verifying
        # your certificates against.  If you're using a self-signed key
        # then this is useless to you (and you shouldn't verify)
        # You'll need these if you want to verify your host
        self._ca_certs = kwargs.get('ca_certs', "/etc/ssl/certs/ca-bundle.crt")

        # These keys are needed for hosting / listen type connections only
        self._keyfile = kwargs.get('keyfile', None)
        self._certfile = kwargs.get('certfile', None)

        # For Statistics
        self.stat_connect_time = None

        # Calculated through connections
        self._local_addr = None
        self._local_port = None
        self._remote_addr = None
        self._remote_port = None

        # Define our hooks (if any)
        self.hooks = HookManager()
        if hooks:
            self.hooks.add(hooks=hooks)

    def hooks(self, hooks, reset=True):
        """
        Sets hooks into our connection object(s)

        """
        if reset:
            # Reset our loaded hooks
            self.hooks.reset()

        if isinstance(hooks, HookManager):
            # update our hooks
            for key in hooks.iterkeys():
                self.hooks.add(hooks[key])

        else:
            self.hooks.add(hooks)

        return

    def can_read(self, timeout=0.0):
        """
        Checks if there is data that can be read from the
        socket (if open). Returns True if there is data and
        False if not.

        It returns None if something very bad happens such as
        a dead connection (bad file descriptor), etc
        """

        # rs = Read Sockets
        # ws = Write Sockets
        # es = Error Sockets
        if self.socket is not None:
            try:
                rs, _, es = select([self.socket], [], [], timeout)
            except (SelectError, socket.error), e:
                if e[0] == errno.EBADF:
                    # Bad File Descriptor... hmm
                    self.close()
                    return None

            if len(es) > 0:
                # Bad File Descriptor
                self.close()
                return None

            return len(rs) > 0

        # no socket or no connection
        return None

    def can_write(self, timeout=0):
        """
        Checks if there is data that can be written to the
        socket (if open). Returns True if writing is possible and
        False if not.

        It returns None if something very bad happens such as
        a dead connection (bad file descriptor), etc
        """

        # rs = Read Sockets
        # ws = Write Sockets
        # es = Error Sockets
        if self.socket is not None:
            try:
                _, ws, es = select([], [self.socket], [], timeout)
            except (SelectError, socket.error), e:
                if e[0] == errno.EBADF:
                    # Bad File Descriptor... hmm
                    self.close()
                    return None

            if len(es) > 0:
                # Bad File Descriptor
                self.close()
                return None

            return len(ws) > 0

        # no socket or no connection
        return None

    def close(self):
        """ Socket Wrapper for people using this class as if it were just
            a python socket class
            """
        data = ''

        # Close socket and exit gracefully as retry count was met
        if self.socket is not None:
            # Copy rest of input buffer
            try:
                data = self.socket.recv()
                if data is None:
                    data = ''

            except Exception:
                pass

            try:
                self.socket.shutdown()
            except:
                # Close connection
                try:
                    self.socket.close()
                except:
                    pass

            # Call a close hook
            self.hooks.call(
                'socket_close',
                host=self._remote_addr,
                port=self._remote_port,
                socket=weakref.proxy(self),
            )

            # Remove Socket Reference
            self.socket = None

        # update connection flag
        self.connected = False

        # reset our peer certificate
        self.peer_certificate = None

        # Reset Connect Time Stat
        self.stat_connect_time = None

        # reset remote connection details only
        # we keep the local ones so we can re-use them
        # if possible (especially the port)

        # Calculated through connections
        self._remote_addr = None
        self._remote_port = None

        # return any lingering data in buffer
        if data:
            return data
        return ''

    def bind(self, timeout=None, retries=3, retry_wait=10.0):
        """
          Perform socket binding if nessisary but otherwise this
          performs the connection itself
        """

        # Ensure we are not connected
        self.close()

        # Set a flag we can use
        established = False

        while not established:
            # Create a new socket
            self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

            # Set Reuse Address flag
            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

            # We don't need to bind connections that are not configured with
            # bind information.
            if not (self.bindaddr or self.bindport):
                # Store local connection details
                return True

            if not self.bindaddr:
                self.bindaddr = '0.0.0.0'
            if not self.bindport:
                self.bindport = 0

            try:
                bind_str = '%s:%s' % (self.bindaddr, self.bindport)
                self.socket.bind((self.bindaddr, self.bindport))
                logger.debug("Socket bound to %s" % bind_str)
                # Store local connection details
                return True

            except socket.error, e:
                logger.debug("Socket binding error: %s" % str(e))

                if e[0] == errno.EINTR:
                    # A Signal was caught, we aren't far enough in the
                    # listening process to continue; break so that we
                    # can handle this
                    raise SignalCaughtException('Signal received')
                # elif e[0] == errno.EADDRINUSE:

            # Close socket
            self.close()

            # Update Counter
            if retries is not None:
                retries = retries - 1

                if retries <= 0:
                    # Fatal situation
                    logger.error("Failed to bind to %s", bind_str)
                    raise SocketException(
                        "Failed to bind to %s", bind_str,
                    )
            else:
                # No retries, then just return
                return False

            # no need to thrash
            sleep(retry_wait)

        # Code can't ever reach here; but to satisfy lint
        # we'll put a return statement here
        return False

    def connect(self, timeout=None, retry_wait=1.00):
        """
           input Parameters:
           -timeout     How long connect() should block for before giving up
                        and moving on to one of the retires

           -retry_wait  How many seconds to wait before trying again on a
                        timeout (only applicable if retries is specified)

           output Parameters:
           -True if connection established, False otherwise

           Description:
           establish connection according to object attributes.
           return established connection as self.socket.

        """

        # Blocking until a connection
        logger.debug("Connecting to host: %s:%d" % (
            self.host,
            self.port,
        ))

        if timeout:
            socket.setdefaulttimeout(timeout)
            logger.debug("Socket timeout set to :%ds" % (timeout))

        # Get reference time
        cur_time = datetime.now()

        while True:
            if not self.bind(timeout=timeout, retries=3):
                return False

            # Keep alive flag
            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)

            try:
                # Simple protocol that returns if good, otherwise
                # it throws an exception,  So the next line is always
                # to break out of the loop (had we not already thrown
                # an exception
                self.socket.connect((
                    socket.gethostbyname(self.host),
                    int(self.port),
                ))

                # Disable Blocking
                self.socket.setblocking(False)

                # Store local details of our socket
                (self._local_addr, self._local_port) = \
                    self.socket.getsockname()
                (self._remote_addr, self._remote_port) = \
                    self.socket.getpeername()

                logger.info(
                    "Connection established to %s:%d" % (
                        self._remote_addr,
                        self._remote_port,
                    )
                )

                if self.secure is False:
                    # A non secure connection; we're done
                    break

                # Encrypt our socket (changing it into an SSLSocket Object)
                self.__encrypt_socket(
                    timeout=timeout,
                    verify=self.verify_cert,
                    server_side=False,
                )

                # If we get here, we were successful in encrypting the
                # connection; so let's go ahead and break out of our
                # connectino loop
                break

            except ssl.SSLError, e:
                # Secure Connection Failed
                self.close()
                logger.debug(
                    "Failed to secure connection using %s / errno=%d" % (
                        SECURE_PROTOCOL_PRIORITY[self.secure_protocol_idx][1],
                        e[0],
                    ),
                )

                if self.secure is True:
                    # Fetch next (but only if nothing was explicitly
                    # specified)
                    self.__ssl_version(try_next=True)
                    continue

                # If we reach here, we had a problem with our secure connection
                # handshaking and we were explicitly told to only use 1 (one)
                # protocol.  Thus there is nothing more to retry.  So throwing
                # a SocketException() is not a good idea.  Instead, we throw
                # a SocketRetryLimit() so it can be handled differently
                # upstream
                raise SocketRetryLimit('There are no protocols left to try.')

            except socket.error, e:
                logger.debug("Socket exception received: %s" % (e))
                if e[0] == errno.EINTR:
                    # Ensure socket is closed
                    self.close()
                    # A Signal was caught,
                    # return so we can handle this
                    raise

            except:
                # Close socket
                self.close()
                # Raise issue
                raise

            # Close socket
            self.close()

            if timeout:
                # Compare reference time with time now and sleep for the
                # difference defined by timeout
                delta_time = datetime.now() - cur_time
                delta_time = (delta_time.days * 86400) + delta_time.seconds \
                    + (delta_time.microseconds / 1e6)

                if delta_time >= timeout:
                    # Connection timeout elapsed
                    self.close()
                    raise SocketException('Connection timeout')

            # Throttle retry
            sleep(retry_wait)

        # Connection Established
        self.stat_connect_time = datetime.now()
        self.connected = True

        # Call a connect hook
        self.hooks.call(
            'socket_connect',
            host=self._remote_addr,
            port=self._remote_port,
            socket=weakref.proxy(self),
        )

        return True

    def listen(self, timeout=None, retry_wait=1.00, reuse_port=True):
        """
           input Parameters:
           -timeout     How long accept() should block for before giving up and
                        moving on to one of the retires

           -retry_wait  How many seconds to wait before trying again on a
                        timeout (only applicable if retries is specified)

           output Parameters:
           -True if connection established, False otherwise

           Description:
           establish connection according to object attributes.
           return established connection as self.socket.

        """

        if reuse_port and self._local_port is not None and self.port == 0:
            # Re-use the last port we acquired that way we can close a
            # connection gracefully and not have to re-acquire a new
            # ephemeral port
            self.port = self._local_port

        # Blocking until a connection
        logger.debug("Listening for a connection at: %s:%d" % (
            self.bindaddr,
            self.port,
        ))
        if timeout:
            socket.setdefaulttimeout(timeout)
            logger.debug("Socket timeout set to :%ds" % (timeout))

        # Bind to address
        if not self.bind(timeout=timeout, retries=3):
            return False

        # Never use blocking
        self.socket.setblocking(False)

        # Listen Enabled, the 1 identifies the number of connections we
        # will accept; never handle more then 1 at a time.
        self.socket.listen(1)

        # Store local details of our socket
        (self._local_addr, self._local_port) = self.socket.getsockname()

        # Get reference time
        cur_time = datetime.now()
        while True:
            try:
                conn, (self._remote_addr, self._remote_port) = \
                    self.socket.accept()
                # If we get here, we've got a connection
                break

            except TypeError, e:
                # Timeout occurred
                pass

            except AttributeError, e:
                # Usually means someone called close() while accept() was
                # blocked. Happens when using this class with threads.
                # No problem... we'll just finish up here
                self.close()
                raise SocketException('Connection broken abruptly')

            except socket.error, e:
                if e[0] == errno.EAGAIN:
                    # Timeout occurred
                    pass

                elif e[0] == errno.EINTR:
                    # A Signal was caught,
                    # return so we can handle this
                    raise
                else:
                    # Raise all other exceptions
                    raise
            except:
                # Close socket
                self.close()
                # Raise issue
                raise

            if timeout:
                # Compare reference time with time now and sleep for the
                # difference defined by timeout
                delta_time = datetime.now() - cur_time
                delta_time = (delta_time.days * 86400) + delta_time.seconds \
                    + (delta_time.microseconds / 1e6)

                if delta_time >= timeout:
                    # Connection timeout elapsed
                    self.close()
                    raise SocketException('Connection timeout')

            # Throttle until data is available
            if self.can_read(retry_wait) is None:
                # Something very bad happened
                self.close()
                raise SocketException('Connection broken')

        # Close listening connection
        self.socket.close()

        # Swap socket with new
        self.socket = conn

        # Update our local information
        (self._local_addr, self._local_port) = self.socket.getsockname()
        (self._remote_addr, self._remote_port) = self.socket.getpeername()

        logger.info(
            "Connection established to %s:%d" % (
                self._remote_addr,
                self._remote_port,
            )
        )

        if self.secure is False:
            # A non secure connection; we're done

            # Toggle our connection flag
            self.connected = True

            # Return our success
            return True

        try:
            # Encrypt our socket (changing it into an SSLSocket Object)
            self.__encrypt_socket(
                timeout=timeout,
                verify=self.verify_cert,
                server_side=True,
            )

            # If we get here, we were successful in encrypting the connection;
            # so let's go ahead and break out of our connection loop

        except ssl.SSLError, e:
            # Secure Connection Failed
            logger.debug(
                "Failed to secure connection using %s / errno=%d" % (
                    SECURE_PROTOCOL_PRIORITY[self.secure_protocol_idx][1],
                    e[0],
                ),
            )
            self.close()

            if self.secure is True:
                # Fetch next (but only if nothing was explicitly
                # specified)
                self.__ssl_version(try_next=True)
                raise SocketException('Secure Connection Failed')

            # If we reach here, we had a problem with our secure connection
            # handshaking and we were explicitly told to only use 1 (one)
            # protocol.  Thus there is nothing more to retry.  So throwing
            # a SocketException() is not a good idea.  Instead, we throw
            # a SocketRetryLimit() so it can be handled differently upstream
            raise SocketRetryLimit('There are no protocols left to try.')

        except socket.error, e:
            # logger.debug("Exception received: %s " % (e));
            if e[0] == errno.EINTR:
                # Ensure socket is closed
                self.close()
                # A Signal was caught,
                # return so we can handle this
                raise

        except:
            # Close socket
            self.close()
            # Raise issue
            raise

        # Toggle our connection flag
        self.connected = True

        # Call a connect hook
        self.hooks.call(
            'socket_listen',
            host=self._remote_addr,
            port=self._remote_port,
            socket=weakref.proxy(self),
        )

        return True

    def read(self, max_bytes=32768, timeout=None, retry_wait=0.25):
        """read()

           max_bytes:  Identify how many bytes to read from TCP stream

           timeout: If specified, then we will read until the amount of
                    bytes are read, or until the timeout (which ever comes
                    first)

            retry_wait: If the value is greater then zero, then more data
                    is requested once the retry wait elapses if available.
                    otherwise we use blocking mode

           raise an exception if connection lost.
        """
        total_data = []

        # Get reference time
        cur_time = datetime.now()

        # track bytes read
        bytes_read = 0

        # Current elapsed time
        elapsed_time = 0.0

        if not self.connected:
            # No connection
            return ''

        if retry_wait:
            # Make sure we're not blocking
            self.socket.setblocking(False)
        else:
            self.socket.setblocking(True)

        while self.connected and bytes_read < max_bytes:
            # put in a while-loop since this can block for some time
            # if there is no data to read, SIGHUP can cause it to bump
            # as well.

            # Update Elapsed Time
            elapsed_time = datetime.now() - cur_time
            elapsed_time = (elapsed_time.days * 86400) \
                + elapsed_time.seconds + (elapsed_time.microseconds / 1e6)

            if timeout and elapsed_time > timeout:
                # Time is up; return what we have
                break

            if timeout and not self.can_read(timeout - elapsed_time):
                # Times up
                break

            try:
                # Fetch data
                data = self.socket.recv(max_bytes - bytes_read)

                if data:
                    # Store data
                    bytes_read += len(data)
                    total_data.append(data)

                    # Get our elapsed transfer time
                    elapsed_xfer_time = datetime.now() - cur_time
                    elapsed_xfer_time = (
                        (elapsed_xfer_time.days * 86400) +
                        elapsed_xfer_time.seconds +
                        (elapsed_xfer_time.microseconds / 1e6)) - elapsed_time

                    # Call read hook
                    self.hooks.call(
                        'socket_read',
                        # host information
                        host=self._remote_addr,
                        port=self._remote_port,
                        # The number of bytes read
                        xfer_bytes=len(data),
                        # The time it took to read these bytes
                        xfer_time=elapsed_xfer_time,
                        # Our sockets
                        socket=weakref.proxy(self),
                    )

                # If we reach here, then we aren't using timeouts and the
                # socket returned nothing...
                if not data:
                    # We lost the connection
                    data = self.close()
                    if data:
                        # Store data
                        bytes_read += len(data)
                        total_data.append(data)

                    if not timeout:
                        raise SocketException('Connection lost')
                    break

                if timeout and bytes_read == 0:
                    # We're done
                    break

                elif retry_wait is None:
                    # we don't try again
                    if bytes_read == 0:
                        # We were blocking and yet we still yielded no
                        # results.  We must be broken
                        self.close()
                        raise SocketException('Connection broken')
                    break

                elif bytes_read < max_bytes and not self.can_read():
                    # Process what what we have because there is nothing
                    # further to be read
                    break

                continue

            except ssl.SSLWantReadError, e:
                # Raised by SSL Socket; This is okay data was received, but not
                # all of it. Be patient and try again.
                if retry_wait and self.can_read(retry_wait) is None:
                    self.close()
                    raise SocketException('Connection broken')
                continue

            except ssl.SSLZeroReturnError, e:
                # Raised by SSL Socket
                self.close()
                raise SocketException('Connection broken')

            except socket.error, e:
                if e[0] == errno.EAGAIN:
                    # Timeout occurred; Sleep for a little bit
                    if retry_wait and self.can_read(retry_wait) is None:
                        self.close()
                        raise SocketException('Connection broken')
                    continue

                elif e[0] == errno.EINTR:
                    # A Signal was caught return early
                    raise SignalCaughtException('Signal received')

                # A signal such as EPIPE was received to get here.
                # we can assume the connection is dead
                data = self.close()
                if data:
                    # Store data
                    bytes_read += len(data)
                    total_data.append(data)

                if not timeout:
                    raise SocketException('Connection broken')

        # Return Buffer
        return ''.join(total_data)

    def send(self, data, max_bytes=None, retry_wait=0.25):
        """ Socket Wrapper for people using this class as if it were just
            a python socket class; always ignore EINTR flags to finish
            sending the data
        """

        # track bytes written
        tot_bytes = 0

        # Get reference time
        cur_time = datetime.now()

        # Current elapsed time
        elapsed_time = 0.0

        if not max_bytes:
            max_bytes = len(data)

        while self.connected and (max_bytes - tot_bytes) > 0:

            # This timer is used to determine how long we should
            # wait before assuming we can't send content and that we
            # have connection problems.

            stale_timeout = max(((max_bytes - tot_bytes) / 10800.0), 15.0)

            if not self.can_write(stale_timeout):
                # can't write down pipe; something has gone wrong
                self.close()
                raise SocketException('Connection write wait timeout')

            # bump 10 seconds onto our timeout
            stale_timeout += 10

            # Initialize our stale timeout timer
            stale_timer = Timeout(stale_timeout)

            # Update Elapsed Time
            elapsed_time = datetime.now() - cur_time
            elapsed_time = (elapsed_time.days * 86400) \
                + elapsed_time.seconds + (elapsed_time.microseconds / 1e6)

            try:
                # Start stale_timer
                stale_timer.start()

                # Send data
                bytes_sent = self.socket.send(
                    data[tot_bytes:tot_bytes + max_bytes],
                )
                stale_timer.cancel()

                # Get our elapsed transfer time
                elapsed_xfer_time = datetime.now() - cur_time
                elapsed_xfer_time = (
                    (elapsed_xfer_time.days * 86400) +
                    elapsed_xfer_time.seconds +
                    (elapsed_xfer_time.microseconds / 1e6)) - elapsed_time

                if not bytes_sent:
                    self.close()
                    raise SocketException('Connection lost')

                # Handle content received
                tot_bytes += bytes_sent

                # Call write hook
                self.hooks.call(
                    'socket_write',
                    # Host information
                    host=self._remote_addr,
                    port=self._remote_port,
                    # The number of bytes read
                    xfer_bytes=bytes_sent,
                    # The time it took to read these bytes
                    xfer_time=elapsed_xfer_time,
                    # Our sockets
                    socket=weakref.proxy(self),
                )

            except Timeout:
                # Timeout occurred; Sleep for a little bit
                sleep(retry_wait)
                if self.can_write() is None:
                    self.close()
                    raise SocketException('Connection broken due to timeout')

            except socket.error, e:
                if e[0] == errno.EAGAIN:
                    # Timeout occurred; Sleep for a little bit
                    sleep(retry_wait)
                    if self.can_write() is None:
                        self.close()
                        raise SocketException('Connection broken')

                elif e[0] == errno.EINTR:
                    # A Signal was caught, resend this data before
                    # raising the signal higher.  signals can wait when
                    # there is data flowing
                    raise SignalCaughtException('Signal received')
                else:
                    # errno.EPIPE (Broken Pipe) usually at this point
                    self.close()
                    raise SocketException('Connection lost')
            except Exception, e:
                self.close()
                raise
            finally:
                # always stop the stale_timer
                stale_timer.cancel()

        return tot_bytes

    def local_connection_info(self):
        """
        Returns a tuple of current address of 'this' server
        if listening, then it is the listing server.  If performing a
        remote connection, then it is the address that was made in
        a bind() call

        If no connection has been established, the connection returns None.
        """

        if self.socket is None:
            return None

        return (self._local_addr, self._local_port)

    def remote_connection_info(self):
        """
        Returns a tuple of current address of 'this' server
        if listening, then it is the listing server.  If performing a
        remote connection, then it is the address that was made in
        a bind() call

        If no connection has been established, the connection returns None.
        """
        if self.socket is None:
            return None

        return (self._remote_addr, self._remote_port)

    def __ssl_version(self, try_next=False):
        """
        Returns an SSL Context Object while handling the many
        supported protocols
        """

        if try_next:
            # Increment version
            self.secure_protocol_idx += 1

        while self.secure_protocol_idx < len(SECURE_PROTOCOL_PRIORITY):
            # Now return it
            return SECURE_PROTOCOL_PRIORITY[self.secure_protocol_idx][0]

        # If we reach here, we had a problem; use SocketRetryLimit() instead
        # of SocketException() since we're at the end of the line now.
        raise SocketRetryLimit('There are no protocols left to try.')

    def __encrypt_socket(self, timeout=None, retry_wait=1.00,
                         verify=True, cert_reqs=ssl.CERT_NONE,
                         server_side=False):
        """
        Wrap an existing Python socket and return an SSLSocket Object.
        this is iternally called if we're dealing with a secure
        connection.

        timeout is used to break from this function if a certain period
        elapses. Otherwise, if None is specified, we'll only break on
        completion or if an error occurs.

        This function has no return value; it either encryptes the socket
        or throws a SocketException() error.
            cert_reqs can be:
                ssl.CERT_REQUIRED:  certificate verification manditory; no
                                     certificate means to fail.
                ssl.CERT_OPTIONAL:  certificate verifcation performed but
                                     only if one was detected.
                ssl.CERT_NONE:      No certificate verification performed

            verify can kick in if ssl.CERT_NONE is specified (otherwise
            it's not nesisary).  It handles those with self-signed certificates
            by doing it's best to perform some very basic checks that can help
            abort the connection when dealing with a man-in-the-middle attack.

        """
        # Disable Blocking
        self.socket.setblocking(False)

        if self.socket is None:
            # Nothing to do if we have no socket to work with
            raise SocketException("No connection")

        # Define our default keyword arguments
        kwargs = {
            'ssl_version': self.__ssl_version(),
            'do_handshake_on_connect': False,
            'suppress_ragged_eofs': True,
        }

        kwargs['server_side'] = server_side
        if server_side:
            # We need to add a few more parameters
            kwargs['keyfile'] = self._keyfile
            kwargs['certfile'] = self._certfile

            # Verify our certificates/keys exist or abort
            # These checks are nessisary otherwise you'll get strange errors
            # like:
            # _ssl.c:341: error:140B0002:SSL \
            #       routines:SSL_CTX_use_PrivateKey_file:system lib
            #
            # The error itself will surface during the call to wrap_socket()
            # which will throw the exception ssl.SSLError
            #
            # it doesn't hurt to just check ahead of time and make the error
            # human readable
            if not isfile(self._certfile):
                raise ValueError(
                    'Could not locate Certificate: %s' % self._certfile)
            if not isfile(self._keyfile):
                raise ValueError(
                    'Could not locate Private Key: %s' % self._keyfile)

        elif self._ca_certs:
            # if we have a ca_certs reference, then let's store them now
            kwargs['ca_certs'] = self._ca_certs
            if not isfile(self._ca_certs):
                raise ValueError(
                    'Could not locate CA Certificates: %s' %
                    self._ca_certs,
                )

            # Store our Certificate Requirements
            kwargs['cert_reqs'] = cert_reqs

        # Wrap our socket with the SSLSocket Object
        self.socket = ssl.wrap_socket(self.socket, **kwargs)

        # Get reference time
        cur_time = datetime.now()

        while True:
            # Infinit loop is nessisary for do_handshake() wrapping
            # we'll either exit this loop with a secure connection
            # or we'll time out and gracefully exit.
            try:
                # This command does a lot of the magic
                self.socket.do_handshake()

                logger.info("Secured connection using %s." % (
                    SECURE_PROTOCOL_PRIORITY[self.secure_protocol_idx][1],
                ))

                if not server_side:
                    # Store our peer certificate
                    try:
                        # self.peer_certificate = \
                        #    self.socket.getpeercert(binary_form=False)
                        # Returns None if there is no certificate for the peer
                        # on the other end; there is always a binary form, but
                        # not always the non-binary version
                        binary_cert = self.socket.getpeercert(binary_form=True)

                    except ValueError:
                        # SSL Handshaking hasn't completed yet; this is a
                        # horrible state to be in if this is the case because
                        # we're only at this part of the code because this
                        # handshaking is presumed to have already been
                        # complete.

                        # Fail at this point for the reason there is something
                        # wrong with our SSL.
                        raise SocketException('Secure handshaking failure')

                    try:
                        # load_der_x509_certificate() also throws a ValueError
                        # if it couldn't parse the certificate. So we need a
                        # separate try/except after acquiring the binary
                        # certificate
                        self.peer_certificate = x509.load_der_x509_certificate(
                            binary_cert,
                            default_backend(),
                        )
                    except ValueError:
                        # we couldn't acquire the certificate
                        self.peer_certificate = None
                        if verify:
                            raise SocketRetryLimit(
                                "Could not acquire site certificate.",
                            )
                        logger.warning("Could not acquire site certificate.")

                    if verify and cert_reqs == ssl.CERT_NONE:
                        # Our own verification process which certainly isn't
                        # bulletproof, but can help with self-signed
                        # certificates and still offer 'some' security
                        try:
                            cert_host = \
                                self.peer_certificate.subject\
                                    .get_attributes_for_oid(
                                        NameOID.COMMON_NAME)\
                                    .pop().value

                            # Perform a reverse lookup on our remote IP Address
                            (host, alias, ips) = \
                                socket.gethostbyaddr(self._remote_addr)

                            # certificate syntax; a simple flick and we make it
                            # a regex supported expression
                            cert_host = cert_host\
                                .replace('.', '\\.')\
                                .replace('*\\.', '.*\\.')

                            # If we get here, we've got a hostname to work with
                            host_match_re = re.compile(
                                cert_host,
                                re.IGNORECASE,
                            )

                            matched_host = next(
                                (h for h in [host] + alias + ips
                                 if host_match_re.match(h) is not None), False)

                            if not matched_host:
                                raise SocketRetryLimit(
                                    "Certificate for '%s' and does not "
                                    "match." % (
                                        cert_host,
                                    )
                                )

                        except socket.herror, e:
                            if e[0] == 2:
                                raise SocketRetryLimit(
                                    "Certificate for '%s' could not be "
                                    "resolved." % (
                                        self._remote_addr,
                                    )
                                )

                            # raise anything else
                            raise

                        # TODO: Store fingerprint (if not stored already)
                        #       If already stored, then verify that it hasn't
                        #       changed.

                        except IndexError:
                            raise SocketRetryLimit(
                                'Certificate hostname not defined!',
                            )

                # We're done
                self.connected = True
                return

            except ssl.SSLWantReadError:
                # SSL Connection will block until it is ready
                # The problem with this is can_read() and can_write()
                # all return True.  the socket needs more time
                # though to initialize so we have to call sleep here
                if self.can_read(retry_wait) is None:
                    self.close()
                    raise SocketException('Connection broken')
                continue

            except ssl.SSLWantWriteError:
                # SSL Connection will block until it is ready
                # The problem with this is can_read() and can_write()
                # all return True.  the socket needs more time
                # though to initialize so we have to call sleep here
                if self.can_write(retry_wait) is None:
                    self.close()
                    raise SocketException('Connection broken')
                continue

            if timeout:
                # Update Elapsed Time
                elapsed_time = datetime.now() - cur_time
                elapsed_time = (elapsed_time.days * 86400) \
                    + elapsed_time.seconds + (elapsed_time.microseconds / 1e6)

                if elapsed_time > timeout:
                    # Times up
                    self.close()
                    raise SocketException(
                        'Secure Connection Timeout')

        # This code is unreachable but just to satisify standards
        # we'll put a return statement here
        return

    def __del__(self):
        """
        Handle object destruction be closing off any open connections
        """
        self.close()

    def __str__(self):
        if self.secure is False:
            return 'tcp%s:%d' % (self._remote_addr, self._remote_port)
        # else
        return 'tcps://%s:%d' % (self._remote_addr, self._remote_port)

    def __unicode__(self):
        if self.secure is False:
            return u'tcp%s:%d' % (self._remote_addr, self._remote_port)
        # else
        return u'tcps://%s:%d' % (self._remote_addr, self._remote_port) class SocketException(Exception):
    """generic socket manager exception class"""
    pass class SocketRetryLimit(Exception):
    """generic socket manager exception for retry limits reached class"""
    pass class SignalCaughtException(Exception):
    """ Generic Signal Caught Exception;
        This is issued if the user interrupts the action
        such as pressing Ctrl-C or sending a SIGTERM, etc
    """
    pass def mkdir(name, perm=0775):
    """
    A more contained wrapper to directory management
    """
    attempt = 3
    if isdir(name):
        return True

    while attempt > 0:
        try:
            makedirs(name, perm)
            logger.debug('Created directory: %s' % name)
            return True

        except OSError, e:
            if e[0] == errno.EEXIST:
                # directory exists; this is okay
                return isdir(name)

            logger.debug('Created directory %s exception: %s' % (
                name, e,
            ))

        # racing condition; just try again
        attempt -= 1

    # To many attempts... fail
    # ... fall through...
    return False SEEK_SET = 0 SEEK_END = 2 class NNTPnzb(NNTPContent):
    """
    This class reads and writes to nzb files.

    It can also be passed into a NNTPConnection() get()
    call for the retrieval of articles identified.

    """

    def __init__(self, nzbfile=None, encoding=XML_ENCODING, work_dir=None,
                 mode=NZBParseMode.Simple, codecs=None, *args, **kwargs):
        """
        Initialize NNTP NZB object

        A nzbfile can optionally be specified identifying the location of the
        nzb file to open/read or even eventually write to if you're generating
        one.

        codecs: define the codecs you want to use to parse the articles
                contents
        """

        self._lazy_is_valid = None
        self._lazy_gid = None

        # XML Stream/Iter Pointer
        self.xml_iter = None
        self.xml_root = None
        self.xml_itr_count = 0

        # Meta information placed into (or read from) the <head/> tag
        self.meta = None

        # Encoding to use when handling the NZB File
        self.encoding = encoding

        # Track segmented files when added
        self.segments = sortedset(key=lambda x: x.key())

        # Segments are loaded on our first iteration through the NZB-File
        # Once these are loaded, most of the functions references come from the
        # self.segments instead of re-parsing the XML file again and again
        # We can use self.segments to rebuild a brand new XML file too
        self._segments_loaded = None
        self._segment_iter = None

        # An NZB-File Mode which can control out it responds
        self._nzb_mode = mode

        # Initialize our parent
        super(NNTPnzb, self).__init__(work_dir=work_dir, *args, **kwargs)

        # Used for it's ability to convert to and
        self._htmlparser = None

        # NNTPContent Object
        self._detached = True

        # The nzbfile
        self.filepath = nzbfile

        # Some pretty nzb printing meta information the padding character to
        # use. You probably wouldn't want to set this to anything other then a
        # space (' ') or a tab ('\t').
        self.padding = ' '

        # The multiplier is how many of these padding characters you want to
        # increase by per indent. Hence if you set this to 2 and the padding
        # to a space (' '), then each time a new sub-element is created, it
        # is indented 2 more spaces.
        #
        # Example of padding_multiplier set to 2 (and padding to space):
        #     1|<root>
        #     2|  <level1>
        #     3|    <level2>
        #     4|      <level3>
        #     5|      </level3>
        #     6|    </level2>
        #     7|  </level1>
        #     8|</root>
        #
        self.padding_multiplier = 2

        # Create our Mime object since we'll be using it a lot
        self._mime = Mime()

        # Load our Codecs
        self._codecs = codecs

        # Provide default codecs if required
        if not self._codecs:
            self._codecs = [CodecYenc(), ]

        elif isinstance(self._codecs, CodecBase):
            self._codecs = [self._codecs, ]

    def save(self, nzbfile=None, pretty=True, dtd_type=XMLDTDType.Public):
        """
        Write an nzbfile to the file and path specified. If no path is
        specified, then the one used to open the class is used.

        If that wasn't specified, then this function will return False.
        The function returns True if the save was successful

        If pretty is set to True then the output is formatted gently
        on the eyes; otherwise it is packed for disk size
        """
        if self.open(filepath=nzbfile, mode=NNTPFileMode.BINARY_RW_TRUNCATE):
            eol = '\n'
            indent = ''

            self.write('<?xml version="%s" encoding="%s"?>%s' % (
                XML_VERSION,
                self.encoding,
                eol,
            ))
            self.write('<!DOCTYPE %s %s %s>%s' % (
                XML_DOCTYPE,
                dtd_type,
                NZB_XML_DTD_MAP[dtd_type],
                eol,
            ))

            if not pretty:
                # No more end-of-lines are nessisary from this point on if
                # we're not out to format the content in any human-readable
                # fashion
                eol = ''

            self.write('<nzb xmlns="%s">%s%s' % (
                NZB_XML_NAMESPACE,
                eol, eol,
            ))

            if self.meta:
                if pretty:
                    indent = ''.ljust(self.padding_multiplier, self.padding)

                # Handle the meta information if there is anything at all to
                # print
                self.write('%s<head>%s' % (
                    indent,
                    eol,
                ))

                if pretty:
                    indent = ''.ljust(self.padding_multiplier*2, self.padding)

                for k, v in self.meta.items():
                    self.write('%s<meta type="%s">%s</meta>%s' % (
                        indent,
                        self.escape_xml(k.encode(self.encoding)),
                        self.escape_xml(v.encode(self.encoding)),
                        eol,
                    ))

                if pretty:
                    indent = ''.ljust(self.padding_multiplier, self.padding)

                self.write('%s</head>%s%s' % (
                    indent,
                    eol,
                    eol,
                ))

            for segno, segment in enumerate(self.segments):
                if not len(segment):
                    logger.error(
                        "NZB-File '%s' has no defined articles." % self.path())
                    self.remove()
                    return False

                if not len(segment.groups):
                    logger.warning(
                        "NZB-File '%s' (segno %d) has no defined groups." % (
                            self.path(),
                            segno,
                        ),
                    )

                if pretty:
                    indent = ''.ljust(self.padding_multiplier, self.padding)

                self.write('%s<file poster="%s" date="%s" subject="%s">%s' % (
                    indent,
                    self.escape_xml(segment.poster),
                    segment.utc.strftime('%s'),
                    self.escape_xml(segment.subject),
                    eol,
                ))

                if pretty:
                    indent = ''.ljust(
                        self.padding_multiplier*2, self.padding)

                self.write('%s<groups>%s' % (
                    indent,
                    eol,
                ))

                if pretty:
                    indent = ''.ljust(
                        self.padding_multiplier*3, self.padding)

                for group in segment.groups:
                    self.write('%s<group>%s</group>%s' % (
                        indent,
                        group,
                        eol,
                    ))

                if pretty:
                    indent = ''.ljust(
                        self.padding_multiplier*2, self.padding)
                self.write('%s</groups>%s' % (
                    indent,
                    eol,
                ))

                self.write('%s<segments>%s' % (
                    indent,
                    eol,
                ))

                for part, article in enumerate(segment):
                    if pretty:
                        indent = ''.ljust(
                            self.padding_multiplier*3, self.padding)

                    for attachment in article:
                        # use enumerated content and not the part assigned.
                        # this is by design because it gives developers the
                        # ability to add/remove items from the attachment and
                        # only use the part numbers for their own personal
                        # ordering.

                        _bytes = len(attachment)
                        self.write(
                            '%s<segment %snumber="%d">%s</segment>%s' % (
                                indent,
                                (('bytes="%d" ' % _bytes) if _bytes else ''),
                                part+1,
                                self.escape_xml(article.msgid()),
                                eol,
                            )
                        )

                if pretty:
                    indent = ''.ljust(self.padding_multiplier*2, self.padding)

                self.write('%s</segments>%s' % (
                    indent,
                    eol,
                ))

                if pretty:
                    indent = ''.ljust(self.padding_multiplier, self.padding)

                self.write('%s</file>%s%s' % (
                    indent,
                    eol, eol,
                ))

            if pretty:
                indent = ''

            self.write('%s<!-- Generated by %s v%s -->%s' % (
                indent, __title__, __version__, eol,
            ))

            self.write('%s</nzb>%s' % (
                indent,
                eol,
            ))
            # close our file
            self.close()
            return True

        logger.error("NZB-File '%s' could not be accessed." % self.path())
        return False

    def gid(self):
        """
        Returns the Global Identifier associated with an NZB File.
        This is just a unique way of associating this file with another
        posted to usenet.

        None is returned if the GID can not be acquired

        """
        if self._lazy_gid is None:

            if self._segments_loaded is True:
                # Use our segments already loaded in memory
                if len(self.segments) > 0:
                    try:
                        # Use the md5 hash of the first message-id of the
                        # first segment
                        _md5sum = hashlib.md5()
                        _md5sum.update(self.segments[0][0].msgid())

                        # Store our data
                        self._lazy_gid = _md5sum.hexdigest()

                    except (TypeError, AttributeError, IndexError):
                        # Can't be done
                        logger.warning(
                            'Cannot calculate GID from NZB-Segments: %s' %
                            self.filepath)

                return self._lazy_gid

            if self.is_valid() is False:
                # Save ourselves the time and cpu of parsing further
                return None

            # get ourselves the gid which is just the md5sum of the first
            # Article-ID
            iter(self)

            if self.xml_iter is None:
                return None

            # get the root element
            try:
                _, self.xml_root = self.xml_iter.next()

                segment = self.xml_root.xpath(
                    '/ns:nzb/ns:file[1]/ns:segments/ns:segment[1]',
                    namespaces=NZB_LXML_NAMESPACES,
                )[0]

                self.xml_root.clear()
                self.xml_root = None
                self.xml_itr_count = 0
                self.xml_iter = None

            except IndexError:
                logger.warning(
                    'NZB-File is missing initial </segment> element: %s' %
                    self.filepath,
                )
                # Thrown if no segment elements were found in the first file
                # entry; this NZBFile is not good.

                # We intentionally do not mark the invalidity of the situation
                # because we allow for nzbfiles that are still being
                # constructed.
                return None

            except StopIteration:
                logger.warning(
                    'NZB-File is missing </file> elements: %s' % self.filepath)
                # Thrown if no <file/> elements were found at all.
                # This NZBFile is not good.

                # We intentionally do not mark the invalidity of the situation
                # because we allow for nzbfiles that are still being
                # constructed.
                return None

            except IOError:
                logger.warning('NZB-File is missing: %s' % self.filepath)
                self.xml_root = None
                self.xml_itr_count = 0
                # Mark situation
                self._lazy_is_valid = False

            except XMLSyntaxError as e:
                logger.error("NZB-File '%s' is corrupt" % self.filepath)
                logger.debug('NZB-File XMLSyntaxError Exception %s' % str(e))
                # Mark situation
                self._lazy_is_valid = False

            except Exception:
                logger.error("NZB-File '%s' is corrupt" % self.filepath)
                logger.debug('NZB-File Exception %s' % str(e))
                # Mark situation
                self._lazy_is_valid = False
                return None

            try:
                # Use the md5 hash of the first message-id of the
                # first segment
                _md5sum = hashlib.md5()
                _md5sum.update(segment.text.strip())

                # Store our data
                self._lazy_gid = _md5sum.hexdigest()

            except (TypeError, AttributeError):
                # Can't be done
                logger.warning(
                    'Cannot calculate GID from NZB-File: %s' % self.filepath)

        return self._lazy_gid

    def load(self, filepath=None, mode=NZBParseMode.Simple, detached=True):
        """
        Loads segments into memory (this object)
        """

        # Reset all entries
        self.segments.clear()
        self._segments_loaded = None
        self._segment_iter = None
        self._nzb_mode = mode

        if filepath is not None:
            # Reset our variables
            self._lazy_is_valid = None
            self._lazy_gid = None
            self.close()

            if not super(NNTPnzb, self).load(filepath=filepath):
                self._segments_loaded = False
                return False

        for sp in self:
            # Load our segmented post entries into memory
            self.segments.add(sp)

        # Toggle our segments loaded flag
        self._segments_loaded = True

        return True

    def is_valid(self):
        """
        Validate if the NZB File is okay; this will generate some overhead
        but at the same time it caches a lot of the results it returns so
        future calls will be speedy

        The function returns True if the nzb file is valid, otherwise it
        returns False
        """

        if self._lazy_is_valid is None:
            if self.open():
                # Open DTD file and create dtd object
                dtdfd = open(NZB_XML_DTD_FILE)
                dtd = etree.DTD(dtdfd)
                # Verify our dtd file against our current stream
                try:
                    nzb = etree.parse(self.filepath)

                except XMLSyntaxError as e:
                    if e[0] is not None:
                        # We have corruption
                        logger.error(
                            "NZB-File '%s' is corrupt" % self.filepath)
                        logger.debug(
                            'NZB-File XMLSyntaxError Exception %s' % str(e))
                        # Mark situation
                        self._lazy_is_valid = False
                        # We failed
                        return False

                self._lazy_is_valid = dtd.validate(nzb)

        return self._lazy_is_valid is True

    def escape_xml(self, unescaped_xml, encoding=None):
        """
        A Simple wrapper to Escape XML charaters from a string
        """

        if isinstance(unescaped_xml, unicode):
            # Encode content as per defined in our XML
            if encoding is None:
                encoding = self.encoding
            unescaped_xml = unescaped_xml.encode(encoding)

        return sax_escape(unescaped_xml, {"'": "&apos;", "\"": "&quot;"})

    def add(self, obj):
        """
        Adds an NNTPSegmentedPost to an nzb object
        """

        if not isinstance(obj, NNTPSegmentedPost):
            return False

        # duplicates are ignored in a blist and therefore
        # we just capture the length of our list before
        # and after so that we can properly return a True/False
        # value
        _bcnt = len(self.segments)

        # Add our segment
        self.segments.add(obj)

        # A reference value we can use to compare against
        file_count = len(self.segments)

        # We're manually loading the segments so we can trust wherever
        # we leave off
        self._segments_loaded = True

        return file_count > _bcnt

    def unescape_xml(self, escaped_xml, encoding=None):
        """
        A Simple wrapper to UnEscape XML characters from a string
        """

        if encoding is None:
            encoding = self.encoding

        if self._htmlparser is None:
            # Define on first use
            self._htmlparser = HTMLParser()

        return self._htmlparser.unescape(escaped_xml).decode(encoding)

    def segcount(self):
        """
        Returns the total number of segments in the NZB File
        """
        seg_count = 0
        seg_count += sum(len(c) for c in self)
        return seg_count

    def deobsfucate(self, filebase=None):
        """
        Scans through the segments, articles and content associated with an
        NZB-File and sets up the filenames defined in the segments to it's
        best effort

        filebase: provide a fallback filename base (the part of the file
                  before the extension) to build on if we can't detect
                  the file on our own.

        """
        # The name from the meta tag
        _name = self.meta.get('name', '').decode(self.encoding).strip()
        if not _name:
            if filebase:
                # Use base provided as a base
                _name = filebase

            else:
                # If we can't get a name from the meta tag to start with, then
                # we use the NZB-Filename itself as a backup
                _name = splitext(basename(self.path()))[0]

        for segment in self.segments:
            filename = segment.deobsfucate(_name)
            if filename:
                # Update
                segment.filename = filename

    def next(self):
        """
        Python 2 support
        Support stream type functions and iterations
        """

        # We track our iterator since we move along if our mode tells us to do
        # so.
        _iter = None

        if self.xml_root is not None:
            # clear our unused memory
            self.xml_root.clear()

        if self._segment_iter:
            while 1:
                _iter = self._segment_iter.next()
                if self._valid_by_mode(_iter):
                    return _iter

        # get the root element
        try:
            _, self.xml_root = self.xml_iter.next()

            # Increment our iterator
            self.xml_itr_count += 1

        except StopIteration:
            # let this pass through
            self.xml_root = None
            self.xml_itr_count = 0

        except IOError:
            logger.warning('NZB-File is missing: %s' % self.filepath)
            self.xml_root = None
            self.xml_itr_count = 0
            # Mark situation
            self._lazy_is_valid = False

        except XMLSyntaxError as e:
            if e[0] is not None:
                # We have corruption
                logger.error("NZB-File '%s' is corrupt" % self.filepath)
                logger.debug('NZB-File XMLSyntaxError Exception %s' % str(e))
                # Mark situation
                self._lazy_is_valid = False
            # else:
            # this is a bug with lxml in earlier versions
            # https://bugs.launchpad.net/lxml/+bug/1185701
            # It occurs when the end of the file is reached and lxml
            # simply just doesn't handle the closure properly
            # it was fixed here:
            # https://github.com/lxml/lxml/commit\
            #       /19f0a477c935b402c93395f8c0cb561646f4bdc3
            # So we can relax and return ok results here
            self.xml_root = None
            self.xml_itr_count = 0

        except Exception as e:
            logger.error("NZB-File '%s' is corrupt" % self.filepath)
            logger.debug('NZB-File Exception %s' % str(e))
            # Mark situation
            self._lazy_is_valid = False

        if self.xml_root is None or len(self.xml_root) == 0:
            self.xml_iter = None
            self.xml_root = None
            self.xml_itr_count = 0

            raise StopIteration()

        if self.meta is None:
            # Attempt to populate meta information
            self.meta = {}

            for meta in self.xml_root.xpath('/ns:nzb/ns:head[1]/ns:meta',
                                            namespaces=NZB_LXML_NAMESPACES):
                # Store the Meta Information Detected
                self.meta[meta.attrib['type'].decode(self.encoding)] = \
                    self.unescape_xml(meta.text.strip())

        # Acquire the Segments Groups
        groups = [
            group.text.strip().decode(self.encoding)
            for group in self.xml_root.xpath(
                'ns:groups/ns:group',
                namespaces=NZB_LXML_NAMESPACES,
            )
        ]

        # The detected filename
        _filename = ''

        # The name from the meta tag
        _name = self.meta.get('name', '').decode(self.encoding).strip()

        if not _name and self.filepath is not None:
            # Lets try to generate a name frome our NZB-File
            tmpfname = basename(self.filepath)

            # Strip our extension off the end (if present)
            result = NZB_EXTENSION_RE(tmpfname)
            if result and result.group('fname'):
                # Store our new filename as our name
                _name = result.group('fname')

        # Subject
        _subject = self.unescape_xml(
                self.xml_root.attrib.get('subject', '')).decode(self.encoding)

        # Poster
        _poster = self.unescape_xml(
                self.xml_root.attrib.get('poster', '')).decode(self.encoding)

        # Use our Codec(s) to extract our Yenc Subject
        matched = None
        for c in self._codecs:
            # for each entry, parse our article
            matched = c.parse_article(
                subject=_subject,
                poster=_poster,
            )
            if matched:
                # We matched
                break

        if matched:
            # We succesfully got a filename from our subject line
            _filename = matched.get('fname', '').strip()
            if _filename and _name:
                # always allow the name to over-ride the detected filename if
                # we actually have a real name we can assciated with it by
                _ext = self._mime.extension_from_filename(_filename)
                if _ext:
                    _filename = '{0}{1}'.format(_name, _ext)

        # Initialize a NNTPSegmented File Object using the data we read
        _file = NNTPSegmentedPost(
            _filename,
            poster=_poster,
            epoch=self.xml_root.attrib.get('date', '0'),
            subject=_subject,
            groups=groups,
            work_dir=self.work_dir,
            sort_no=self.xml_itr_count,
        )

        # index tracker
        _last_index = 0

        # Now append our segments
        for segment in self.xml_root.xpath(
                'ns:segments/ns:segment', namespaces=NZB_LXML_NAMESPACES):

            _cur_index = int(segment.attrib.get('number', _last_index+1))
            try:
                _size = int(segment.attrib.get('bytes'))
                if _size < 0:
                    _size = 0

            except (TypeError, ValueError):
                _size = 0

            article = NNTPArticle(
                subject=_file.subject,
                poster=_file.poster,
                id=self.unescape_xml(segment.text),
                no=_cur_index,
                work_dir=self.work_dir,
                codecs=self._codecs,
            )

            # Store our empty content Placeholder
            article.add(
                NNTPEmptyContent(
                    filepath=_filename,
                    part=self.xml_itr_count,
                    total_size=_size,
                    work_dir=self.work_dir,
                )
            )

            # Add article
            _file.add(article)

            # Track our index
            _last_index = _cur_index

        if not self._valid_by_mode(_file):
            # Not used; recursively move along
            return self.next()

        # Return our object
        return _file

    def _valid_by_mode(self, segmented):
        """
        Internally called function that takes a segmented object and matches
        it against the mode defined by this class. We return True if the mode
        would otherwise accept this entry, otherwise we return False.
        """

        if self._nzb_mode == NZBParseMode.Simple:
            # We accept everything
            return True

        # Our Mime Response
        mr = None

        # Check our segmented content to see if it's a par-file
        if segmented.filename:
            # use filename
            mr = self._mime.from_filename(segmented.filename)

        if mr is None or mr == DEFAULT_MIME_TYPE:
            try:
                for article in segmented:
                    for content in article:
                        if content.filename:
                            mr = self._mime.from_filename(content.filename)

                        if mr is None or mr == DEFAULT_MIME_TYPE:
                            mr = self._mime.from_file(content.path())

                        if not (mr is None or mr == DEFAULT_MIME_TYPE):
                            raise StopIteration()

            except StopIteration:
                # we're done
                pass

        if mr is None or mr == DEFAULT_MIME_TYPE:
            # We don't know the type, so we can't do much here
            return True

        if (self._nzb_mode & NZBParseMode.IgnorePars):
            # our mime type is going to be one of the following
            #  - application/par2
            #  - application/x-par2
            #
            # it wil always be in lower case, so the following should
            # get our match for us:
            if mr.type().endswith('par2'):
                return False

        if (self._nzb_mode & NZBParseMode.IgnoreAttached):
            if segmented[0][0].is_attached():
                return False

        # If we get here, we can go ahead and process our file
        return True

    def __next__(self):
        """
        Python 3 support
        Support stream type functions and iterations
        """
        return self.next()

    def __iter__(self):
        """
        Grants usage of the next()
        """

        # First get a ptr to the head of our data
        super(NNTPnzb, self).__iter__()

        if self._segments_loaded is True:
            self._segment_iter = iter(self.segments)
            return self

        elif len(self.segments) > 0:
            # make sure we don't have any segments lingering
            self.segments.clear()

        if self.xml_root is not None:
            # clear our unused memory
            self.xml_root.clear()
            # reset our variable
            self.xml_root = None
            self.xml_itr_count = 0

        try:
            self.xml_iter = iter(etree.iterparse(
                self.filepath,
                tag="{%s}file" % NZB_XML_NAMESPACE,
            ))

        except IOError:
            logger.warning('NZB-File is missing: %s' % self.filepath)
            # Mark situation
            self._lazy_is_valid = False

        except XMLSyntaxError as e:
            if e[0] is not None:
                # We have corruption
                logger.error("NZB-File '%s' is corrupt" % self.filepath)
                logger.debug('NZB-File Exception %s' % str(e))
                # Mark situation
                self._lazy_is_valid = False
            # else:
            # this is a bug with lxml in earlier versions
            # https://bugs.launchpad.net/lxml/+bug/1185701
            # It occurs when the end of the file is reached and lxml
            # simply just doesn't handle the closure properly
            # it was fixed here:
            # https://github.com/lxml/lxml/commit\
            #       /19f0a477c935b402c93395f8c0cb561646f4bdc3
            # So we can relax and return ok results here

        except Exception as e:
            logger.error("NZB-File '%s' is corrupt" % self.filepath)
            logger.debug('NZB-File Exception %s' % str(e))
            # Mark situation
            self._lazy_is_valid = False

        return self

    def size(self):
        """
        Returns the calculated size of all of the contents residing in the
        NZB-File.  If it has not been downloaded yet, the size is calculated
        based on the contents of the NZB-File

        """
        if self._segments_loaded:
            return sum(s.size() for s in self.segments)

        else:
            return sum(c.size() for c in self)

    def __len__(self):
        """
        Returns the number of files in the NZB File
        """
        return len(self.segments) if self._segments_loaded is True\
            else sum(1 for c in self)

    def __getitem__(self, index):
        """
        Support accessing NNTPSegmentedPost objects by index
        """
        if self._segments_loaded is None:
            # Attempt to preload our content
            self.load()

        return self.segments[index]

    def __repr__(self):
        """
        Return a printable version of the file being read
        """
        return '<NNTPnzb filename="%s" />' % (
            self.filepath,
        ) DEFAULT_TMP_DIR = expanduser(join('~', '.config', 'newsreap', 'var', 'tmp')) NNTP_EOL = '\r\n' NNTP_EOD = '.\r\n' class CodecBase(object):

    def __init__(self, work_dir=None, throttle_cycles=8000, throttle_time=0.2,
                 max_bytes=0, *args, **kwargs):
        """
        The dir identfies the directory to store our sessions in
        until they can be properly handled.

        Throttling is used to give a break to the thread and not consume all
        of the cpu usage while their are still other threads in play.

        max_bytes is set to the number of bytes you want to have received
        before you automatically abort the connection and flag the object as
        having only been partially complete  Use this option when you need to
        inspect the first bytes of a binary file. Set this to zero to download
        the entire thing (this is the default value)

        """

        # CRC Masking
        self._crc = BIN_MASK

        self._escape = 0

        # Track the number of bytes encoded
        self._encoded = 0

        # Track the number of bytes decoded
        self._decoded = 0

        # Initialize our byte tracking maximum threshold
        self._max_bytes = max_bytes

        # Our Decoded content should get placed here
        self.decoded = None

        if work_dir is None:
            self.work_dir = DEFAULT_TMP_DIR
        else:
            self.work_dir = abspath(expanduser(work_dir))

        if not isdir(self.work_dir):
            # create directory
            if mkdir(self.work_dir):
                logger.info('Created directory: %s' % self.work_dir)
            else:
                logger.error('Failed to created directory: %s' % self.work_dir)
                # Should not continue under this circumstance
                raise IOError((
                    errno.EACCES,
                    'Failed to create directory: %s' % self.work_dir,
                ))

        # Tracks the lines processed
        self._lines = 0

        # Tracks the lines scanned
        self._total_lines = 0

        # Throttle relief
        self.throttle_cycles = throttle_cycles
        self.throttle_time = throttle_time
        self.throttle_iter = 0

    def decode_loop(self):
        """
        Throttle system usage

        This call should exist within the while() loop of all decode() calls
        """
        # TODO: Generate some statistics here too based on lines processed

        self.throttle_iter += 1
        if ((self.throttle_cycles % self.throttle_iter) == 0):
            sleep(self.throttle_time)
        return True

    def _calc_crc(self, decoded):
        """
        Calculate the CRC based on the decoded content passed in
        """
        self._escape = crc32(decoded, self._escape)
        self._crc = (self._escape ^ -1)

    def max_bytes(self):
        """
        Allows one to wrap the return of the _max_bytes  if the wish
        in child classes.  Otherwise we just return the max_bytes

        """
        return self._max_bytes

    def parse_article(self, subject, *args, **kwargs):
        """
        Decode a article subject line if possible and return a dictionary
        of key -> value pair.

        The following arguments are allowed (these are the same results
        returned from the NNTPConnection.get() call.

            id: u'the unique identifier',
            article_no: 12345678,
            poster: u'the poster's information',
            date: datetime() object,
            subject: u'a subject line in unicode',
            size: 2135  // the message size in bytes
            lines: 53   // the number of lines
            group: u'alt.group.one'
            xgroups : {
                // references the Message-ID (id) per cross post
                 u'alt.group.two': 987654321,
                 u'alt.group.three': 12341234,
            }

        Always return None if you can't resolve the keys. Valid keys are:

        'fname' : Tranlates to the parsed filename (required!)
        'desc'  : Should always translate to the description (if there is one)

        """
        # Unless this object is overridden, nothing magical happens here
        return None

    def detect(self, line, relative=True):
        """
        A Simple function that can be used to determine if there is
        content on the line matches what the codec handles

        The relative flag should be used for doing additional internal
        checking with the line presented.  It's to be interpreted as
        relative to the decoder.

        For example if we get an 'end' token before receiving a 'begin' token,
        we may want to fail and ignore the data. This is what relative is for
        since it checks for a matched token relative to what we might expect.
        Without the relative switch the line is just processed (it's contents
        returned).

        It returns None if there is the codec doesn't match otherwise
        it returns a dictionary of the keys and their mapped values.

        """
        # Until this is over-ridden, it is always assumed the codec
        # can not handle the the line in question
        return None

    def decode(self, stream):
        """
        A function must be written that will read from the
        stream and decode the contents.

        decode should always return a type NNTPContent() or
        something that inherits from the class. There is
        an exception made with Header processing since it
        returns a type NNTPHeader().

        The function should return True when it is possible
        for more content to exist but you processed all that
        was available in the current stream. It tells the calling
        library to keep passing in more data as it's read next time.
        This the decode() function must be smart enough to handle
        partial content and be able to resume when more content
        is passed to it.

        Any other value returned (None, False, etc) will be
        treated as though the decoding flat out failed. It will be
        reset() so that it can process new content.

        """
        return False

    def encode(self, stream):
        """
        A function must be written that will read from the
        stream and encodes the contents it's provided.

        The function should return an NNTPContent object or None if
        it can't perform the encoding.
        """
        return None

    def crc32(self):
        """ Returns the calculated crc32 string for the decoded data.
        """
        return "%08x" % (self._crc ^ BIN_MASK)

    def reset(self, *args, **kwargs):
        """ Simply resets the internal variables of the class
            so that it can be re-used without being re-initialized
        """
        self._crc = BIN_MASK
        self._decoded = 0
        self._escape = 0

        # Tracks the lines processed
        self._lines = 0

        # Tracks the lines scanned
        self._total_lines = 0

        # Reset our Decoded Content
        self.decoded = None

        # Reset our throttle iterator
        self.throttle_iter = 0

        return None

    def __len__(self):
        """ Returns the total number of decoded bytes
        """
        return self._decoded

    def __str__(self):
        """
        Return a printable version of the codec
        """
        return repr(self)

    def __repr__(self):
        """
        Return a printable object
        """
        return '<CodecBase lines_processed=%d />' % (
            self._lines,
        ) class CodecHeader(CodecBase):
    """
    This class is used for interpreting NNTP Headers
    """
    def __init__(self, descriptor=None, encoding=None, *args, **kwargs):
        super(CodecHeader, self)\
            .__init__(descriptor=descriptor, *args, **kwargs)

        # Used for internal meta tracking when using the decode()
        self.decoded = NNTPHeader(work_dir=self.work_dir)

        # Initialize Header Parsed Flag; This is used to ensure
        # the decoding of headers is only performed once
        self.header_parsed = False

        # We track the last key we stored because lines that start with a tab
        # are assumed to be concatinated to this entry
        self.last_key = None

        # The character set encoding usenet content is retrieved in
        if encoding is None:
            self.encoding = NNTP_DEFAULT_ENCODING
        else:
            self.encoding = encoding

    def detect(self, line, relative=True):
        """
        A Simple function that can be used to determine if this is a
        header entry or not.

        If relative is set to true, we additionally check the line
        content against content relative to the decoding process (`What are
        we expecting to have right now?`). For example, if we already
        successfully parsed a header, then we would return None because
        we're not expecting any more header data at this point.

        It returns None if there is no header key line, otherewise
        it returns a dictionary of the headers in the form of
        keys/value pairs

        """

        header_re = HEADER_RE.match(line)
        if not header_re:
            return None

        if relative and self.header_parsed:
            # Content has already been parsed, we now disable
            # this function
            return None

        _key = header_re.group('key')
        if _key:
            _value = header_re.group('value')
            self.last_key = _key

        elif self.last_key:
            # append our old value
            _key = self.last_key
            _value = header_re.group('extra')

        else:
            # Can't be a header
            return None

        return {
            'key': _key.decode(self.encoding),
            'value': _value.decode(self.encoding),
        }

    def decode(self, stream):
        """ read from the specified stream and process
            all of the headers. Returns a NNTPHeader object
            when completed.

            Header decoding works as follows:
                1. we ignore all white space and blank lines
                   at the head of the decoding process.
                2. once the first header is found, we then
                   process one header after another until an
                   empty line is found telling us to stop
                   or if we detect a line that simply isn't
                   decodable anymore.
        """
        if self.header_parsed:
            return self.decoded

        ws_head = True
        while self.decode_loop():

            # fall_back ptr
            ptr = stream.tell()

            # Read in our data
            data = stream.readline()
            if not data:
                if not len(self.decoded):
                    # Not a header we're dealing with
                    return None

                # We're done
                break

            # Total Line Tracking
            self._total_lines += 1

            # Detect a header line
            header = self.detect(data, relative=False)
            if header is None:
                # Check for White Space
                if not len(data.strip()):
                    # We found white space if we get here

                    if ws_head:
                        # We ignore all white space at head
                        # so we keep going
                        continue
                else:
                    # We found data that may need to be processed
                    # by someting else; we want to backtrack our
                    # pointer.
                    stream.seek(ptr, SEEK_SET)

                    # Fix our line count
                    self._total_lines -= 1

                # We're done!
                self.header_parsed = True

                # Return our Header
                return self.decoded

            # Processing Line Tracking
            self._lines += 1

            # We processed meta data, therefore we have started reading in
            # header content.  We want to toggle the ws_head flag so that
            # we no longer accept white space as acceptable content
            if ws_head:
                ws_head = False

            # Now we store our key/value pair into our header
            # To keep things fast; we specifically place content directly
            # into the content object to avoid the extra un-nessisary
            # function call per line
            if not header['key'] in self.decoded:
                self.decoded[header['key']] = header['value']

            else:
                # Append our content to the last match
                self.decoded[header['key']] += u',' + header['value']

        # Returns true because we're still expecting more content
        return True

    def is_valid(self):
        """
        A simple function that returns whether or not the block just read
        checks out okay.

        The function returns True if content is valid, False if it isn't
        and None if there isn't enough information to make a valid guess.
        """

        if not self.header_parsed:
            # We never decoded anything
            logger.debug("%s has not been decoded." % self)
            return None

        # If we reach here; we've decoded the file, the next thing
        # we need to do is check for some common entries that
        # would oherwise have made it so it's not valid
        if next((True for k in self.decoded.iterkeys()
                 if NNTP_NUKED_MSG.match(k) is not None), False):
            # Headers check failed!
            return False

        # Nothing bad to indicate bad headers
        return True

    def reset(self):
        """
        Reset our decoded content
        """
        super(CodecHeader, self).reset()

        # Reset Our Result set
        self.decoded = NNTPHeader(work_dir=self.work_dir)

        # Initialize Header Parsed Flag; This is used to ensure
        # the decoding of headers is only performed once
        self.header_parsed = False

    def __setitem__(self, key, item):
        """
        Mimic Dictionary:  dict[key] = value
        """
        self.decoded[key] = item

    def __getitem__(self, key):
        """
        Mimic Dictionary:  value = dict[key]
        """
        return self.decoded[key]

    def __delitem__(self, key):
        """
        Mimic Dictionary:  del dict[key]
        """
        del self.decoded[key]

    def __str__(self):
        """
        Return a printable version of the codec
        """
        return repr(self)

    def __len__(self):
        """
        Returns the dictionary length
        """
        return len(self.decoded)

    def __repr__(self):
        """
        Return a printable object
        """
        return '<CodecHeader lines_processed=%d />' % (
            self._lines,
        ) class CodecArticleIndex(CodecBase):
    """
    This is the codec used to store general content parsed that is not encoded
    on an NNTP Server.
    """

    def __init__(self, descriptor=None, filters=None, sort=None,
                 encoding=None, work_dir=None, *args, **kwargs):
        super(CodecArticleIndex, self).__init__(descriptor=descriptor,
            work_dir=work_dir, *args, **kwargs)

        # Our Meta Content
        self.decoded = NNTPMetaContent(work_dir=self.work_dir)

        # Switch our content subvalue to be a sorteddict()
        self.decoded.content = sorteddict()

        # Filters
        self.filters = filters

        # Sort Order
        self.sort = sort
        if self.sort is None or self.sort not in XOVER_GROUPINGS:
            self.sort = XoverGrouping.BY_POSTER_TIME

        # The character set encoding usenet content is retrieved in
        if encoding is None:
            self.encoding = NNTP_DEFAULT_ENCODING
        else:
            self.encoding = encoding


    def detect(self, line, relative=True):
        """
        A Simple function that can be used to determine if there is a group
        entry or not. The relative flag is not used for this codec but is
        defined as per the codec standards.

        It returns None if there this is not a group entry line, otherwise
        it returns a dictionary of the keys and their mapped values.

        """
        result = NNTP_XOVER_RESPONSE_RE.match(line)
        if not result:
            return None

        try:
            # handle datetime
            article_date = parse(result.group('date'))

        except (ValueError, TypeError):
            try:
                # We weren't able to parse the date
                article_date = parse(result.group('date'), fuzzy=True)

            except (ValueError, TypeError):
                # It would be great if users of the product would email
                # us these errors so we can work on making them better
                logger.warning(
                    "Unparsable XOVER date '%s'" % (result.group('date')),
                )
                article_date = datetime.fromtimestamp(0)

        if article_date.tzinfo is not None:
            # ensure all times are naive
            if isinstance(article_date.tzinfo, tzutc):
                article_date = article_date.replace(tzinfo=None)
            else:
                article_date = article_date.astimezone(UTC)\
                        .replace(tzinfo=None)
        try:
            lines = int(result.group('lines'))
        except (TypeError, ValueError):
            # Initialize lines to -1 if they weren't specified
            lines = -1

        entry = {
            'id': result.group('id'),
            'article_no': int(result.group('article_no')),
            'poster': result.group('poster').decode(self.encoding),
            'date': article_date,
            'subject': result.group('subject').decode(self.encoding),
            'size': int(result.group('size')),
            'lines': lines,
            'group': None,
            'score': 0,
            'xgroups': {},
        }

        # Split results
        results = re.split('\s+', result.group('group').strip())

        # Append remaining groups
        for x in results:
            grp = x.split(':')
            if len(grp) > 1:
                # we're dealing with a Cross Post
                try:
                    entry['xgroups'][grp[0]] = int(grp[1])
                except ValueError:
                    # This happens from time to time when a group
                    # doesn't have it's cross-group identifier
                    # identified; so we set it to zero for
                    # compatibility for anyone else using this
                    # wrapper
                    entry['xgroups'][grp[0]] = 0
            else:
                entry['group'] = grp[0]

        return entry


    def decode(self, stream):
        """ Decode the group content
        """

        # We need to parse the content until we either reach
        # the end of the file or get to an 'end' tag
        while self.decode_loop():

            # Read in our data
            data = stream.readline()
            if not data:
                # We're done
                break

            # Total Line Tracking
            self._total_lines += 1

            # Detect a group line
            entry = self.detect(data, relative=False)
            if entry is None:
                continue

            # A key we'll use hash into our btree with.
            # This grouping will allow us to destroy content
            # as it is processed as well as the handling of content
            # together
            if self.sort == XoverGrouping.BY_POSTER_TIME:
                key = '%s:%s:%.10d' % (
                    entry['poster'],
                    entry['date'].strftime('%Y%m%d%H%M%S'),
                    entry['article_no'],
                )

            elif self.sort == XoverGrouping.BY_TIME:
                key = '%s:%.10d' % (
                    entry['date'].strftime('%Y%m%d%H%M%S'),
                    entry['article_no'],
                )

            else: # XoverGrouping.BY_ARTICLE_NO
                key = '%.10d' % entry['article_no']

            # Check to see if we've added filters
            if self.filters:
                # First Apply our scores
                entry['score'] += sum(
                    [ f.score(**entry) for f in self.filters ],
                )

                # Scan our whitelist(s) and break on our first match
                if not next((True for f in self.filters \
                             if f.whitelist(**entry) == True), False):

                    # Scan our blacklist(s) and break on our first
                    # match
                    if next((True for f in self.filters \
                            if f.blacklist(**entry) == True), False):

                        # Skip entries matched on our blacklist
                        continue

            self.decoded.content[key] = entry
            #logger.debug('len=%d' % NNTP_XOVER_RESPONSE_RE.groups)
            #for x in range(1, NNTP_XOVER_RESPONSE_RE.groups):
            #    logger.debug('  %d=%s' % (x, str(result.group(x))))

            # Track the number of bytes decoded
            self._decoded += len(data)

        # Line Tracking
        self._lines = len(self.decoded.content)

        # With this group type; we're always expecting more
        # there are no if/ands or buts!
        return True


    def is_valid(self):
        """
        A simple function that returns whether or not the block just read
        checks out okay.
        """
        return len(self.decoded)


    def reset(self):
        """
        Reset our decoded content
        """
        super(CodecArticleIndex, self).reset()

        # Our Meta Content
        self.decoded = NNTPMetaContent(work_dir=self.work_dir)

        # Switch our decoded subvalue to be a sorteddict()
        self.decoded.content = sorteddict()


    def __str__(self):
        """
        Return a printable version of the codec
        """
        return repr(self)


    def __repr__(self):
        """
        Return a printable object
        """
        return '<CodecArticleIndex lines_processed=%d />' % (
            self._lines,
        ) class XoverGrouping(object):
    """
    Defines the xover grouping
    """
    BY_ARTICLE_NO = 0
    BY_POSTER_TIME = 1
    BY_TIME = 2 class CodecGroups(CodecBase):
    """
    This is the codec used to store general content parsed that is not encoded
    on an NNTP Server.
    """

    def __init__(self, descriptor=None, work_dir=None, *args, **kwargs):
        super(CodecGroups, self).__init__(descriptor=descriptor,
            work_dir=work_dir, *args, **kwargs)

        # Our Decoded Content
        self.decoded = NNTPMetaContent(work_dir=self.work_dir)

    def detect(self, line, relative=True):
        """
        A Simple function that can be used to determine if there is a group
        entry or not. The relative flag is not used for this codec but is
        defined as per the codec standards.

        It returns None if there this is not a group entry line, otherwise
        it returns a dictionary of the keys and their mapped values.

        """
        group_re = NNTP_LIST_ACTIVE_RESPONSE_RE.match(line)
        if not group_re:
            return None

        try:
            low = int(group_re.group('low'))
            high = int(group_re.group('high'))
        except (ValueError, TypeError):
            # can't be a group line
            return None

        if low < 0:
            # can't be a group line
            return None

        if high < 0:
            # can't be a group line
            return None

        # Detect empty (based on the 3 possibilities)
        # * high is >= low
        # * low - high == 1
        # * low - high == 0 (this check isn't needed if we check >=)
        if high >= low or (low - high) == 1:
            count = 0
        else:
            count = low-high

        try:
            # We intentionally do not strip unsupported flags
            # so it alright for usenet servers to impliment new ones
            # and we can keep on trucking with what we have. Feel
            # free to use the mapped valid defined in here though
            # in the GroupStatus object
            flags = list(group_re.group('flags'))

        except TypeError:
            # There were no flags defined
            # Always set a list (for consistency)
            flags = []

        return {
            'group': group_re.group('group').lower(),
            'low': low,
            'high': high,
            'count': count,
            'flags': flags,
        }

    def decode(self, stream):
        """ Decode the group content
        """

        # We need to parse the content until we either reach
        # the end of the file or get to an 'end' tag
        # We do not need to use the decode_loop() simply because this
        # function will never be called concurrently with other threads.
        while True:
            # Read in our data
            data = stream.readline()
            if not data:
                # We're done
                break

            # Total Line Tracking
            self._total_lines += 1

            # Detect a group line
            decoded = self.detect(data, relative=False)
            if decoded is not None:
                # We're good
                self.decoded.content.append(decoded)

            # Track the number of bytes decoded
            self._decoded += len(data)

        # Line Tracking
        self._lines = len(self.decoded.content)

        # With this group type; we're always expecting more
        # there are no if/ands or buts!
        return True

    def reset(self):
        """
        Reset our decoded content
        """
        super(CodecGroups, self).reset()

        # Reset Our Result set
        self.decoded = NNTPMetaContent(work_dir=self.work_dir)

    def __str__(self):
        """
        Return a printable version of the codec
        """
        return repr(self)

    def __repr__(self):
        """
        Return a printable object
        """
        return '<CodecGroups lines_processed=%d />' % (
            self._lines,
        ) class CodecUU(CodecBase):
    """
    This codec is used to manage all UUEncoded content found on an NNTP Server.
    """
    def __init__(self, work_dir=None, perm=0664, linelen=128, *args, **kwargs):
        super(CodecUU, self).__init__(work_dir=work_dir, *args, **kwargs)

        # Used for internal meta tracking when using the decode()
        self._meta = {}

        # Our Binary Object we can reference while we decode
        # content
        self.decoded = None

        # Permission
        self.perm = perm

        # Used for encoding; This defines the maximum number of (encoded)
        # characters to display per line.
        self.linelen = linelen

    def encode(self, content, mem_buf=UUENCODE_BLOCK_SIZE):
        """
        Encodes an NNTPContent object passed in
        """

        if isinstance(content, NNTPContent):
            # Create our ascii instance
            _encoded = NNTPAsciiContent(
                filepath=content.filename,
                part=content.part,
                total_parts=content.total_parts,
                sort_no=content.sort_no,
                work_dir=self.work_dir,
                # We want to ensure we're working with a unique attached file
                unique=True,
            )

        else:
            # If we reach here, we presume our content is a filename

            # Create our ascii instance
            _encoded = NNTPAsciiContent(
                filepath=content,
                work_dir=self.work_dir,
                # We want to ensure we're working with a unique attached file
                unique=True,
            )

            # Convert our content object into an NNTPContent object
            content = NNTPContent(
                filepath=content,
                work_dir=self.work_dir,
            )

        if mem_buf > UUENCODE_BLOCK_SIZE:
            logger.warning(
                "mem_buf (%d) reduced to %d due to uuencode limitations" %
                mem_buf, UUENCODE_BLOCK_SIZE,
            )
            mem_buf = UUENCODE_BLOCK_SIZE

        # Header/Footer
        fmt_ubegin = 'begin %o %s' % (self.perm, content.filename)

        # EOL Separation for UU Encoding just consists of a single \n
        fmt_uend = '`\nend'

        # Write ubegin line
        _encoded.write(fmt_ubegin + EOL)

        if not content.open():
            return None

        # Prepare our result set
        results = ""

        # We need to parse the content until we either reach
        # the end of the file or get to an 'end' tag
        while True:
            # Read in our data
            data = content.stream.read(UUENCODE_BLOCK_SIZE)
            if not data:
                # We're done
                break

            # Encode content
            results = b2a_uu(data)

            if results:
                # write our content as is
                _encoded.write(results)

        # We're done reading our data
        content.close()

        # Write footer
        _encoded.write(fmt_uend + EOL)

        if _encoded:
            # close article when complete
            _encoded.close()

        # Return our encoded object
        return _encoded

    def detect(self, line, relative=True):
        """
        A Simple function that can be used to determine if there is
        uuencoding content on the line being checked.

        If relative is set to true, we additionally check the line
        content against content relative to the decoding process (`What are
        we expecting to have right now?`). For example, the `end` token would
        be ignored if we haven't received a `begin` first.

        It returns None if there is no uuencoding key line, otherwise
        it returns a dictionary of the keys and their mapped values.

        """
        uuencode_re = UUENCODE_RE.match(line)
        if not uuencode_re:
            return None

        # Merge Results
        f_map = dict(
            (UUENCODE_KEY_MAP[k], v) for k, v
            in uuencode_re.groupdict().iteritems() if v
        )

        if relative:
            # detect() relative to what has been decoded
            if f_map['key'] in self._meta:
                # We already processed this key
                return None

            if f_map['key'] == 'end' and 'begin' not in self._meta:
                # We can't handle this key
                return None

        # Tidy filename (whitespace)
        if 'name' in f_map:
            f_map['name'] = basename(f_map['name']).strip()

        # Permission Flag (Octal Type)
        if 'perm' in f_map:
            try:
                f_map['perm'] = int(f_map['perm'], 8)
            except (TypeError, ValueError):
                # Eliminate bad key
                del f_map['perm']

        return f_map

    def decode(self, stream):
        """ Decode some data and decode the data
            to descriptor identified (by the stream)
        """

        while self.decode_loop():
            # fall_back ptr
            ptr = stream.tell()

            # Read in our data
            data = stream.readline()
            if not data:
                # We're done for now
                return True

            # Total Line Tracking
            self._total_lines += 1

            # Detect a uuenc line
            _meta = self.detect(data, relative=False)
            if _meta is not None:
                #
                # We just read a uu keyword token such as
                # begin, or end
                #
                if _meta['key'] in self._meta:
                    # We already processed this key; uh oh
                    # Fix our stream
                    stream.seek(ptr, SEEK_SET)

                    # Fix our line count
                    self._total_lines -= 1

                    # we're done
                    break

                if _meta['key'] == 'end' and 'begin' not in self._meta:
                    # Why did we get an end before a begin?
                    # Just ignore it and keep going
                    continue

                # store our key
                self._meta[_meta['key']] = _meta

                if 'end' in self._meta:
                    # Mark the binary as being valid
                    self.decoded._is_valid = True

                    # We're done!
                    break

                elif EOM in self._meta:
                    # Mark the binary as being valid
                    self.decoded._is_valid = True

                    # But keep going because we'll probably get an 'end' next
                    continue

                elif _meta['key'] == 'begin':

                    if 'name' not in _meta:
                        # Why did we get a begin before a part
                        # Just ignore it and keep going
                        continue

                    # Create our binary instance
                    self.decoded = NNTPBinaryContent(
                        filepath=_meta['name'],
                        work_dir=self.work_dir,
                    )

                    # Open our file for writing
                    self.decoded.open()
                continue

            if 'begin' not in self._meta:
                # We haven't found the start yet which means we should just
                # keep going until we find it
                continue

            try:
                decoded = a2b_uu(data)

            except BinAsciiError:
                # Workaround for broken uuencoders by
                # Fredrik Lundh (taken from uu.py code)
                nbytes = (((ord(data[0])-32) & 63) * 4 + 5) / 3

                try:
                    decoded = a2b_uu(data[:nbytes])

                except BinAsciiError:
                    # Log corruption
                    logger.warning(
                        "Corruption on line %d." % (
                            self._lines,
                        )
                    )

                    # Line Tracking
                    self._lines += 1

                    # keep storing our data
                    continue

                # CRC Calculations
                self._calc_crc(decoded)

            # Line Tracking
            self._lines += 1

            # Track the number of bytes decoded
            self._decoded += len(decoded)

            # Write data to out stream
            self.decoded.write(decoded)

            if self._max_bytes > 0 and self._decoded >= self._max_bytes:
                # If we specified a limit and hit it then we're done at
                # this point. Before we do so; advance to the end of our
                # stream
                stream.seek(0, SEEK_END)
                break

        # Reset our meta tracking
        self._meta = {}

        if self.decoded:
            # close article when complete
            self.decoded.close()

        # Return what we do have
        return self.decoded

    def reset(self):
        """
        Reset our decoded content
        """
        super(CodecUU, self).reset()

        # Used for internal meta tracking when using the decode()
        self._meta = {}

        # Our Binary Object we can reference while we decode
        # content
        self.decoded = None

    def __lt__(self, other):
        """
        Sorts by filename
        """
        return str(self) < str(other)

    def __str__(self):
        """
        Return a printable version of the file being read
        """

        # Build a string using the data we know
        if 'begin' in self._meta:
            fname = self._meta.get('name', 'Unknown.File')
        else:
            fname = 'Undetermined.File'

        return '%s' % (
            fname
        )

    def __repr__(self):
        """
        Return a printable object
        """
        return '<CodecUU lines_processed=%d />' % (
            self._lines,
        ) class CodecYenc(CodecBase):

    def __init__(self, descriptor=None, work_dir=None,
                 linelen=128, *args, **kwargs):
        super(CodecYenc, self).__init__(
                descriptor=descriptor, work_dir=work_dir, *args, **kwargs)

        # Used for internal meta tracking when using the decode()
        self._meta = {}

        # Our Binary Object we can reference while we decode
        # content
        self.decoded = None

        # Used for encoding; This defines the maximum number of (encoded)
        # characters to display per line.
        self.linelen = linelen

    def parse_article(self, subject, *args, **kwargs):
        """
        Takes a an article header and returns it's parsed content if it's
        successful. Otherwise it returns None.
        """

        matched = NZB_SUBJECT_PARSE.match(subject)
        if matched is None:
            # subject is not parsable
            return None

        results = {}

        # Trim results
        if matched.group('desc') is not None:
            results['desc'] = re.sub('[\s-]+$', '', matched.group('desc'))
        if matched.group('fname') is not None:
            results['fname'] = matched.group('fname').strip()

        # Support conversion of integers
        for _attr in ['index', 'count', 'yindex', 'ycount', 'size']:
            if matched.group(_attr) is not None:
                results[_attr] = int(matched.group(_attr))

        return results

    def encode(self, content, mem_buf=DEFAULT_BUFFER_SIZE):
        """
        Encodes an NNTPContent object passed in
        """

        if isinstance(content, NNTPContent):
            # Create our ascii instance
            _encoded = NNTPAsciiContent(
                filepath=content.filename,
                part=content.part,
                total_parts=content.total_parts,
                sort_no=content.sort_no,
                work_dir=self.work_dir,
                # We want to ensure we're working with a unique attached file
                unique=True,
            )

        else:
            # If we reach here, we presume our content is a filename

            # Create our ascii instance
            _encoded = NNTPAsciiContent(
                filepath=content,
                work_dir=self.work_dir,
                # We want to ensure we're working with a unique attached file
                unique=True,
            )

            # Convert our content object into an NNTPContent object
            content = NNTPContent(
                filepath=content,
                work_dir=self.work_dir,
            )

        # yEnc (v1.3) begin
        fmt_ybegin = '=ybegin part=%d total=%d line=%d size=%d name=%s' % (
            content.part, content.total_parts, self.linelen,
            len(content), content.filename,
        )

        # yEnc part
        fmt_ypart = '=ypart begin=%d end=%d' % (
            content.begin() + 1,
            content.end(),
        )

        if isinstance(content._parent, NNTPContent):
            # yEnc end
            fmt_yend = '=yend size=%d part=%d pcrc32=%s crc32=%s' % (
                len(content), content.part,
                content.crc32(), content._parent.crc32(),
            )

        else:
            # yEnc end
            fmt_yend = '=yend size=%d part=%d pcrc32=%s' % (
                len(content), content.part, content.crc32(),
            )

        # Write =ybegin line
        _encoded.write(fmt_ybegin + EOL)
        # Write =ypart line
        _encoded.write(fmt_ypart + EOL)

        if not content.open():
            return None

        # Prepare our result set
        results = ""

        # Column is used for decoding
        column = 0
        crc = BIN_MASK

        # We need to parse the content until we either reach
        # the end of the file or get to an 'end' tag
        while True:
            # Read in our data
            data = content.stream.read(mem_buf)
            if not data:
                # We're done
                break

            if FAST_YENC_SUPPORT:
                try:
                    _results, crc, column = encode_string(data, crc, column)
                    # Append our parsed content onto our ongoing buffer
                    results += _results

                except YencError as e:
                    logger.error("Failed to encode Yenc for %s." % content)
                    logger.debug('Yenc exception: %s' % (str(e)))
                    return None

            else:
                # The slow and painful way, the below looks complicated
                # but it really isn't at the the end of the day; yEnc is
                # pretty basic;
                #  - first we translate the all of the characters by adding
                #    42 to their value with the exception of a few special
                #    characters that are explicitly reserved for the yEnc
                #    language (and conflict with the NNTP Server language).
                #
                #  - next, we need to apply our ENCODE_SPECIAL_MAP to be
                #    sure to handle the characters that are reserved as
                #    special keywords used by both NNTP Servers and the yEnc
                #    protocol itself.
                #
                #  - finally we want to prevent our string from going on for
                #    to many characters (horizontally).  So we need to split
                #    our content up
                #

                idx = 0
                while idx < len(data):
                    _byte = (ord(data[idx]) + 42) & 0xff
                    if _byte in YENC_ENCODE_ESCAPED_CHARACTERS:
                        _byte = (_byte + 64) & 0xff
                        # Esape Sequence
                        results += '='

                    # Store our character
                    results += chr(_byte)

                    # Increment Index
                    idx += 1

            # Our offset
            offset = 0

            while offset < (len(results)-self.linelen+1):
                eol = offset+self.linelen
                if results[offset:eol][-1] == '=':
                    # Lines can't end with the escape sequence (=). If we get
                    # here then this one did. We just adjust our end-of-line
                    # by 1 and keep moving
                    eol -= 1

                _encoded.write(results[offset:eol] + EOL)
                offset = eol

            if offset < len(results):
                results = results[-(len(results) - offset):]

            else:
                # reset string
                results = ''

        # We're done reading our data
        content.close()

        if len(results):
            # We still have content left in our buffer
            _encoded.write(results + EOL)

        # Write footer
        _encoded.write(fmt_yend + EOL)

        if _encoded:
            # close article when complete
            _encoded.close()

        # Return our encoded object
        return _encoded

    def detect(self, line, relative=True):
        """
        A Simple function that can be used to determine if there is
        yEnc content on the line being checked.

        If relative is set to true, we additionally check the line
        content against content relative to the decoding process (`What are
        we expecting to have right now?`). For example, the `end` token would
        be ignored if we haven't received a `begin` first.

        It returns None if there is no yEnc key line, otherwise
        it returns a dictionary of the keys and their mapped values.

        """
        yenc_re = YENC_RE.match(line)
        if not yenc_re:
            return None

        # Merge Results
        f_map = dict((YENC_KEY_MAP[k], v) for k, v
                     in yenc_re.groupdict().iteritems() if v)

        # Tidy filename (whitespace)
        if 'name' in f_map:
            f_map['name'] = basename(f_map['name']).strip()

        if relative:
            # detect() relative to what has been decoded
            if f_map['key'] in self._meta:
                # We already processed this key
                return None

            if f_map['key'] == 'end' and 'begin' not in self._meta:
                # We can't handle this key
                return None

            if f_map['key'] == 'part' and 'begin' not in self._meta:
                # We can't handle this key
                return None

        # Integer types
        for kw in ['line', 'size', 'total', 'begin', 'end', 'part']:
            if kw in f_map:
                try:
                    f_map[kw] = int(f_map[kw])

                except (TypeError, ValueError):
                    # Eliminate bad kw
                    del f_map[kw]

        return f_map

    def decode(self, stream):
        """ Decode some data and decode the data
            to descriptor identified (by the stream)
        """

        # We need to parse the content until we either reach
        # the end of the file or get to an 'end' tag
        while self.decode_loop():
            # fall_back ptr
            ptr = stream.tell()

            # Read in our data
            data = stream.readline()
            if not data:
                # We're done for now
                return True

            # Total Line Tracking
            self._total_lines += 1

            # Detect a yEnc line
            _meta = self.detect(data, relative=False)
            if _meta is not None:
                #
                # We just read a yEnc keyword token such as
                # begin, part, or end
                #
                if _meta['key'] in self._meta:
                    # We already processed this key; uh oh
                    # Fix our stream
                    stream.seek(ptr, SEEK_SET)

                    # Fix our line count
                    self._total_lines -= 1

                    # We're done
                    break

                if _meta['key'] == 'end' and \
                   len(set(('begin', 'part')) - set(self._meta)) == 2:
                    # Why did we get an end before a begin or part?
                    # Just ignore it and keep going
                    continue

                # store our key
                self._meta[_meta['key']] = _meta

                if 'end' in self._meta:
                    # Mark the binary as being valid
                    self.decoded._is_valid = True

                    # We're done!
                    break

                elif _meta['key'] == 'begin':
                    # Depending on the version of yEnc we're using binary
                    # content starts now; thefore we create our binary
                    # instance now

                    if 'name' not in _meta:
                        # Why did we get a begin before a part
                        # Just ignore it and keep going
                        continue

                    # Save part no globally if present (for sorting)
                    self._part = _meta.get('part', 1)

                    # Create our binary instance
                    self.decoded = NNTPBinaryContent(
                        filepath=_meta['name'],
                        part=self._part,
                        work_dir=self.work_dir,
                    )

                elif _meta['key'] == 'part':

                    if 'begin' not in self._meta:
                        # we must have a begin if we have a part
                        # This is a messed up message; treat this
                        # as junk and keep going
                        continue

                    # Save part no globally if present (for sorting)
                    self._part = _meta.get('part', self._part)

                    # Update our Binary File if nessisary
                    self.decoded.part = self._part

                continue

            if len(set(('begin', 'part')) - set(self._meta)) == 2:
                # We haven't found the start yet which means we should just
                # keep going until we find it
                continue

            if FAST_YENC_SUPPORT:
                try:
                    decoded, self._crc, self._escape = \
                        decode_string(data, self._crc, self._escape)

                except YencError:
                    logger.warning(
                        "Yenc corruption detected on line %d." %
                        self._lines,
                    )

                    # Line Tracking
                    self._lines += 1

                    # keep storing our data
                    continue

            else:
                # The slow and painful way, the below looks complicated
                # but it really isn't at the the end of the day; yEnc is
                # pretty basic;
                #  - first we need to translate the special keyword tokens
                #    that are used by the yEnc language. We also want to
                #    ignore any trailing white space or new lines. This
                #    occurs by applying our DECODE_SPECIAL_MAP to the line
                #    being processed.
                #
                #  - finally we translate the remaining characters by taking
                #    away 42 from their value.
                #
                decoded = YENC_DECODE_SPECIAL_RE.sub(
                    lambda x: YENC_DECODE_SPECIAL_MAP[x.group()], data,
                ).translate(YENC42)

                # CRC Calculations
                self._calc_crc(decoded)

            # Line Tracking
            self._lines += 1

            # Track the number of bytes decoded
            self._decoded += len(decoded)

            # Write data to out stream
            self.decoded.write(decoded)

            if self._max_bytes > 0 and self._decoded >= self._max_bytes:
                # If we specified a limit and hit it then we're done at
                # this point. Before we do so; advance to the end of our
                # stream
                stream.seek(0, SEEK_END)

                # We're done
                break

        # Reset our meta tracking
        self._meta = {}

        # Reset part information
        self._part = 1

        if self.decoded:
            # close article when complete
            self.decoded.close()

        # Return what we do have
        return self.decoded

    def reset(self):
        """
        Reset our decoded content
        """
        super(CodecYenc, self).reset()

        # Tracks part no; defaults to 1 and shifts if it's determined
        # that we're another part
        self._part = 1

        # Used for internal meta tracking when using the decode()
        self._meta = {}

        # Our Binary Object we can reference while we decode
        # content
        self.decoded = None

    def __lt__(self, other):
        """
        Sorts by part number
        """
        return self._part < other._part

    def __str__(self):
        """
        Return a printable version of the file being read
        """

        # Build a string using the data we know
        if self.decoded:
            return str(self.decoded)

        if 'begin' in self._meta:
            fname = self._meta.get('name', 'Unknown.File')
        else:
            fname = 'Undetermined.File'

        return '%s' % (
            fname
        )

    def __repr__(self):
        """
        Return a printable object
        """
        return '<CodecYenc lines_processed=%d />' % (
            self._lines,
        ) NEWSREAP_ENGINE = '%s.engine' % NEWSREAP_LOGGER
import gevent.monkey
import re
import weakref
import logging
from zlib import decompressobj
from zlib import error as ZlibException
from os.path import isdir
from os.path import abspath
from os.path import expanduser
from io import BytesIO
from datetime import datetime
from blist import sortedset
from .NNTPHeader import NNTPHeader
from .NNTPArticle import NNTPArticle
from .NNTPResponse import NNTPResponse
from .NNTPResponse import NNTPResponseCode
from .NNTPContent import NNTPContent
from .NNTPSegmentedPost import NNTPSegmentedPost
from .NNTPMetaContent import NNTPMetaContent
from .NNTPFilterBase import NNTPFilterBase
from .NNTPIOStream import NNTPIOStream
from .NNTPIOStream import NNTP_SUPPORTED_IO_STREAMS
from .NNTPIOStream import NNTP_DEFAULT_ENCODING
from .SocketBase import SocketBase
from .SocketBase import SocketException
from .SocketBase import SocketRetryLimit
from .SocketBase import SignalCaughtException
from .Utils import mkdir
from .Utils import SEEK_SET
from .Utils import SEEK_END
from .NNTPnzb import NNTPnzb
from .NNTPSettings import DEFAULT_TMP_DIR
from .NNTPSettings import NNTP_EOL
from .NNTPSettings import NNTP_EOD
from .codecs.CodecBase import CodecBase
from .codecs.CodecHeader import CodecHeader
from .codecs.CodecArticleIndex import CodecArticleIndex
from .codecs.CodecArticleIndex import XoverGrouping
from .codecs.CodecGroups import CodecGroups
from .codecs.CodecUU import CodecUU
from .codecs.CodecYenc import CodecYenc
from newsreap.Logging import NEWSREAP_ENGINE
            # them into 1 file. We do this until we've processed all the
            # segments and we return a list of articles
            for segpost_no, segpost in enumerate(id):
                for article in iter(segpost):
                    _results = self.get(article)
                    if _results is not None:
                        # Append our results
                        results |= _results

        else:
            # Unsupported
            return None

        # Return our results
        return results

    def _get_wrapper(self, id, work_dir, decoders=None, group=None,
                     max_bytes=0):
        """
        A wrapper to the __get() function so that we can capture it
        with our hooks

        """
        # Initialize a bound object
        response = None
        try:
            weak_content = weakref.proxy(id)

        except TypeError:
            # Some types just can't be converted into a weak reference
            # no problem...
            weak_content = id

        try:
            _response = self.hooks.call(
                'pre_get',
                work_dir=work_dir,
                group=group,
                max_bytes=max_bytes,
                content=weak_content,
            )

            # Allow a response over-ride which forces a denied stat
            if next((r for r in _response
                     if r is not False), None) is not False:

                response = self._get(
                    id=id,
                    work_dir=work_dir,
                    group=group,
                    max_bytes=max_bytes,
                )

            else:
                response = NNTPResponse(
                    NNTPResponseCode.HOOK_OVERRIDE,
                    'NNTP GET action blocked by hook',
                )

        finally:
            # Prepare our status for our hook call
            status = False
            if isinstance(response, NNTPArticle):
                status = response.header.article_exists()

            try:
                weak_response = weakref.proxy(response)

            except TypeError:
                # Some types just can't be converted into a weak reference
                # no problem...
                weak_response = response

            self.hooks.call(
                'post_get',
                content=id,
                group=self.group_name,
                response=weak_response,
                status=status,
            )

        # Return our response
        return response

    def _get(self, id, work_dir, decoders=None, group=None, max_bytes=0):
        """
        Download a specified message to the work_dir specified. This function
        returns an NNTPArticle() object if it can.

        None is returned if the content could not be retrieved due to an error
        or DMCA.

        This function will always attempt to retrieve missing content from any
        identified backup servers in the order they were saved.

        max_bytes is set to the number of bytes you want to have received
        before you automatically abort the connection and flag the object as
        having only been partially complete  Use this option when you need to
        inspect the first bytes of a binary file. Set this to zero to download
        the entire thing (this is the default value)

        """

        if self.join_group and group is not None and group != self.group_name:
            # allow us to switch groups if nessisary
            if self.group(group)[0] is None:
                # Could not select group
                logger.error('Could not select group %s' % group)
                return None

        if not isdir(work_dir) and not mkdir(work_dir):
            logger.error('Could not create directory %s' % work_dir)
            return None

        if decoders is None:
            # Prepare a default list of decoders
            decoders = list()
            if not self.use_body:
                # BODY calls pull down header information too
                decoders.append(

                    CodecHeader(encoding=self.encoding, work_dir=work_dir),