class CodecBase(object):

    def __init__(self, work_dir=None, throttle_cycles=8000, throttle_time=0.2,
                 max_bytes=0, *args, **kwargs):
        """
        The dir identfies the directory to store our sessions in
        until they can be properly handled.

        Throttling is used to give a break to the thread and not consume all
        of the cpu usage while their are still other threads in play.

        max_bytes is set to the number of bytes you want to have received
        before you automatically abort the connection and flag the object as
        having only been partially complete  Use this option when you need to
        inspect the first bytes of a binary file. Set this to zero to download
        the entire thing (this is the default value)

        """

        # CRC Masking
        self._crc = BIN_MASK

        self._escape = 0

        # Track the number of bytes encoded
        self._encoded = 0

        # Track the number of bytes decoded
        self._decoded = 0

        # Initialize our byte tracking maximum threshold
        self._max_bytes = max_bytes

        # Our Decoded content should get placed here
        self.decoded = None

        if work_dir is None:
            self.work_dir = DEFAULT_TMP_DIR
        else:
            self.work_dir = abspath(expanduser(work_dir))

        if not isdir(self.work_dir):
            # create directory
            if mkdir(self.work_dir):
                logger.info('Created directory: %s' % self.work_dir)
            else:
                logger.error('Failed to created directory: %s' % self.work_dir)
                # Should not continue under this circumstance
                raise IOError((
                    errno.EACCES,
                    'Failed to create directory: %s' % self.work_dir,
                ))

        # Tracks the lines processed
        self._lines = 0

        # Tracks the lines scanned
        self._total_lines = 0

        # Throttle relief
        self.throttle_cycles = throttle_cycles
        self.throttle_time = throttle_time
        self.throttle_iter = 0

    def decode_loop(self):
        """
        Throttle system usage

        This call should exist within the while() loop of all decode() calls
        """
        # TODO: Generate some statistics here too based on lines processed

        self.throttle_iter += 1
        if ((self.throttle_cycles % self.throttle_iter) == 0):
            sleep(self.throttle_time)
        return True

    def _calc_crc(self, decoded):
        """
        Calculate the CRC based on the decoded content passed in
        """
        self._escape = crc32(decoded, self._escape)
        self._crc = (self._escape ^ -1)

    def max_bytes(self):
        """
        Allows one to wrap the return of the _max_bytes  if the wish
        in child classes.  Otherwise we just return the max_bytes

        """
        return self._max_bytes

    def parse_article(self, subject, *args, **kwargs):
        """
        Decode a article subject line if possible and return a dictionary
        of key -> value pair.

        The following arguments are allowed (these are the same results
        returned from the NNTPConnection.get() call.

            id: u'the unique identifier',
            article_no: 12345678,
            poster: u'the poster's information',
            date: datetime() object,
            subject: u'a subject line in unicode',
            size: 2135  // the message size in bytes
            lines: 53   // the number of lines
            group: u'alt.group.one'
            xgroups : {
                // references the Message-ID (id) per cross post
                 u'alt.group.two': 987654321,
                 u'alt.group.three': 12341234,
            }

        Always return None if you can't resolve the keys. Valid keys are:

        'fname' : Tranlates to the parsed filename (required!)
        'desc'  : Should always translate to the description (if there is one)

        """
        # Unless this object is overridden, nothing magical happens here
        return None

    def detect(self, line, relative=True):
        """
        A Simple function that can be used to determine if there is
        content on the line matches what the codec handles

        The relative flag should be used for doing additional internal
        checking with the line presented.  It's to be interpreted as
        relative to the decoder.

        For example if we get an 'end' token before receiving a 'begin' token,
        we may want to fail and ignore the data. This is what relative is for
        since it checks for a matched token relative to what we might expect.
        Without the relative switch the line is just processed (it's contents
        returned).

        It returns None if there is the codec doesn't match otherwise
        it returns a dictionary of the keys and their mapped values.

        """
        # Until this is over-ridden, it is always assumed the codec
        # can not handle the the line in question
        return None

    def decode(self, stream):
        """
        A function must be written that will read from the
        stream and decode the contents.

        decode should always return a type NNTPContent() or
        something that inherits from the class. There is
        an exception made with Header processing since it
        returns a type NNTPHeader().

        The function should return True when it is possible
        for more content to exist but you processed all that
        was available in the current stream. It tells the calling
        library to keep passing in more data as it's read next time.
        This the decode() function must be smart enough to handle
        partial content and be able to resume when more content
        is passed to it.

        Any other value returned (None, False, etc) will be
        treated as though the decoding flat out failed. It will be
        reset() so that it can process new content.

        """
        return False

    def encode(self, stream):
        """
        A function must be written that will read from the
        stream and encodes the contents it's provided.

        The function should return an NNTPContent object or None if
        it can't perform the encoding.
        """
        return None

    def crc32(self):
        """ Returns the calculated crc32 string for the decoded data.
        """
        return "%08x" % (self._crc ^ BIN_MASK)

    def reset(self, *args, **kwargs):
        """ Simply resets the internal variables of the class
            so that it can be re-used without being re-initialized
        """
        self._crc = BIN_MASK
        self._decoded = 0
        self._escape = 0

        # Tracks the lines processed
        self._lines = 0

        # Tracks the lines scanned
        self._total_lines = 0

        # Reset our Decoded Content
        self.decoded = None

        # Reset our throttle iterator
        self.throttle_iter = 0

        return None

    def __len__(self):
        """ Returns the total number of decoded bytes
        """
        return self._decoded

    def __str__(self):
        """
        Return a printable version of the codec
        """
        return repr(self)

    def __repr__(self):
        """
        Return a printable object
        """
        return '<CodecBase lines_processed=%d />' % (
            self._lines,
        ) class CodecYenc(CodecBase):

    def __init__(self, descriptor=None, work_dir=None,
                 linelen=128, *args, **kwargs):
        super(CodecYenc, self).__init__(
                descriptor=descriptor, work_dir=work_dir, *args, **kwargs)

        # Used for internal meta tracking when using the decode()
        self._meta = {}

        # Our Binary Object we can reference while we decode
        # content
        self.decoded = None

        # Used for encoding; This defines the maximum number of (encoded)
        # characters to display per line.
        self.linelen = linelen

    def parse_article(self, subject, *args, **kwargs):
        """
        Takes a an article header and returns it's parsed content if it's
        successful. Otherwise it returns None.
        """

        matched = NZB_SUBJECT_PARSE.match(subject)
        if matched is None:
            # subject is not parsable
            return None

        results = {}

        # Trim results
        if matched.group('desc') is not None:
            results['desc'] = re.sub('[\s-]+$', '', matched.group('desc'))
        if matched.group('fname') is not None:
            results['fname'] = matched.group('fname').strip()

        # Support conversion of integers
        for _attr in ['index', 'count', 'yindex', 'ycount', 'size']:
            if matched.group(_attr) is not None:
                results[_attr] = int(matched.group(_attr))

        return results

    def encode(self, content, mem_buf=DEFAULT_BUFFER_SIZE):
        """
        Encodes an NNTPContent object passed in
        """

        if isinstance(content, NNTPContent):
            # Create our ascii instance
            _encoded = NNTPAsciiContent(
                filepath=content.filename,
                part=content.part,
                total_parts=content.total_parts,
                sort_no=content.sort_no,
                work_dir=self.work_dir,
                # We want to ensure we're working with a unique attached file
                unique=True,
            )

        else:
            # If we reach here, we presume our content is a filename

            # Create our ascii instance
            _encoded = NNTPAsciiContent(
                filepath=content,
                work_dir=self.work_dir,
                # We want to ensure we're working with a unique attached file
                unique=True,
            )

            # Convert our content object into an NNTPContent object
            content = NNTPContent(
                filepath=content,
                work_dir=self.work_dir,
            )

        # yEnc (v1.3) begin
        fmt_ybegin = '=ybegin part=%d total=%d line=%d size=%d name=%s' % (
            content.part, content.total_parts, self.linelen,
            len(content), content.filename,
        )

        # yEnc part
        fmt_ypart = '=ypart begin=%d end=%d' % (
            content.begin() + 1,
            content.end(),
        )

        if isinstance(content._parent, NNTPContent):
            # yEnc end
            fmt_yend = '=yend size=%d part=%d pcrc32=%s crc32=%s' % (
                len(content), content.part,
                content.crc32(), content._parent.crc32(),
            )

        else:
            # yEnc end
            fmt_yend = '=yend size=%d part=%d pcrc32=%s' % (
                len(content), content.part, content.crc32(),
            )

        # Write =ybegin line
        _encoded.write(fmt_ybegin + EOL)
        # Write =ypart line
        _encoded.write(fmt_ypart + EOL)

        if not content.open():
            return None

        # Prepare our result set
        results = ""

        # Column is used for decoding
        column = 0
        crc = BIN_MASK

        # We need to parse the content until we either reach
        # the end of the file or get to an 'end' tag
        while True:
            # Read in our data
            data = content.stream.read(mem_buf)
            if not data:
                # We're done
                break

            if FAST_YENC_SUPPORT:
                try:
                    _results, crc, column = encode_string(data, crc, column)
                    # Append our parsed content onto our ongoing buffer
                    results += _results

                except YencError as e:
                    logger.error("Failed to encode Yenc for %s." % content)
                    logger.debug('Yenc exception: %s' % (str(e)))
                    return None

            else:
                # The slow and painful way, the below looks complicated
                # but it really isn't at the the end of the day; yEnc is
                # pretty basic;
                #  - first we translate the all of the characters by adding
                #    42 to their value with the exception of a few special
                #    characters that are explicitly reserved for the yEnc
                #    language (and conflict with the NNTP Server language).
                #
                #  - next, we need to apply our ENCODE_SPECIAL_MAP to be
                #    sure to handle the characters that are reserved as
                #    special keywords used by both NNTP Servers and the yEnc
                #    protocol itself.
                #
                #  - finally we want to prevent our string from going on for
                #    to many characters (horizontally).  So we need to split
                #    our content up
                #

                idx = 0
                while idx < len(data):
                    _byte = (ord(data[idx]) + 42) & 0xff
                    if _byte in YENC_ENCODE_ESCAPED_CHARACTERS:
                        _byte = (_byte + 64) & 0xff
                        # Esape Sequence
                        results += '='

                    # Store our character
                    results += chr(_byte)

                    # Increment Index
                    idx += 1

            # Our offset
            offset = 0

            while offset < (len(results)-self.linelen+1):
                eol = offset+self.linelen
                if results[offset:eol][-1] == '=':
                    # Lines can't end with the escape sequence (=). If we get
                    # here then this one did. We just adjust our end-of-line
                    # by 1 and keep moving
                    eol -= 1

                _encoded.write(results[offset:eol] + EOL)
                offset = eol

            if offset < len(results):
                results = results[-(len(results) - offset):]

            else:
                # reset string
                results = ''

        # We're done reading our data
        content.close()

        if len(results):
            # We still have content left in our buffer
            _encoded.write(results + EOL)

        # Write footer
        _encoded.write(fmt_yend + EOL)

        if _encoded:
            # close article when complete
            _encoded.close()

        # Return our encoded object
        return _encoded

    def detect(self, line, relative=True):
        """
        A Simple function that can be used to determine if there is
        yEnc content on the line being checked.

        If relative is set to true, we additionally check the line
        content against content relative to the decoding process (`What are
        we expecting to have right now?`). For example, the `end` token would
        be ignored if we haven't received a `begin` first.

        It returns None if there is no yEnc key line, otherwise
        it returns a dictionary of the keys and their mapped values.

        """
        yenc_re = YENC_RE.match(line)
        if not yenc_re:
            return None

        # Merge Results
        f_map = dict((YENC_KEY_MAP[k], v) for k, v
                     in yenc_re.groupdict().iteritems() if v)

        # Tidy filename (whitespace)
        if 'name' in f_map:
            f_map['name'] = basename(f_map['name']).strip()

        if relative:
            # detect() relative to what has been decoded
            if f_map['key'] in self._meta:
                # We already processed this key
                return None

            if f_map['key'] == 'end' and 'begin' not in self._meta:
                # We can't handle this key
                return None

            if f_map['key'] == 'part' and 'begin' not in self._meta:
                # We can't handle this key
                return None

        # Integer types
        for kw in ['line', 'size', 'total', 'begin', 'end', 'part']:
            if kw in f_map:
                try:
                    f_map[kw] = int(f_map[kw])

                except (TypeError, ValueError):
                    # Eliminate bad kw
                    del f_map[kw]

        return f_map

    def decode(self, stream):
        """ Decode some data and decode the data
            to descriptor identified (by the stream)
        """

        # We need to parse the content until we either reach
        # the end of the file or get to an 'end' tag
        while self.decode_loop():
            # fall_back ptr
            ptr = stream.tell()

            # Read in our data
            data = stream.readline()
            if not data:
                # We're done for now
                return True

            # Total Line Tracking
            self._total_lines += 1

            # Detect a yEnc line
            _meta = self.detect(data, relative=False)
            if _meta is not None:
                #
                # We just read a yEnc keyword token such as
                # begin, part, or end
                #
                if _meta['key'] in self._meta:
                    # We already processed this key; uh oh
                    # Fix our stream
                    stream.seek(ptr, SEEK_SET)

                    # Fix our line count
                    self._total_lines -= 1

                    # We're done
                    break

                if _meta['key'] == 'end' and \
                   len(set(('begin', 'part')) - set(self._meta)) == 2:
                    # Why did we get an end before a begin or part?
                    # Just ignore it and keep going
                    continue

                # store our key
                self._meta[_meta['key']] = _meta

                if 'end' in self._meta:
                    # Mark the binary as being valid
                    self.decoded._is_valid = True

                    # We're done!
                    break

                elif _meta['key'] == 'begin':
                    # Depending on the version of yEnc we're using binary
                    # content starts now; thefore we create our binary
                    # instance now

                    if 'name' not in _meta:
                        # Why did we get a begin before a part
                        # Just ignore it and keep going
                        continue

                    # Save part no globally if present (for sorting)
                    self._part = _meta.get('part', 1)

                    # Create our binary instance
                    self.decoded = NNTPBinaryContent(
                        filepath=_meta['name'],
                        part=self._part,
                        work_dir=self.work_dir,
                    )

                elif _meta['key'] == 'part':

                    if 'begin' not in self._meta:
                        # we must have a begin if we have a part
                        # This is a messed up message; treat this
                        # as junk and keep going
                        continue

                    # Save part no globally if present (for sorting)
                    self._part = _meta.get('part', self._part)

                    # Update our Binary File if nessisary
                    self.decoded.part = self._part

                continue

            if len(set(('begin', 'part')) - set(self._meta)) == 2:
                # We haven't found the start yet which means we should just
                # keep going until we find it
                continue

            if FAST_YENC_SUPPORT:
                try:
                    decoded, self._crc, self._escape = \
                        decode_string(data, self._crc, self._escape)

                except YencError:
                    logger.warning(
                        "Yenc corruption detected on line %d." %
                        self._lines,
                    )

                    # Line Tracking
                    self._lines += 1

                    # keep storing our data
                    continue

            else:
                # The slow and painful way, the below looks complicated
                # but it really isn't at the the end of the day; yEnc is
                # pretty basic;
                #  - first we need to translate the special keyword tokens
                #    that are used by the yEnc language. We also want to
                #    ignore any trailing white space or new lines. This
                #    occurs by applying our DECODE_SPECIAL_MAP to the line
                #    being processed.
                #
                #  - finally we translate the remaining characters by taking
                #    away 42 from their value.
                #
                decoded = YENC_DECODE_SPECIAL_RE.sub(
                    lambda x: YENC_DECODE_SPECIAL_MAP[x.group()], data,
                ).translate(YENC42)

                # CRC Calculations
                self._calc_crc(decoded)

            # Line Tracking
            self._lines += 1

            # Track the number of bytes decoded
            self._decoded += len(decoded)

            # Write data to out stream
            self.decoded.write(decoded)

            if self._max_bytes > 0 and self._decoded >= self._max_bytes:
                # If we specified a limit and hit it then we're done at
                # this point. Before we do so; advance to the end of our
                # stream
                stream.seek(0, SEEK_END)

                # We're done
                break

        # Reset our meta tracking
        self._meta = {}

        # Reset part information
        self._part = 1

        if self.decoded:
            # close article when complete
            self.decoded.close()

        # Return what we do have
        return self.decoded

    def reset(self):
        """
        Reset our decoded content
        """
        super(CodecYenc, self).reset()

        # Tracks part no; defaults to 1 and shifts if it's determined
        # that we're another part
        self._part = 1

        # Used for internal meta tracking when using the decode()
        self._meta = {}

        # Our Binary Object we can reference while we decode
        # content
        self.decoded = None

    def __lt__(self, other):
        """
        Sorts by part number
        """
        return self._part < other._part

    def __str__(self):
        """
        Return a printable version of the file being read
        """

        # Build a string using the data we know
        if self.decoded:
            return str(self.decoded)

        if 'begin' in self._meta:
            fname = self._meta.get('name', 'Unknown.File')
        else:
            fname = 'Undetermined.File'

        return '%s' % (
            fname
        )

    def __repr__(self):
        """
        Return a printable object
        """
        return '<CodecYenc lines_processed=%d />' % (
            self._lines,
        ) DEFAULT_TMP_DIR = expanduser(join('~', '.config', 'newsreap', 'var', 'tmp')) class NNTPGroup(object):
    """
    An object for manipulating and looking up group information.

    It handles short-form references such as a.b can be transformed
    to alt.binaries., etc.

    """

    # used for translation lookups
    _translations = None

    def __init__(self, name, *args, **kwargs):
        """
        Initialize NNTP Group

        """
        # The Group Name
        self.name = NNTPGroup.normalize(name)

        if self.name is None:
            raise AttributeError(
                "Invalid group {} set specified.".format(name))

    @staticmethod
    def normalize(group, shorthand=True):
        """
        takes a group identifier and normalizes it. For example:
        a.b.test           returns alt.binaries.test
        a.b.pictures.test  returns alt.binaries.pictures.test
        A.Binaries.test    returns alt.binaries.test

        items that can be normalized or are of an invalid group causes the
        function to return None.

        """

        if isinstance(group, NNTPGroup):
            # Support passing in ourselves
            return group.name

        try:
            group = GROUP_INVALID_CHAR_RE.sub('', group).lower()

        except (AttributeError, TypeError):
            # Invalid content passed in
            pass

        if not group:
            return None

        if not shorthand:
            return group

        # If we reach here, we want to additionally try to look up any
        # shorthand notation and expand it
        entries = re.split('[.]+', group)

        if NNTPGroup._translations is None:
            # Populate it
            line_re = re.compile(
                r'^(?P<index>\d)\s+'
                r'(?P<key>[a-z0-9.-]+)\s+'
                r'(?P<lookup>[a-z0-9.-]+)', re.I)

            NNTPGroup._translations = list()
            path = join(dirname(abspath(__file__)), 'var', 'groups.dat')
            with open(path) as fd:
                for line in fd:
                    result = line_re.match(line)
                    if not result:
                        continue

                    index = int(result.group('index'))

                    while len(NNTPGroup._translations) < index+1:
                        NNTPGroup._translations.append({})

                    # store our lookup
                    NNTPGroup._translations[index][result.group('key')] = \
                        result.group('lookup')

        # Iterate over our names
        for depth in range(len(NNTPGroup._translations)):
            if depth >= len(entries):
                # we're done
                break

            # Perform our lookup and translate if we can
            match = NNTPGroup._translations[depth].get(entries[depth])
            if not match:
                # We can't skip some entries and translate others, the chain
                # is broken on our first-mismatch
                break

            # Store our match and keep going
            entries[depth] = match

        # Return our compiled list
        return '.'.join(entries)

    @staticmethod
    def split(groups):
        """
        Takes a string containing multiple groups and returns a normalized
        group set()
        """

        # Initialize our return result set
        result = set()

        if isinstance(groups, basestring):
            groups = GROUP_INVALID_CHAR_RE.split(groups)

        if isinstance(groups, (set, list, tuple)):
            for group in groups:
                if isinstance(group, NNTPGroup):
                    result.add(group)

                elif isinstance(group, basestring):
                    try:
                        result.add(NNTPGroup(group))

                    except AttributeError:
                        pass

                elif isinstance(group, (set, list, tuple)):
                    # a little bit of recursion
                    result |= NNTPGroup.split(group)

        return result

    def shorthand(self):
        """
        Returns the shorthand version of the group
        """

        # TODO
        return self.name

    def __str__(self):
        """
        Return a printable version of the article
        """
        return '%s' % self.name

    def __unicode__(self):
        """
        Return a printable version of the article
        """
        return u'%s' % self.name

    def __lt__(self, other):
        """
        Handles group sorting while sticking 'None' type records (if they
        exist at the end)
        """
        return self.name < other.name

    def __eq__(self, other):
        """
        Handles equality

        """
        if isinstance(other, NNTPGroup):
            return self.name == other.name

        elif isinstance(other, basestring):
            return self.name == NNTPGroup.normalize(other)

        return False

    def __hash__(self):
        """
        allows us to make use of the 'in' keyword. Hence this object can
        reside within a set, and you can still type:
        if 'alt.binaries.test' in set(NNTPGroup(), NNTPGroup())
        """
        return hash(self.name)

    def __len__(self):
        """
        Returns the length of our group
        """
        return len(self.name)

    def __repr__(self):
        """
        Return an unambigious version of the object
        """
        return '<NNTPGroup name="%s" />' % (
            self.name,
        ) class NNTPArticle(object):
    """
    An object for maintaining retrieved article content. The primary
    reason for this object is it makes it easier to sort the results
    in memory tracking both the content downloaded (using the
    NNTPConnection.get() call along with the file stored locally on disk
    that it's responsible for.

    This function does it's best to behave like a stream. But provides
    some functions to make manipulating and merging with other articles
    easier to do.

    Articles by default assume a roll of 'attached'.  This means that
    the files written to disk are removed the second the object is
    destroyed.  This is intentional!  You can call detach() at any
    time you want but now you are responsible for cleaning up the
    filename.

    """

    def __init__(self, id=None, subject=None, poster=None, groups=None,
                 work_dir=None, body=None, codecs=None, *args, **kwargs):
        """
        Initialize NNTP Article

        """

        # The Article Message-ID
        self.id = id if isinstance(id, basestring) else ''

        # The NNTP Group Index #
        self.no = 1000
        try:
            self.no = int(kwargs.get(u'no', self.no))
        except:
            # Just use the default
            pass

        # The Subject
        self.subject = subject
        if self.subject is None:
            self.subject = DEFAULT_NNTP_SUBJECT

        # The Poster
        self.poster = poster
        if self.poster is None:
            self.poster = DEFAULT_NNTP_POSTER

        if work_dir is None:
            self.work_dir = DEFAULT_TMP_DIR
        else:
            self.work_dir = abspath(expanduser(work_dir))

        # Contains a list of decoded content; it's effectively the articles
        # attachments
        self.decoded = sortedset(key=lambda x: x.key())

        # The group(s) associated with our article
        self.groups = NNTPGroup.split(groups)

        # A hash of header entries
        self.header = NNTPHeader(work_dir=self.work_dir)

        if isinstance(body, NNTPAsciiContent):
            self.body = body
        else:
            # Our body contains non-decoded content
            self.body = NNTPAsciiContent(work_dir=self.work_dir)

        if isinstance(body, basestring) and len(body) > 0:
            # Store our body content
            self.body.write(body)

        # Load our Codecs
        self._codecs = codecs

        # Provide default codecs if required
        if not self._codecs:
            self._codecs = [CodecYenc(), ]

        elif isinstance(self._codecs, CodecBase):
            self._codecs = [self._codecs, ]

    def load(self, response):
        """
        Loads an article by it's NNTPResponse or from another NNTPArticle

        """
        if isinstance(response, NNTPResponse):
            # Our body contains non-decoded content
            self.body = response.body

            # Store decoded content
            self.decoded = response.decoded

            # Store Header
            self.header = next(
                (d for d in self.decoded if isinstance(d, NNTPHeader)), None)

            # Our groups associated with the post (if we know it)
            self.groups = set()

            if self.header is not None:
                # Remove Header from decoded list
                self.decoded.remove(self.header)

                if u'Newsgroups' in self.header:
                    # Parse our groups out of the header
                    self.groups = NNTPGroup.split(self.header[u'Newsgroups'])

        elif isinstance(response, NNTPArticle):
            # We basically save everything except the work-dir since
            # our new work dir could be a new location

            # The NNTP Message-ID
            self.id = response.id

            # The NNTP Group Index
            self.no = response.no

            # Store the subject
            self.subject = response.subject

            # Store the poster
            self.poster = response.poster

            # The NNTP Groups
            self.groups = response.groups

            # Store the header from the other side
            self.header = response.header

            # Our body contains non-decoded content
            self.body = response.body

            # Store decoded content
            self.decoded = response.decoded

        else:
            # Unsupported
            return False

        return True

    def post_iter(self, update_headers=True):
        """
        Returns NNTP string as it would be required for posting
        """

        if not self.id:
            # Generate ourselves a Message-ID
            self.msgid()

        if not len(self.body) and not len(self.decoded):
            # Nothing to post
            logger.error(
                'Post Denied / Article %s has no content to post.' %
                self.msgid())
            return None

        if not len(self.groups):
            # Nothing to post
            logger.error(
                'Post Denied / Article %s has no groups defined.' %
                self.msgid())
            return None

        try:
            if not self.subject.strip():
                logger.error(
                    'Post Denied / Article %s has no subject.' %
                    self.msgid())
                return None

        except AttributeError:
            logger.error(
                'Post Denied / Article %s has no subject.' % self.msgid())
            return None

        try:
            if not self.poster.strip():
                logger.error(
                    'Post Denied / Article %s has no poster.' % self.msgid())
                return None

        except AttributeError:
            logger.error(
                'Post Denied /Article %s has no poster.' % self.msgid())
            return None

        if update_headers:
            self.update_headers()

        args = [
            iter(self.header.post_iter()),
        ]

        if len(self.body):
            args.append(iter(self.body.post_iter()))
            args.append(NNTP_EOL)

        if len(self.decoded):
            for entry in self.decoded:
                args.append(iter(entry.post_iter()))
                args.append(NNTP_EOL)

        return chain(*args)

    def encode(self, encoders):
        """
        A wrapper to the encoding of content. The function returns None if
        a problem occurs, otherwise the function returns an NNTPArtice()
        object.

        The power of this function comes from the fact you can pass in
        multiple encoders to have them all fire after one another.
        """
        if len(self) == 0:
            # Nothing to encode
            return None

        objs = sortedset(key=lambda x: x.key())
        for content in self:
            obj = content.encode(encoders)
            if obj is None:
                return None

            # Successful, add our object to our new list
            objs.add(obj)

        # If we reach here we encoded our entire article
        # Create a copy of our article
        article = self.copy(include_attachments=False)

        # In our new copy; store our new encoded content
        article.decoded = objs

        # Return our article
        return article

    def split(self, size=81920, mem_buf=1048576, body_mirror=0):
        """
        Split returns a set of NNTPArticle() objects containing the split
        version of the data it already represents.

        Even if the object can't be split any further given the parameters, a
        set of at least 1 entry will always be returned.  None is returned if
        an error occurs. None is also returned if split() is called while there
        is more then one NNTPContent objects since it makes the situation
        Ambiguous.

        The body_mirror flag has a series of meanings; since we start with a
        single post before calling this (which goes on and splits the post) we
        have to decide if we want the contents of our message body to appear
        in every post. So here is how it breaks down.

        If you set it to False, None then the body content is ignored
        and nothing is done with it.

        If you set it to True, then the body is copied to every single part.

        If you set it to -X (negative #) then the body is stored in the
        part generated as if you called the python slice expression:
            parts[-X].body <-- store body here

        If you set it to X (positive #), then the body is stored in
        that index value similar to above:
            parts[X].body <-- store body here

        IndexErrors are automatially and silently handled.  If you access
        a part that is out of bounds then the bounds are limited to fit
        automtatically.  For instance, if you specified -100 as the
        part and there were only 4 parts generated, (-4 is the maximum
        it could have been) then -4 is used in place of the -100.

        consiquently if you passed in 100 (a positive value). since
        (positive) indexes are measured started at 0 (zero), this means a
        with a list of 4 items the largest value can be a 3.  Therefore
        the index of 3 is used instead.
        """
        if len(self) > 1:
            # Ambiguous
            return None

        if len(self) == 0:
            # Nothing to split
            return None

        content = next((
            c for c in self.decoded if isinstance(c, NNTPContent)), None)

        if not content:
            # Well this isn't good; we have decoded entries that are not
            # of type NNTPContent.
            return None

        # ensure our size is set to some value
        size = strsize_to_bytes(size)
        if size is None:
            # You can't call split() with a size value set at None
            return None

        # Size needs to consider our message body
        # The length of our headers are not applicable
        if len(self.body):
            size -= len(self.body) + len(NNTP_EOL)

        if size < 0:
            # we can't perform the split
            return None

        # Split our content
        new_content = content.split(size=size, mem_buf=mem_buf)

        if new_content is None:
            # something bad happened
            return None

        # If we get here, we have content to work with.  We need to generate
        # a list of articles based on our existing one.
        articles = sortedset(key=lambda x: x.key())

        for no, c in enumerate(new_content):
            a = NNTPArticle(
                subject=self.subject,
                poster=self.poster,
                groups=self.groups,
                # Increment our index #
                no=self.no + no,
                work_dir=self.work_dir,
            )

            # Set our header to be a copy of what we already have
            a.header = deepcopy(self.header)

            # Store our NNTPContent() object
            a.add(c)

            if body_mirror is True:
                # body is mirrored to everything
                a.body = self.body

            # Store our Article
            articles.add(a)

        # Now we transfer over our body if nessisary
        if isinstance(body_mirror, int):
            if body_mirror > -1:
                try:
                    articles[body_mirror].body = self.body

                except IndexError:
                    # Store at last entry
                    articles[-1].body = self.body
            else:
                # Negative Number
                try:
                    articles[body_mirror].body = self.body

                except IndexError:
                    # Store at first entry
                    articles[0].body = self.body

        # Return our articles
        return articles

    def append(self, articles):
        """
        Appends one article onto another. This is the reverse of split()
        articles must be an NNTPArticle object, or a set-of NNTPArticle()
        objects.

        """
        if isinstance(articles, NNTPArticle):
            # Convert our entry into a list object
            articles = [articles, ]

        # Change to an iterator
        articles = iter(articles)
        if len(self) > 1:
            # Ambiguous; we can't append content if we have
            # more then one possible thing to append to
            return False

        for article in articles:
            if len(article) == 0:
                # No attachments; move along silently
                continue

            elif len(article) > 1:
                # To many attachments in article
                return False

            if len(self.decoded) == 0:
                # Store a copy of our decoded object
                content = article[0].copy()
                if content is None:
                    # We failed to create a copy
                    return False

                # Add our content to our object
                self.decoded.add(content)
                continue

            # If we reach here, we have 1 entry to append our content to
            if not self.decoded[0].append(article[0]):
                return False

        # Return our success
        return True

    def copy(self, include_attachments=True):
        """
        Creates a copy of the article returning one

        """

        article = NNTPArticle(
            subject=self.subject,
            poster=self.poster,
            groups=self.groups,
        )

        article.id = self.id
        article.no = self.no
        article.work_dir = self.work_dir
        article.header = self.header.copy()
        article.body = self.body.copy()

        if include_attachments:
            for content in self.decoded:
                obj = content.copy()
                if obj is None:
                    return None

                # Add our copied NNTP object
                article.decoded.add(obj)

        # Return our copy
        return article

    def is_valid(self):
        """
        Iterates over article content an returns True if all of it is valid
        based on it's crc32/md5 and/or whatever mechanism is used to determine
        a NNTPContents validity
        """
        if len(self.decoded) == 0:
            # No articles means no validity
            return False

        return next((
            False for c in self.decoded if c.is_valid() is False), True)

    def files(self):
        """
        Returns a list of the files within article
        """
        return [x.path() for x in self.decoded]

    def key(self):
        """
        Returns a key that can be used for sorting with:
            lambda x : x.key()
        """
        return '%.5d%s' % (self.no, self.id)

    def detach(self):
        """
        Detach the article stored on disk from being further managed by this
        class
        """
        for a in self.decoded:
            if isinstance(a, NNTPBinaryContent):
                a.detach()
        return

    def attach(self):
        """
        Detach the article stored on disk from being further managed by this
        class
        """
        for a in self.decoded:
            if isinstance(a, NNTPBinaryContent):
                a.attach()
        return

    def add(self, content):
        """
        Used for adding content to the self.decoded class
        """
        if isinstance(content, basestring):
            # A mime object we can use to detect the type of file
            m = Mime()

            if isfile(content):
                # file relative to current path; get our mime response
                mr = m.from_bestguess(content)

                # Our NNTPContent object will depend on whether or not we're
                # dealing with an ascii file or binary
                instance = NNTPBinaryContent \
                    if mr.is_binary() else NNTPAsciiContent

                content = instance(
                    filepath=content,
                    work_dir=self.work_dir,
                )

            else:
                # Not relative; is the file relative to the work_dir?
                with pushd(self.work_dir, create_if_missing=True):
                    if isfile(content):

                        # file relative to current path; get our mime response
                        mr = m.from_bestguess(content)

                        # Our NNTPContent object will depend on whether or not
                        # we're dealing with an ascii file or binary
                        instance = NNTPBinaryContent \
                            if mr.is_binary() else NNTPAsciiContent

                        # load our file
                        content = instance(
                            filepath=content,
                            work_dir=self.work_dir,
                        )

                    else:
                        # we're dealing with a new file; just save what we have
                        content = NNTPBinaryContent(
                            filepath=content,
                            work_dir=self.work_dir,
                        )

        if not isinstance(content, NNTPContent):
            return False

        # duplicates are ignored in a blist and therefore
        # we just capture the length of our list before
        # and after so that we can properly return a True/False
        # value
        _bcnt = len(self.decoded)
        self.decoded.add(content)

        return len(self.decoded) > _bcnt

    def msgid(self, host=None, reset=False):
        """
        Returns a message ID; if one hasn't been generated yet, it is
        based on the content in the article and used

        If reset is set to True, then a new ID is generated
        """
        if not reset and self.id:
            return self.id

        if not host:
            # Generate a 32-bit string we can use
            host = random_str(32)

        if len(self.decoded):
            partno = self.decoded[0].part
        else:
            partno = 1

        m = hashlib.sha1()
        reference = datetime.utcnow()
        m.update(datetime.utcnow().strftime('%Y%m%d%H%M%S%f'))
        key = m.hexdigest()

        # If we reach here an ID hasn't been generated yet; generate one
        self.id = '%s%x.%x!%x%x%x%x%x@%s%s%s' % (
            key[0:5],
            reference.second,
            reference.microsecond,
            reference.year,
            reference.month,
            reference.day,
            reference.minute,
            partno,
            reference.hour,
            host,
            key[5:10],
        )

        # Then return it
        return self.id

    def save(self, filepath=None, copy=False):
        """
        Saves all of the NNTPContent into the directory specified by
        the filepath; if no filepath is specified, then content is written
        into it's work_dir
        """

        if filepath:
            if not isdir(filepath) and not mkdir(filepath):
                # We failed
                return False

        for attachment in self.decoded:
            if not attachment.save(filepath=filepath, copy=copy):
                return False

        return True

    def deobsfucate(self, filebase='', codecs=None):
        """
        Using the information we have in the article generate a filename to
        the best of our ability.

        filebase: provide a fallback filename base (the part of the file
                  before the extension) to build on if we can't detect
                  the file on our own.

        codecs:   The codec(s) you wish to use to assist in the deobsfucation
        """

        if len(self.decoded) != 1:
            # There is ambiguity if there is not one attachment
            return None

        if filebase is None:
            # Safety
            filebase = ''

        # The detected article filename and extension
        a_name = ''
        a_fext = ''
        a_mime = None

        # The detected attachment filename and extension
        d_name = ''
        d_fext = ''
        d_mime = None

        # Create our Mime object since we'll be using it a lot
        m = Mime()

        if codecs is None:
            codecs = self._codecs

        elif isinstance(codecs, CodecBase):
            codecs = [codecs, ]

        # Use our Codec(s) to extract our Yenc Subject
        matched = None
        for c in codecs:
            # Use our Defined Codec(s) to extract content from our subject
            matched = c.parse_article(
                id=self.id,
                article_no=self.no,
                subject=self.subject,
                poster=self.poster,
            )

            if matched:
                # We succesfully got a filename from our subject line
                a_fname = matched.get('fname', '').strip()
                if a_fname:
                    # always allow the name to over-ride the detected filename
                    # if we actually have a real name we can assciated with it
                    # by
                    a_fext = m.extension_from_filename(a_fname)
                    a_name = a_fname[:-len(a_fext)]
                    a_mime = m.from_filename(a_fname)
                    if a_mime and a_mime.type() == DEFAULT_MIME_TYPE:
                        a_mime = None

                # We can break out
                break

        ####################################
        # Now we want to scan our attachment
        ####################################

        # Our mime object for our attachment
        d_mime = self.decoded[0].mime()
        d_fname = self.decoded[0].filename if self.decoded[0].filename \
            else basename(self.decoded[0].path())
        d_mime = m.from_bestguess(d_fname)
        if d_mime and d_mime.type() == DEFAULT_MIME_TYPE:
            d_mime = None

        d_fext = m.extension_from_filename(d_fname)
        d_name = d_fname[:-len(d_fext)]

        # At this point we have to make a decision to go with the filename
        # pulled from our content object, or go with the filename detected
        # from the article.

        _name = filebase
        _fext = None

        # Using the mime types, we decide which one is more likely the name
        if a_mime is None:
            # Attachment was not parsable
            if d_mime is not None:
                # However we did detect a file through this method
                _fext = d_fext
                _name = filebase if filebase else d_name

        elif d_mime is not None:
            # Attachment is parsable and so is our attachment
            if d_mime == a_mime:
                # Same type
                _fext = d_fext if d_fext else a_fext
                _name = _name if _name else d_name

            else:
                # We need to choose one over the other
                if len(d_fext):
                    # we have a filesize associated with our content which
                    # means there is a good chance we parsed our mime typefrom
                    # here
                    _fext = d_fext if d_fext else d_mime.extension()
                else:
                    # Use whatever we parsed
                    _fext = a_fext if a_fext else a_mime.extension()

                _name = _name if _name else d_name

        else:
            # Attachment is parsable and our Article isn't, easy-peasy
            # just use the content
            _fext = a_fext if a_fext else a_mime.extension()
            _name = filebase if filebase else a_name

        if not _fext:
            # Last resort
            _fext = d_fext if d_fext else a_fext
            if not _fext:
                return None

        if not _name:
            # Last resort
            _name = d_name if d_name else a_name
            if not _name:
                return None

        # Return our assembled identifier
        return "{0}{1}".format(_name, _fext)

    def size(self):
        """
        return the total size of our decoded content; we factor in the body if
        it exists.
        """
        if len(self.body):
            return sum(len(d) for d in self.decoded) + \
                len(self.body) + len(NNTP_EOL)
        return sum(len(d) for d in self.decoded)

    def strsize(self):
        """
        return the total size of our decoded content in a human readable
        string.
        """
        return bytes_to_strsize(self.size())

    def update_headers(self):
        """
        Updates our header information based on specified content
        """
        # Our Newsgroups can be of type NNTPGroup, but it can also just
        # be a plain string 'alt.binaries.test', etc. to support both
        # types we need to do a loop inside of our join below
        self.header['Newsgroups'] = ','.join([str(x) for x in self.groups])
        self.header['Subject'] = self.subject.strip()
        self.header['From'] = self.poster.strip()
        self.header['X-Newsposter'] = '%s v%s' % (__title__, __version__)
        self.header['Message-ID'] = '<%s>' % self.msgid()

    def __iter__(self):
        """
        Grants usage of the next()
        """
        # Ensure our stream is open with read
        return iter(self.decoded)

    def __len__(self):
        """
        Returns the number of decoded content entries found
        """
        return len(self.decoded)

    def __lt__(self, other):
        """
        Handles less than for storing in btrees
        """
        return self.key() < other.key()

    def __eq__(self, other):
        """
        Handles equality

        """
        return self.__dict__ == other.__dict__

    def __getitem__(self, index):
        """
        Support accessing NNTPContent objects by index
        """
        return self.decoded[index]

    def __str__(self):
        """
        Return a printable version of the article
        """
        return '%s' % self.id

    def __unicode__(self):
        """
        Return a printable version of the article
        """
        return u'%s' % self.id

    def __repr__(self):
        """
        Return an unambigious version of the object
        """

        return '<NNTPArticle Message-ID="%s" attachments="%d" />' % (
            self.id,
            len(self.decoded),
        ) DEFAULT_NNTP_SUBJECT = 'unknown.file' DEFAULT_NNTP_POSTER = 'newsreaper <news@reap.er>' class NNTPContent(object):
    """
    An object for maintaining retrieved article content. There can only
    be 1 article, however that 1 article can have a lot of content
    found within it.

    This identifies the content found internally within an article.

    This function does it's best to behave like a stream. But provides
    some functions to make manipulating and merging with other articles
    easier to do.

    Articles by default assume a roll of 'attached'.  This means that
    the files written to disk are removed the second the object is
    destroyed.  This is intentional!  You can call detach() at any
    time you want but now you are responsible for cleaning up the
    filename.

    """

    def __init__(self, filepath=None, part=None, total_parts=None,
                 begin=None, end=None, total_size=None, work_dir=None,
                 sort_no=10000, unique=False, *args, **kwargs):
        """
        Initialize NNTP Content

        If a filepath is specified, it can be either a stream (already opened
        file or ByteIO or StringIO class is fine too), or it can be a path to
        a filename which will be open in 'wb' mode.

        if unique is set to True, then if the file passed in already exists,
        a temporary file is used instead. But the filename info isn't lost.

        Set the unique to True when you don't want to accidently over-write
        or alter a file you may already be working with.
        """

        # The sort is used with sorting; different filetypes/content types
        # should be processed before otheres.

        # For example, the NNTPHeader() and NNTPMetaContent() area always kept
        # at the head where as NNTPAsciiContent() and NNTPBinaryContent() are
        # kept at the back.

        # The lower sort no is always processe first; but default we choose
        # a rather large sort value. Grouped content should share the same sort
        # value so that they sort their content together.
        self.sort_no = sort_no

        # A _unique string is used to ensure our file is always different
        # from another
        self._unique = False
        if unique:
            # Convert to string for indexing speed in key() calls since
            # the index is already in string format
            self._unique = str(id(self))

        # Default filename
        self.filename = ''

        # The filepath is automatically set up when the temporary file is
        # created
        self.filepath = None

        # used to track the filemode (saves on time from opening and closing
        # un-nessisarily).  These flags are set during an open and a close
        self.filemode = None

        # Prepare temporary folder we intend to use by default
        if work_dir:
            self.work_dir = abspath(expanduser(work_dir))
        else:
            self.work_dir = DEFAULT_TMP_DIR

        # A Stream object
        self.stream = None

        # Detached prevents the article from cleaning up all of
        # the data it otherwise tracks (such as the article stored
        # on disk)
        #
        # If set to None, then it hasn't been initalized yet
        self._detached = None

        # Dirty flag is set to true if a write is made; all data
        # is considered dirty until flush() is called (forcing
        # contents out of cache and onto disk)
        self._dirty = False

        # A flag that can be toggled if the data stored is
        # corrupted in some way. Such as through CRC Failing
        # or part construction (a part missing perhaps, etc)
        # if all is good, then we just leave the flag as is
        self._is_valid = False

        # Store part
        self.part = 1
        if part is not None:
            try:
                self.part = int(part)

            except (ValueError, TypeError):
                raise AttributeError(
                    "Invalid part specified (%s)." %
                    str(part),
                )

        # Tracks parts (most used for posting/encoding)
        self.total_parts = 1
        if total_parts is not None:
            try:
                self.total_parts = int(total_parts)

            except (ValueError, TypeError):
                raise AttributeError(
                    "Invalid total_parts specified (%s)." %
                    str(total_parts),
                )

            if self.total_parts < self.part:
                raise AttributeError(
                    "Invalid parts/total_parts specified (%s/%s)." % (
                        str(part), str(total_parts),
                    )
                )
        else:
            self.total_parts = self.part

        # Used for tracking the indexes (head/tail) that make
        # up the block of data this NNTPContent object represents.
        # this is only used if split() is called
        self._begin = 0
        if begin is not None:
            try:
                self._begin = int(begin)

            except (ValueError, TypeError):
                raise AttributeError(
                    "Invalid begin specified (%s)." % str(begin),
                )

        self._end = None
        if end is not None:
            try:
                self._end = int(end)

            except (ValueError, TypeError):
                self._end = None

        # Tracks total_size (used for posting when we have parts of files
        self._total_size = None
        if total_size is not None:
            try:
                self._total_size = int(total_size)

            except (ValueError, TypeError):
                raise AttributeError(
                    "Invalid total_size specified (%s)." %
                    str(total_size),
                )

        # Set blocksize as a variable for those who want to tweak it
        # later on.  This controls the maximum amount of content read
        # from a file in one chunk (thus how much memory will be occupied
        # when called).  Setting this too high will utilize a lot of
        # memory if concurrent calls are made but will be faster if your
        # system can handle it.
        self._block_size = BLOCK_SIZE

        # Reserved for split() function which populates any children
        # spawned from this object with a pointer back to this as
        # object as a reference.
        self._parent = None

        # TODO: all md5, crc32, len() calls etc should all cache their results
        # here and retrieve from here if present. If a write() or load() is
        # made, the cache should be destroyed. The cache is only populated on
        # demand.
        self._lazy_cache = {}

        # NNTPContent supports directory storing too. This is toggle in the
        # event we're dealing with a directory
        self._isdir = False

        if not filepath:
            # Will use load() or open() which causes temp file
            # to be created
            return

        elif isinstance(filepath, file):

            # store stream
            self.stream = filepath

            # Always detach streams
            self._detached = True

            if hasattr('name', self.stream):
                self.filepath = abspath(self.stream.name)
                self.filename = basename(filepath)

            if hasattr('mode', self.stream):
                # Store our mode
                self.filemode = self.stream.mode
        else:
            if self._unique is False and isfile(filepath):
                self.load(filepath, sort_no=sort_no)

            elif isdir(filepath):
                # Toggle Flag
                self._isdir = True
                # Directories are always detached
                self._detached = True
                # Store our dirname
                self.filename = basename(filepath)
                # Store our path
                self.filepath = abspath(expanduser(filepath))

            else:
                # Store our filename
                self.filename = basename(filepath)

    def getvalue(self):
        """
        This is mostly just used for unit testing, but it
        greatly makes life easier anyway.

        Effectively we put the pointer at the head of our
        file and read back the entire chunk into memory
        and return it.
        """

        if not self.open(mode=NNTPFileMode.BINARY_RO, eof=False):
            # Error
            return None

        # Head of data
        self.stream.seek(0L, SEEK_SET)

        return self.read()

    def can_post(self):
        """
        Similar to is_valid() except only returns true if the item is
        postable. The big difference is is_valid() is toggle after something
        has been downloaded, where as can_post() is a field checked prior
        to posting to an NNTP Server.

        Content is only postable in ascii form, so binaries and directories
        will always return a False here.
        """
        return False

    def is_valid(self):
        """
        A simple function that returns whether the article is valid or not

        The function returns True if it is valid, and False if it isn't
        and None if there isn't enough information to make a valid guess.

        The basic version (no overloading) just returns what the flag
        was set to.

        A directory can never be valid because it isn't content per say
        but a container to content.
        """
        return self._is_valid and not self._isdir

    def open(self, filepath=None, mode=None, eof=False):
        """
        Opens a filepath specified and re-attaches to it.
        You can also pass in an already open stream which
        causes it to operate in a detached state

        default open mode is BINARY_RW

        if eof is set to True, then after the file is opened, the
        pointer is placed at the end of the file (oppose to
        the head)
        """

        if not mode:
            # Read and write
            mode = NNTPFileMode.BINARY_RW

        if self.stream is not None:
            if self.filemode is not None and self.filemode == mode:
                # ensure we're at the head of the file
                if not eof:
                    self.stream.seek(0L, SEEK_SET)
                else:
                    self.stream.seek(0L, SEEK_END)

                return weakref.ref(self.stream)

        if not filepath and self.filepath:
            # Update filepath
            filepath = self.filepath

        elif not filepath:
            if not isdir(self.work_dir):
                # create directory
                mkdir(self.work_dir)

            # Create a Temporary File
            fileno, self.filepath = mkstemp(dir=self.work_dir)
            try:
                self.stream = fdopen(fileno, mode)
                if self._detached is None:
                    self._detached = False

                # save the last mode the file was opened as
                self.filemode = mode

                logger.debug(
                    'Opened %s (mode=%s)' %
                    (self.filepath, mode),
                )

            except (IOError, OSError) as e:
                logger.error(
                    'Could not open %s (mode=%s)' %
                    (self.filepath, mode),
                )
                logger.debug(
                    'fdopen({0}, {1}, wd={2}) exception ({3})'.format(
                        fileno, mode, self.work_dir, str(e)))
                return False

            return weakref.ref(self.stream)

        if isinstance(filepath, basestring):

            # expand our path to be absolute
            filepath = abspath(expanduser(filepath))

            # Create our stream
            try:
                self.stream = open(filepath, mode)

                self.filepath = filepath
                if self._detached is None:
                    self._detached = True

                # save the last mode the file was opened as
                self.filemode = mode

                logger.debug(
                    # D flag for Detached
                    'Opened %s (mode=%s) (flag=D)' %
                    (self.filepath, mode),
                )

            except (IOError, OSError) as e:
                logger.error(
                    'Could not open %s (mode=%s) (flag=D)' %
                    (self.filepath, mode),
                )
                logger.debug(
                    'open({0}, {1}, wd={2}) exception ({3})'.format(
                        filepath, mode, self.work_dir, str(e)))
                return False

        elif hasattr(filepath, 'seek'):
            # assume we're dealing with an already open stream and therefore
            # we work in a detached state
            self.stream = filepath
            self.filepath = filepath.get('name')
            self.filemode = filepath.get('mode')

            if self.filepath:
                self.filename = basename(self.filepath)
            else:
                self.filename = ''

            # You can never have an attached file without a filepath
            self._detached = False

            # Reset dirty flag
            self._dirty = False

        else:
            logger.error(
                'Could not open object %s' % (type(self.filepath))
            )
            return False

        if not eof:
            # Ensure we're at the head of the file
            self.stream.seek(0L, SEEK_SET)

        else:
            # Ensure we're at the end of the file
            self.stream.seek(0L, SEEK_END)

        return weakref.ref(self.stream)

    def encode(self, encoder):
        """
        A wrapper to the encoding of content. The function returns None if
        a problem occurs, otherwise the function returns an NNTPContent()
        object.

        The power of this function comes from the fact you can pass in
        multiple encoders to have them all fire after one another.
        """
        # Python does not allow recursive inclusion; since NNTPContent is
        # included via the codec paths we test if it's an encoder by just
        # looking for the encode() function
        if not isinstance(encoder, object):
            return None

        if not isinstance(encoder, (list, tuple, sortedset, list)):
            # work with a tuple for now
            encoder = (encoder, )

        # Content object we chain to
        content = self

        for _enc in encoder:
            # Support Type initializations
            if isinstance(_enc, type):
                enc = _enc()
            else:
                enc = _enc

            if hasattr(enc, 'encode') and \
               isinstance(enc.encode, MethodType):
                # We're dealing with a stream based encoder
                content = enc.encode(self)
                if content is None:
                    return None

            else:
                # We don't support this
                return None

        return content

    def load(self, filepath, sort_no=10000):
        """
        This causes the function to point to the file specified and acts in a
        detached manner to it.

        Loaded files are 'always' detached; if the file already existed in the
        filesystem it would be silly to tie it to the scope of this object.

        If the filepath identifie is an NNTPContent() object, then it is copied
        into an 'attached' NNTPContent() object.

        If the filepath identified is a set, sortedset or list of
        NNTPContent() objects, a new single 'attached' file will be generated
        based on the contains passed in.  NNTPContent() objects are
        automatically combined/merged based on their ordering by automating
        the use of the .append() function of this class for the caller.

        """

        if self.stream is not None:
            if self.filemode == NNTPFileMode.BINARY_WO_TRUNCATE:
                # Truncate remaining portions of the file
                self.truncate()

            # Close any existing open file
            self.close()

        if self._detached is False and self.filepath:
            # We're changing so it's better we unlink this (but only if we're
            # attached to it)
            rm(self.filepath)

        # Support directories but initialize field to false
        self._isdir = False

        # Set Detached since we don't want to obstruct our newly loaded
        # file in any way
        self._detached = True

        # Reset Valid Flag
        self._is_valid = False

        # Reset Unique Flag
        self._unique = False

        if isinstance(filepath, NNTPContent):
            # Support NNTPContent object copying; by simply storing
            # the object in a list, we are able to catch it in the
            # next check
            self.part = filepath.part
            filepath = [filepath]

        if isinstance(filepath, (tuple, set, sortedset, list)):
            # Perform merge if we detected a set of NNTPContent objects
            count = 0
            for content in filepath:
                if isinstance(content, NNTPContent):
                    self.append(content)
                    count += 1

            if count == 0 or len(filepath) != count:
                # Return True if we iterated over everything
                return False

            # update our filepath to be that of the file that was actually
            # created
            filepath = self.filepath

            # Our file is not detached in this state
            self._detached = False

        elif isdir(filepath):
            # Toggle our flag and fall through as we support directories
            self._isdir = True

        elif not isfile(filepath):
            # we can't load the file so reset some common variables
            self.filepath = None
            self.filename = ''

            return False

        # Assign new file
        self.filepath = filepath

        # Set Flag
        self._is_valid = True

        # Store our filename
        self.filename = basename(filepath)

        return True

    def copy(self):
        """
        copy is very close to save(); this is especially the case since
        the save() function has a copy parameter.

        the difference is copy actually duplicates/clones the content
        object as a completely new file that is not detached and therefore
        this copy (by default) is destroyed when the object goes out
        of scope.

        This is useful for handling downloads; one might want to create
        a copy of the original object and build onto it or alter it. With
        a copy to work with, you don't have to worry about obstructing the
        original file.

        copy() is just a clean way to simplify your code by wrapping
        the save() function with error handling.  Unlike save()
        which returns True if it was successful and False if it failed),
        copy() returns a new NNTPContent object.

        The function returns None if there was a failure

        """

        with pushd(self.work_dir, create_if_missing=True):
            # mkstemp used to genrate an unused temporary file
            _, filepath = mkstemp(dir=self.work_dir)
            try:
                # Remove the created file to silence any warnings
                # from the save() call coming next
                unlink(filepath)
            except:
                pass

        if not self.save(filepath, copy=True):
            return None

        # Initialize a new object
        obj = NNTPContent(
            filepath, part=self.part, total_parts=self.total_parts,
            begin=self._begin, end=self._end, total_size=self._total_size,
            work_dir=self.work_dir, sort_no=self.sort_no, unique=False,
        )

        # Save our official filename
        obj.filename = self.filename

        # Ensure our copy is attached
        obj.attach()

        return obj

    def save(self, filepath=None, copy=False):
        """
        This function writes the content to disk using the filename
        specified.

        If copy is False, then the content detaches itself from internal
        management and is moved to the new path specified by filepath.
        If copy is True, then content is written to the new location
        and remains in it's current detached state (whatever it was).

        If no filepath is specified, then the detected filename and
        work_dir specified during the objects initialization is used instead.

        The function returns True if it successfully saved the file and
        False otherwise. If you never passed in a filepath, then the path
        that was last loaded is saved to instead and the file is automatically
        detached from the Object.
        """
        if filepath:
            if not isfile(filepath):
                # If the file wasn't found relative to where we are, we'll try
                # again but relative to the work dir
                with pushd(self.work_dir, create_if_missing=True):
                    if isfile(filepath):
                        # Ensure we've expanded the file path
                        filepath = abspath(filepath)
                    else:
                        # Attempt to expand path
                        filepath = abspath(expanduser(filepath))
            else:
                # Ensure we've expanded the file path
                filepath = abspath(filepath)

        if filepath is None:
            if self.filename:
                filepath = join(self.work_dir, self.filename)
            else:
                filepath = join(self.work_dir, basename(self.filepath))

        elif isdir(filepath):
            if self.filename:
                filepath = join(filepath, basename(self.filename))
            else:
                filepath = join(self.work_dir, basename(self.filepath))

        if isfile(filepath):
            if self.path() != filepath:
                try:
                    unlink(filepath)
                    logger.warning('%s already existed (removed).' % (
                            filepath,
                        )
                    )
                except:
                    logger.error(
                        '%s already existed (and could not be removed).' % (
                            filepath,
                        )
                    )
                    return False

        # else: treat it as full path and filename included
        if not isdir(dirname(filepath)):
            # Attempt to pre-create save path
            if not mkdir(dirname(filepath)):
                return False

        if self.stream:
            # close the file if it's open
            self.close()

        # Function Wrapping
        if copy:
            action = _copy
            action_str = "copy"

        else:
            action = _move
            action_str = "move"

        if self.path() != filepath:
            try:
                action(self.filepath, filepath)

                logger.debug('%s(%s, %s)' % (
                    action_str, self.filepath, filepath,
                ))

            except ShutilError, e:
                logger.debug('%s(%s, %s) exception %s' % (
                    action_str, self.filepath, filepath, str(e),
                ))
                return False

        if not copy:
            # If we reach here, we want to treat moves() as an official
            # way of saving/writing the file to it's final destination
            # and therefore we can update our object

            # Detach File
            self._detached = True
            # Update filepath
            self.filepath = filepath
            # Update filename
            self.filename = basename(filepath)

        return True

    def path(self):
        """
        Always returns a filepath of the file, if one hasn't been created yet
        then one is automatically generated and returned.
        """
        if not self.filepath:
            # Create a Temporary File
            _, self.filepath = mkstemp(dir=self.work_dir)

        return self.filepath

    def split(self, size=81920, mem_buf=1048576):
        """Returns a set of NNTPContent() objects containing the split version
        of this object based on the criteria specified.

        Even if the object can't be split any further given the parameters, a
        set of at least 1 entry will always be returned.  None is returned if
        an error occurs.

        """
        # File Length
        file_size = len(self)
        if file_size == 0:
            # Object can not be split
            return None

        if not isinstance(size, int):
            # try to support other types
            size = strsize_to_bytes(size)

        if not size or size < 0:
            return None

        if not isinstance(mem_buf, int):
            # try to support other types
            mem_buf = strsize_to_bytes(mem_buf)

        if not mem_buf or mem_buf < 0:
            return None

        # Initialize Part #
        part = 0

        # Initialize Total Part #
        total_parts, partial = divmod(len(self), size)
        if partial:
            total_parts += 1

        # A lists of NNTPContent() objects to return
        objs = sortedset(key=lambda x: x.key())

        if not self.open(mode=NNTPFileMode.BINARY_RO):
            return None

        # Calculate the total length of our data
        total_size = len(self)

        # File length of our first object
        f_length = 0

        # Total Bytes Read
        total_bytes = 0

        # Initialize dummy value
        obj = None

        # Now read our chunks as per our memory restrictions
        while True:

            if total_bytes == 0:
                # Read memory chunk
                data = BytesIO(self.stream.read(mem_buf-total_bytes))

                # Retrieve length of our buffer
                total_bytes = data.seek(0L, SEEK_END)

                # Reset our pointer to the head of our data stream
                data.seek(0L, SEEK_SET)

                if total_bytes == 0:
                    if f_length > 0:
                        # Store our last object before we wrap up
                        obj.close()
                        objs.add(obj)
                    # Return our list of NNTPContent() objects
                    return objs

            while total_bytes > 0 and size-f_length > 0:
                # Store content up to our alotted amount
                block_size = size-f_length
                if total_bytes < block_size:
                    block_size = total_bytes

                if f_length == 0:

                    # Create a new object
                    obj = NNTPContent(
                        filepath=self.filename,
                        part=part+1,
                        total_parts=total_parts,
                        begin=(part*size),
                        end=((part*size)+size),
                        total_size=total_size,
                        work_dir=self.work_dir,
                        sort_no=self.sort_no,
                    )

                    # Increment our part
                    part += 1

                    # Create a pointer to the parent
                    obj._parent = weakref.proxy(self)

                    # Open the new file
                    obj.open(mode=NNTPFileMode.BINARY_WO_TRUNCATE)

                try:
                    obj.write(data.read(block_size))

                except IOError, e:
                    if e[0] is errno.ENOSPC:
                        # most probably a disk space issue
                        logger.error(
                            'Ran out of disk space while writing %s.' %
                            (obj.filepath),
                        )
                    else:
                        # most probably a disk space issue
                        logger.error(
                            'An I/O error '
                            '(%d) occured while writing %s to disk.' %
                            (e[0], obj.filepath),
                        )

                    # Tidy
                    self.close()

                    # Return None
                    return None

                f_length += block_size
                total_bytes -= block_size

            if f_length == size:
                # We're done
                obj.close()
                objs.add(obj)

                # File length reset
                f_length = 0

        # code will never reach here
        return None

    def write(self, data, eof=True):
        """
        Writes data to stream

        eof is only considered if the file wasn't open prior to the write()
        call. Otherwise the pointer remains where it last was. If set to True
        and the file wasn't previously open, the pointer is automatically
        placed at the end of the stream.

        """
        if self.stream is None:
            # open the file if it's not already open
            self.open(mode=NNTPFileMode.BINARY_RW, eof=eof)

        response = self.stream.write(data)

        if not self._dirty:
            # Set dirty flag
            self._dirty = True

            # We can't trust self._end anymore now because content was
            # written to the file.
            self._end = None

        return response

    def read(self, n=-1):
        """
        read up to n bytes from the stream
        """
        if self.stream is None:
            # open the file if it's not already open
            self.open(mode=NNTPFileMode.BINARY_RO, eof=False)

        return self.stream.read(n)

    def close(self):
        """
        Closes the file but retains any attachment to it.
        """
        if self.stream is not None:
            try:
                self.stream.close()
                if self.filepath:
                    logger.debug('Closed %s' % (self.filepath))
                else:
                    logger.debug('Closed stream.')
            except:
                pass

            self.stream = None
            self.filemode = None

            # A closed file can't be dirty as content is
            # flushed to disk at this point; therefore reset the flag
            # back to False
            self._dirty = False

        return

    def append(self, content):
        """
        This function takes a content object (or list of content objects) and
        appends them to `this` object.

        """
        if isinstance(content, NNTPContent):
            content = [content]

        if not self.open(mode=NNTPFileMode.BINARY_WO, eof=True):
            return False

        for entry in content:
            if isinstance(entry, NNTPContent):
                # Just append the current content
                if not entry.open(mode=NNTPFileMode.BINARY_RO, eof=False):
                    logger.debug('Error handling content: %s' % entry)
                    continue

                logger.debug('Appending content %s' % entry)

                while True:
                    buf = entry.stream.read(self._block_size)
                    if not buf:
                        # Set dirty flag
                        self._dirty = True
                        break
                    self.stream.write(buf)

                entry.close()

        return True

    def begin(self):
        """
        Returns the beginning ptr; this is nessisary when building encoded
        parts to be posted on an NNTPServer
        """
        if self._begin:
            return self._begin
        return 0

    def end(self):
        """
        Returns the end ptr; this is nessisary when building encoded
        parts to be posted on an NNTPServer
        """
        if self._end is None:
            self._end = self._begin + len(self)
        return self._end

    def total_size(self):
        """
        Returns the total size of the entire object (all parts included).
        This result is the same as len() if there is only 1 part to
        the entire object
        """
        if self.total_parts <= 1:
            return len(self)

        return self.end() - self.begin()

    def is_attached(self):
        """
        Simply returns whether or not the file is attached to the object or
        not.  Files that are attached are destroyed when the object goes
        out of scope.

        """
        return not self._detached

    def detach(self):
        """
        Detach the article stored on disk from being further managed by this
        class
        """
        self._detached = True
        return

    def attach(self):
        """
        Attach the file and it's data directly to this objects life expectancy
        """
        self._detached = False
        return

    def remove(self):
        """
        Gracefully remove the file (attached or not)
        """
        if self.stream is not None:
            self.close()

        if self.filepath:
            return rm(self.filepath)

        return False

    def key(self):
        """
        Returns a key that can be used for sorting with:
            lambda x : x.key()
        """
        if self.part is not None:
            result = '%.5d/%s/%.5d' % (self.sort_no, self.filename, self.part)
        else:
            result = '%.5d/%s//' % (self.sort_no, self.filename)

        if self._unique is not False:
            return result + self._unique

        return result

    def post_iter(self, block_size=BLOCK_SIZE):
        """
        Returns NNTP string as it would be required for posting to an
        NNTP Server
        """
        if not block_size:
            block_size = self._block_size

        if self.open(mode=NNTPFileMode.BINARY_RO):
            while 1:
                data = self.stream.read(block_size)
                if not data:
                    break
                yield data
            self.close()

    def crc32(self):
        """
        A little bit old-fashioned, but some encodings like yEnc require that
        a crc32 value be used.  This calculates it based on the file
        """
        # block size defined as 2**16
        block_size = 65536

        # The mask to apply to all CRC checking
        BIN_MASK = 0xffffffffL

        # Initialize
        _crc = 0

        if self.open(mode=NNTPFileMode.BINARY_RO):
            for chunk in iter(lambda: self.stream.read(block_size), b''):
                _crc = crc32(chunk, _crc)

            return format(_crc & BIN_MASK, '08x')

        return None

    def mime(self):
        """
        Returns the mime of the object
            Source: https://github.com/ahupp/python-magic
        """

        # Initialize our Mime object
        m = Mime()

        if self.open(mode=NNTPFileMode.BINARY_RO):
            mr = m.from_content(self.stream.read(128))
        else:
            mr = None

        if mr is None or mr.type() == DEFAULT_MIME_TYPE:
            # Try one more time by the filename
            mr = m.from_filename(
                self.filename if self.filename else self.filepath)

        # Return our type
        return mr

    def md5(self):
        """
        Simply return the md5 hash value associated with the content file.

        If the file can't be accessed, then None is returned.
        """
        md5 = hashlib.md5()
        if self.open(mode=NNTPFileMode.BINARY_RO):
            for chunk in \
                    iter(lambda: self.stream.read(128*md5.block_size), b''):
                md5.update(chunk)
            return md5.hexdigest()
        return None

    def sha1(self):
        """
        Simply return the sha1 hash value associated with the content file.

        If the file can't be accessed, then None is returned.
        """
        sha1 = hashlib.sha1()
        if self.open(mode=NNTPFileMode.BINARY_RO):
            for chunk in \
                    iter(lambda: self.stream.read(128*sha1.block_size), b''):
                sha1.update(chunk)
            return sha1.hexdigest()
        return None

    def sha256(self):
        """
        Simply return the sha256 hash value associated with the content file.

        If the file can't be accessed, then None is returned.
        """
        sha256 = hashlib.sha256()
        if self.open(mode=NNTPFileMode.BINARY_RO):
            for chunk in \
                    iter(lambda: self.stream.read(128*sha256.block_size), b''):
                sha256.update(chunk)
            return sha256.hexdigest()
        return None

    def tell(self):
        """
        Allows reference to our object from within a Codec()

        """
        if not self.filepath:
            # If there is no filepath, then we're probably dealing with a
            # stream in memory like a StringIO or BytesIO stream.
            if self.stream:
                # Advance to the end of the file
                return self.stream.tell()

        else:
            if self.stream and self._dirty is True:
                self.stream.flush()
                self._dirty = False

        if not self.stream:
            if not self.open(mode=NNTPFileMode.BINARY_RO):
                return None

        return self.stream.tell()

    def readline(self, *args, **kwargs):
        """
        Returns a single line from the stream
        """
        if not self.stream:
            return ''
        return self.stream.readline(*args, **kwargs)

    def next(self):
        """
        Python 2 support
        Support stream type functions and iterations
        """
        data = self.stream.read(self._block_size)
        if not data:
            self.close()
            raise StopIteration()

        return data

    def __next__(self):
        """
        Python 3 support
        Support stream type functions and iterations
        """
        data = self.stream.read(self._block_size)
        if not data:
            self.close()
            raise StopIteration()

        return data

    def __iter__(self):
        """
        Grants usage of the next()
        """

        # Ensure our stream is open with read
        self.open(mode=NNTPFileMode.BINARY_RO)
        return self

    def __len__(self):
        """
        Returns the length of the content
        """
        if not self.filepath:
            # If there is no filepath, then we're probably dealing with a
            # stream in memory like a StringIO or BytesIO stream.
            if self.stream:
                # Advance to the end of the file
                ptr = self.stream.tell()
                # Advance to the end of the file and get our length
                length = self.stream.seek(0L, SEEK_END)
                if length != ptr:
                    # Return our pointer
                    self.stream.seek(ptr, SEEK_SET)
            else:
                # No Stream or Filepath; nothing has been initialized
                # yet at all so just return 0
                length = 0
        else:
            if self.stream and self._dirty is True:
                self.stream.flush()
                self._dirty = False

            # Get the size
            length = getsize(self.filepath)

        return length

    def hexdump(self, max_bytes=128):
        """
        Returns a hex dump of characters up to the defined max_bytes
        If max_bytes is 0 then all content is dumped

        """

        if not self.open(mode=NNTPFileMode.BINARY_RO, eof=False):
            # Error
            return None

        # Head of data
        self.stream.seek(0L, SEEK_SET)

        if not max_bytes:
            return hexdump(self.read())
        return hexdump(self.read(max_bytes))

    def __del__(self):
        """
        Gracefully remove the file retrieved as it was removed
        from scope for a good reason. We can easily avoid
        having this step called by calling the detach() function
        """
        if self.stream is not None:
            self.close()

        if not self._detached and self.filepath:
            # We need to do some cleanup
            rm(self.filepath)

    def __lt__(self, other):
        """
        Support Less Than (<) operator for sorting
        """
        return self.key() < self.key()

    def __cmp__(self, content):
        """
        Support comparative checks
        """
        return cmp(self.key(), content.key())

    def __str__(self):
        """
        Return a printable version of the file being read
        """
        if self.part is not None:
            return '%s.%.5d' % (self.filename, self.part)

        return self.filename

    def __enter__(self):
        """
        supports use of the 'with' clause.  You can use the expression:

        with myobj as fp:
            # write and/or read content here

        """
        # Open our file and return a pointer to it
        if self.open():
            return self

        # Throw an exception
        raise IOError(errno.EIO, 'Could not open NNTPContent', self.path())

    def __exit__(self, type, value, traceback):
        """
        our exit function executed from the 'with' clause
        """
        # Close our file (if open)
        self.close()

    def __repr__(self):
        """
        Return a printable version of the file being read
        """
        if self.part is not None:
            return \
                '<NNTPContent sort=%d filename="%s" part=%d/%d len=%s />' % (
                    self.sort_no,
                    self.filename,
                    self.part,
                    self.total_parts,
                    bytes_to_strsize(len(self))
                )
        else:
            return '<NNTPContent sort=%d filename="%s" len=%s />' % (
                self.sort_no,
                self.filename,
                bytes_to_strsize(len(self)),
            ) class NNTPBinaryContent(NNTPContent):
    """
    A Binary file representation
    """
    def __init__(self, filepath=None, part=None, total_parts=None,
                 begin=None, end=None, total_size=None,
                 work_dir=None, sort_no=10000, *args, **kwargs):
        """ Intitialize NNTPBinaryContent
        """

        super(NNTPBinaryContent, self).__init__(
            filepath=filepath,
            part=part, total_parts=total_parts,
            begin=begin, end=end, total_size=total_size,
            work_dir=work_dir,
            sort_no=sort_no, *args, **kwargs)

    def __repr__(self):
        """
        Return a printable version of the file being read
        """
        if self.part is not None:
            return '<NNTPBinaryContent sort=%d filename="%s" part=%d/%d len=%s />' % (
                self.sort_no,
                self.filename,
                self.part,
                self.total_parts,
                bytes_to_strsize(len(self)),
            )
        else:
            return '<NNTPBinaryContent sort=%d filename="%s" len=%s />' % (
                self.sort_no,
                self.filename,
                bytes_to_strsize(len(self)),
            ) class NNTPAsciiContent(NNTPContent):
    """
    An Ascii file representation
    """
    def __init__(self, filepath=None, part=None, total_parts=None,
                 work_dir=None, sort_no=20000, *args, **kwargs):
        super(NNTPAsciiContent, self).__init__(
            filepath=filepath,
            part=part, total_parts=total_parts, work_dir=work_dir,
            sort_no=sort_no, *args, **kwargs)

    def __next__(self):
        """
        Python 3 support
        Support stream type functions and iterations
        """
        data = self.stream.readline()
        if not data:
            self.close()
            raise StopIteration()

        return data

    def can_post(self):
        """
        AsciiContent is postable as long as it exists
        """
        return (self._isdir is False) and exists(self.filepath)

    def next(self):
        """
        Python 2 support
        Support stream type functions and iterations
        """
        data = self.stream.readline()
        if not data:
            self.close()
            raise StopIteration()

        return data

    def __str__(self):
        """ Returns content """
        return self.getvalue()

    def __unicode__(self):
        """ Returns content """
        return unicode(self.getvalue())

    def __repr__(self):
        """
        Return a printable version of the file being read
        """
        if self.part is not None:
            return '<NNTPAsciiContent sort=%d filename="%s" part=%d/%d len=%s />' % (
                self.sort_no,
                self.filename,
                self.part,
                self.total_parts,
                bytes_to_strsize(len(self)),
            )
        else:
            return '<NNTPAsciiContent sort=%d filename="%s" len=%s />' % (
                self.sort_no,
                self.filename,
                bytes_to_strsize(len(self)),
            ) def bytes_to_strsize(byteval):
    """
    This function returns the string size equivalent (as a string) that best
    represents the integer (in bytes) passed in.

    """
    unit = 'B'
    try:
        byteval = float(byteval)

    except(ValueError, TypeError):
        return None

    if byteval >= 1024.0:
        byteval = byteval / 1024.0
        unit = 'KB'
        if byteval >= 1024.0:
            byteval = byteval / 1024.0
            unit = 'MB'
            if byteval >= 1024.0:
                byteval = byteval / 1024.0
                unit = 'GB'
                if byteval >= 1024.0:
                    byteval = byteval / 1024.0
                    unit = 'TB'

    return '%.2f%s' % (round(byteval, 2), unit) @contextmanager
def pushd(newdir, create_if_missing=False, perm=0775):
    """
    # A pushd/popd implimentation
    # Based on : http://stackoverflow.com/questions/6194499/\
                    pushd-through-os-system

    # It's use is pretty straight forward:
    # with pushd('somewhere'):
    #     # somewhere
    #     print os.getcwd()
    #
    # # wherever you started
    # print os.getcwd()

    """
    prevdir = getcwd()
    if not isdir(newdir) and create_if_missing:
        # Don't bother checking the success or not
        # we'll find out soon enough with chdir()
        mkdir(newdir, perm)

    chdir(newdir)
    try:
        yield

    finally:
        # Fall back to previous directory popd()
        chdir(prevdir) class Mime(object):
    """
    Our mime response object.
    """
    def __init__(self, magic_file=None):
        """
        Create a new libmagic wrapper.
        magic_file - Another magic file other then the default
        """

        # The lock allows us to be thread-safe
        self.lock = Semaphore(value=1)

        # Tracks the errno/errstr set after each call
        self.errno = 0
        self.errstr = ''

        # Load our magic file
        self.magic_file = magic_file

        # Initialize our flags
        # our flags
        self.flags = (MAGIC_MIME | MAGIC_MIME_ENCODING)

    def from_content(self, content, uncompress=False, fullscan=False):
        """
        detect the type based on the content specified.

          content - the content to process
          uncompress - Try to look inside compressed files.
          fullscan - Scan entire file and extract as much details as possible

        """
        if not HAS_LIBMAGIC:
            return None

        if not content:
            return MimeResponse()

        # our pointer to our libmagic object
        ptr = None

        # our flags
        flags = self.flags

        if fullscan:
            flags |= MAGIC_CONTINUE

        if uncompress:
            flags |= MAGIC_COMPRESS

        try:
            self.lock.acquire(blocking=True)
            # 'application/octet-stream; charset=binary'

            # Acquire a pointer
            ptr = _magic['open'](flags)

            # Acquire a pointer
            _magic['load'](ptr, self.magic_file)
            self.errno = _magic['errno'](ptr)
            if self.errno == 0:

                # Achieve our results as a list
                _typ, _enc = MAGIC_LIST_RE.split(self._tostr(_magic['buffer'](
                    ptr,
                    self._tobytes(content),
                    len(content),
                )))

                # Acquire our errorstr (if one exists)
                self.errstr = _magic['error'](ptr)

                _typs = []
                for n, _typ_re in enumerate(TYPE_PARSE_RE.finditer(_typ)):
                    if n is 0:
                        _typs.append((0, _typ_re.group('mtype')))
                    else:
                        _typs.append(
                            (
                                int(_typ_re.group('offset')),
                                _typ_re.group('mtype'),
                            ),
                        )

                _enc_re = ENCODING_PARSE_RE.match(_enc)
                if _enc_re:
                    _enc = _enc_re.group('encoding')

                mr = MimeResponse(mime_type=_typs, mime_encoding=_enc)

                # return our object
                return mr

        except TypeError:
            # This occurs if buffer check returns None
            # Acquire our errorstr (if one exists)
            self.errstr = _magic['error'](ptr)
            if self.errstr:
                # an error occured; return None
                return None

            # If we get here, we didn't even get an error
            return MimeResponse(
                mime_type=DEFAULT_MIME_TYPE,
                mime_encoding=DEFAULT_MIME_ENCODING,
            )

        finally:
            if ptr is not None:
                # Release our pointer
                _magic['close'](ptr)

            # Release our lock
            self.lock.release()

        # We failed if we got here, return nothing
        return None

    def from_file(self, path, uncompress=False, fullscan=False):
        """
        detect the type based on the content specified.

          path - the file to process
          uncompress - Try to look inside compressed files.
          fullscan - Scan entire file and extract as much details as possible

        """
        if not HAS_LIBMAGIC:
            return None

        if not path:
            return None

        # our pointer to our libmagic object
        ptr = None

        # our flags
        flags = self.flags

        if fullscan:
            flags |= MAGIC_CONTINUE

        if uncompress:
            flags |= MAGIC_COMPRESS

        # Default Response
        res = ""

        try:
            self.lock.acquire(blocking=True)
            # 'application/octet-stream; charset=binary'

            # Acquire a pointer
            ptr = _magic['open'](flags)

            # Acquire a pointer
            _magic['load'](ptr, self.magic_file)
            self.errno = _magic['errno'](ptr)
            if self.errno == 0:

                res = self._tostr(_magic['file'](ptr, self._tobytes(path)))

                # Acquire our errorstr (if one exists)
                self.errstr = _magic['error'](ptr)

                # Achieve our results as a list
                _typ, _enc = MAGIC_LIST_RE.split(res)

                _typs = []
                for n, _typ_re in enumerate(TYPE_PARSE_RE.finditer(_typ)):
                    if n is 0:
                        _typs.append((0, _typ_re.group('mtype')))
                    else:
                        _typs.append(
                            (
                                int(_typ_re.group('offset')),
                                _typ_re.group('mtype')
                            ),
                        )

                _enc_re = ENCODING_PARSE_RE.match(_enc)
                if _enc_re:
                    _enc = _enc_re.group('encoding')

                mr = MimeResponse(
                    mime_type=_typs,
                    mime_encoding=_enc,
                    extension=self.extension_from_filename(path),
                )

                # return our object
                return mr

        except TypeError:
            # This occurs if buffer check returns None
            # Acquire our errorstr (if one exists)
            self.errstr = _magic['error'](ptr)
            if self.errstr:
                # an error occured; return None
                return None

            # If we get here, we didn't even get an error
            return MimeResponse(
                mime_type=DEFAULT_MIME_TYPE,
                mime_encoding=DEFAULT_MIME_ENCODING,
                extension=self.extension_from_filename(path),
            )

        except ValueError:
            # This occurs during our regular expression extraction which
            # couldn't accomplish it's feat because an error string (such as
            # no file exists) was returned as part of the response instead
            # of our expected mime type. Store this error
            self.errstr = res

            # Now return None
            return None

        finally:
            if ptr is not None:
                # Release our pointer
                _magic['close'](ptr)

            # Release our lock
            self.lock.release()

        # We failed if we got here, return nothing
        return None

    def from_filename(self, filename):
        """
        detect the type based on the filename (and/or extension) specified.

        """

        if not filename:
            # Invalid
            return None

        mime_type = next((m for m in MIME_TYPES if m[1].match(filename)), None)

        if mime_type:
            return MimeResponse(
                mime_type=mime_type[0],
                mime_encoding=mime_type[2],
                extension=self.extension_from_filename(filename),
            )

        # No match; default response
        return MimeResponse(
            mime_type=DEFAULT_MIME_TYPE,
            mime_encoding=DEFAULT_MIME_ENCODING,
            extension=self.extension_from_filename(filename),
        )

    def from_bestguess(self, path):
        """
        First attempts to look at the files contents (if it can), if not it
        falls back to looking at the filename and always returns it's best
        guess.
        """
        mr = self.from_file(path)
        if mr is None or mr.type() in \
                (DEFAULT_MIME_TYPE, DEFAULT_MIME_EMTPTY_FILE):

            _mr = self.from_filename(path)
            if _mr is None:
                if mr is None:
                    # Not parseable
                    return None

                # Otherwise return a stream and intentionally
                # do not count the empty file (if detected)
                return MimeResponse(
                    mime_type=DEFAULT_MIME_TYPE,
                    mime_encoding=DEFAULT_MIME_ENCODING,
                    extension=self.extension_from_filename(path),
                )

            elif mr is not None and _mr == DEFAULT_MIME_TYPE:
                return mr
            return _mr
        return mr

    def extension_from_mime(self, mime_type):
        """
        takes a mime type and returns the file extension that bests matches
        it. This function returns an empty string if the mime type can't
        be looked up correctly, otherwise it returns the matching extension.
        """

        if not mime_type:
            # Invalid; but return an empty string
            return ''

        # iterate over our list and return on our first match
        return next((m[3] for m in MIME_TYPES if mime_type == m[0]), '')

    def extension_from_filename(self, filename):
        """
        Takes a filename and extracts the extension from it (if possible).
        This function is a little like the os.path.splitext() which can
        take a file like:  test.jpg and return the .jpg. But it can not
        however handle names like text.jpeg.gz (to which it would just
        return .gz in this case.

        This function (specifically written for newsreap/usenet parsing)
        will attempt to extract a worthy extension (all of it) from the
        filename. Hence:

            extension_from_filename('myfile.pdf.vol03+4.par2')
               will return '.pdf.vol03+4.par2'

        """

        if not filename:
            # Invalid; but return an empty string
            return ''

        result = EXTENSION_SEARCH_RE.search(filename)
        if result:
            ext = result.group('ext')
            if ext is not None:
                return ext

        # Nothing found if we reach here
        return ''

    def _tostr(self, s, encoding='utf-8'):
        if s is None:
            return None
        if isinstance(s, str):
            return s
        try:  # keep Python 2 compatibility
            return str(s, encoding)
        except TypeError:
            return str(s)

    def _tobytes(self, b, encoding='utf-8'):
        if b is None:
            return None
        if isinstance(b, bytes):
            return b
        try:  # keep Python 2 compatibility
            return bytes(b, encoding)
        except TypeError:
            return bytes(b) DEFAULT_MIME_TYPE = 'application/octet-stream' NEWSREAP_ENGINE = '%s.engine' % NEWSREAP_LOGGER
import re
import logging
from blist import sortedset
from datetime import datetime
from os.path import isfile
from os.path import abspath
from os.path import expanduser
from os.path import splitext
from os.path import basename
from .codecs.CodecBase import CodecBase
from .codecs.CodecYenc import CodecYenc
from .NNTPSettings import DEFAULT_TMP_DIR
from .NNTPGroup import NNTPGroup
from .NNTPArticle import NNTPArticle
from .NNTPArticle import DEFAULT_NNTP_SUBJECT
from .NNTPArticle import DEFAULT_NNTP_POSTER
from .NNTPContent import NNTPContent
from .NNTPBinaryContent import NNTPBinaryContent
from .NNTPAsciiContent import NNTPAsciiContent
from .Utils import bytes_to_strsize
from .Utils import pushd
from .Mime import Mime
from .Mime import DEFAULT_MIME_TYPE
from newsreap.Logging import NEWSREAP_ENGINE
# -*- coding: utf-8 -*-
#
# NNTPSegmentedPost is an object that manages several NNTPArticles.
#
# Copyright (C) 2015-2017 Chris Caron <lead2gold@gmail.com>
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.



# Logging
logger = logging.getLogger(NEWSREAP_ENGINE)


class NNTPSegmentedPost(object):
    """
    An object for maintaining retrieved nzb content. Large files need
    to be split across multiple Articles in order to be posted.

    When combined into one, They create a SegmentedPost

    Similarily; this file makes posting easy by allowing you to add files
    to it in order to prepare an object worthy of posting.

    Every file you wish to post to an NNTP Server should be done through
    this file for the most power/control

    """

    def __init__(self, filename, subject=DEFAULT_NNTP_SUBJECT,
                 poster=DEFAULT_NNTP_POSTER, groups=None,
                 utc=None, work_dir=None, sort_no=None, codecs=None,
                 *args, **kwargs):
        """Initialize NNTP Segmented File

        Args:
            filename (str): Either a path of an existing filename or if
                            you're initializing a new file from scratch this
                            should be the name of the filename being
                            assembled.
        Kwargs:
            subject (str): The subject to associate with the file. This is
                            only required if you're posting the content (to
                            Usenet).
            poster (str): The poster to associate with the file. This is
                           only required if you're posting the content (to
                           Usenet).
            group (str, set): This can be either a string identifying a single
                               Usenet group, or it can be a set() of strings
                               identifying all of the Usenet groups you want
                               to cross-post to. This is only required if you
                               intend on posting the content (to Usenet).
            utc (int, datetime): Should always be in UTC format and can be
                                either defined as the number of seconds from
                                epoch (usually produced from gmtime()) or it
                                can be a datetime() object. This identifies the
                                date to associate with the file.  It's mostly
                                used for sorting.  If no time is specified
                                then it defaults to datetime.utcnow(). If a
                                valid `filename` was specified then the utc
                                defaults to the time associated with the files
                                modification date.
            codecs (CodecBase): The codec to use as our deobsfucation engine
                                You can specify as many Codecs as you want in
                                an interable form.

        Returns:
            Nothing

        Raises:
            AttributeError() if you don't follow group isn't a set() of str(),
                              None or str().

        """
        # The Filename
        self.filename = filename

        # These fields get populated when reading in an nzb file
        self.poster = poster
        self.utc = utc
        self.subject = subject

        # The sort order which helps with file ordering, if set to None
        # or a common value, then it has no bearing on anything.  Its
        # main goal is to provide a method of sorting multiple SegmentedPosts
        self.sort_no = sort_no

        # When writing files, this is the memory buffer to fill when
        # dealing with very large files. If this is set to None, then
        # no buffer is used. Default 500K
        self.mem_buffer = 512000

        if not isinstance(self.utc, datetime):
            # Convert into datetime
            try:
                self.utc = datetime.fromtimestamp(int(self.utc))

            except (TypeError, ValueError):
                # Used None, uninitialized, or Used bad value
                # Default timezone to 'now' but make it consistent
                # with the world, use the UTC as a common source
                self.utc = datetime.utcnow()

        # Load our Codecs
        self._codecs = codecs

        # Provide default codecs if required
        if not self._codecs:
            self._codecs = [CodecYenc(), ]


        elif isinstance(self._codecs, CodecBase):