class Group(ObjectBase):
    """
    A Group class contains a mapping of groups supported by the
    server in question.

    This group is rather heavy on the indexing; but it's generally
    pretty static and changes very infrequently after it's built
    once. So the heavy indexing only poses an initial expense
    """
    __tablename__ = 'group'

    # primary key
    id = Column(Integer, Sequence('id_seq'), primary_key=True)

    # group name
    name = Column(String(128), nullable=False, unique=True, index=True)

    # Some statistics that are just fetched from the NNTP Server; this
    # is a lose value since counts will vary across multiple usenet
    # servers with multiple retentions.  This value is based on the
    # primary server.
    count = Column(Integer, nullable=False, default=0)

    # group flags (associated with group)
    flags = Column(String, nullable=False, default='')

    # watch flag
    watch = Column(Boolean, nullable=False, default=False, index=True)

    # Track the time the group statistics were last updated
    last_updated = Column(DateTime, server_default=func.now())

    def __init__(self, *args, **kwargs):
        super(Group, self).__init__(*args, **kwargs)

    def __repr__(self):
        return "<Group(id='%d', name='%s', flag=%s', watch='%s')>" % (
                self.id, self.name, self.flag, self.watch) class GroupAlias(ObjectBase):
    """
    Group Aliases allow you to map NNTP Group names into you're
    own notation.  Hence map 'alt.binaries.test' into just 'a.b.test'.
    Or maybe just map alt.binaries.test to just the word 'test'.

    Short forming group names makes it easier to work with them
    from the CLI (Command Line Interface)

    Aliases are intentionally kept in their own group so they don't
    conflict with peoples configurations to which they've defined
    multiple servers. One alias to rule them all.
    """
    __tablename__ = 'group_alias'

    # A pointer to the group we're creating an alias to
    group_id = Column(Integer, ForeignKey("group.id"))

    # Allow alias mapping to groups, ie, swap alt.binaries to a.b.
    # Aliases greatly simplify command line control
    name = Column(String(64), index=True, nullable=False)

    # Create our primary key based on the server and group id's
    __mapper_args__ = {"primary_key": (group_id, name)}

    def __repr__(self):
        return "<GroupAlias(name='%s')>" % (self.name) class GroupTrack(ObjectBase):
    """
    A GroupTrack class contains a mapping of groups supported by the
    server in question in addition to any tracking details about it.

    Since we support multiple servers, we want to track each of
    their tracking pointers seperately.

    This table only only contains entries for the watched tables
    """
    __tablename__ = 'group_track'

    # A pointer to the server table tracking the NNTP Server
    server_id = Column(Integer, ForeignKey("server.id"))

    # A pointer to the group these statistics pertain to
    group_id = Column(Integer, ForeignKey("group.id"))

    # Low Watermark (head)
    low = Column(Integer, nullable=False, default=0)

    # High Watermark (tail)
    high = Column(Integer, nullable=False, default=0)

    # Current Pointer
    index_pointer = Column(Integer, nullable=False)

    # Scan Pointer
    scan_pointer = Column(Integer, nullable=False)

    # Tracks the last time a scan was done (moving the current pointer)
    last_scan = Column(DateTime, nullable=False, default=datetime.fromtimestamp(0))

    # Tracks the last time an index was done (moving the current pointer)
    last_index = Column(DateTime, nullable=False, default=datetime.fromtimestamp(0))

    # Track the last time the record was updated
    last_updated = Column(DateTime, nullable=False, server_default=func.now())

    # Associate our database relation
    group = relationship("Group")
    server = relationship("Server")

    # Create our primary key based on the server and group id's
    __mapper_args__ = { "primary_key": (server_id, group_id) }


    def __init__(self, *args, **kwargs):
        super(GroupTrack, self).__init__(*args, **kwargs)


    def __repr__(self):
        return "<GroupTrack(name='%s', current='%s', end='%s')>" % (
                             self.name, self.current, self.end) class Server(ObjectBase):
    """
    A Server class contains a mapping of nntp servers we're
    scanning and connection details for them.
    """

    __tablename__ = 'server'

    id = Column(Integer, Sequence('server_id_seq'), primary_key=True)

    # Descriptive NNTP Server name
    name = Column(String(256), nullable=False)

    # NNTP Host Identifier
    host = Column(String(128), index=True,
                  nullable=False, unique=True)

    def __init__(self, *args, **kwargs):
        super(Server, self).__init__(*args, **kwargs)

    def __repr__(self):
        """
        Printable string
        """
        return "<Server(id=%s, host='%s')>" % (self.id, self.host) def get_groups(session, lookup=None, watched=False):
    """
    Returns a dictionary of groups indexed by their names and their value
    set to the database key in the Group table.

     - If watched is set to True, then the watched groups are automatically
       considered in the returned results.

     - If the lookup is an alias, it's looked up and the matched groups are
        returned.
     - If the lookup just matches a group; then that group object is returned.
     - If the lookup is an integer; then the group is looked up and fetched
     - If the lookup is a tuple or list, then all of the entries in the list
       are processed using the above criteria and results merged in the list
       returned.

       None is returned if there was a problem fetching the information.

       A sample response might look like this:
           {
                u'alt.binaries.test': 34,
                u'alt.binaries.another.group': 3234,
            }
    """

    if not session:
        return None

    # Return list
    results = None

    if watched:
        # PEP8 E712 does not allow us to make a comparison to a boolean value
        # using the == instead of the keyword 'in'.  However SQLAlchemy
        # requires us to do just because that's how the amazing tool works.
        # so to get around the pep8 error, we'll just define a variable equal
        # to True and then we can compare to it
        pep8_e712 = True

        # Fetch our watch list
        results = dict(session.query(Group.name, Group.id)
                       .filter(Group.watch == pep8_e712).all())

    if isinstance(lookup, (basestring, int)):
        lookup = [lookup, ]

    elif lookup is None:
        # Return whatever results we have
        return results

    elif not isinstance(lookup, (dict, tuple, list)):
        # Not supported; return what we have
        logger.warning(
            "An unsupported group/alias lookup type (%s) was specified." %
            type(lookup),
        )
        return results

    for group_id in lookup:
        if isinstance(group_id, NNTPGroup):
            # We already know we're dealing with an NNTPGroup at this point
            groups = dict(session.query(Group.name, Group.id)
                          .filter(Group.name == str(group_id)).all())
            if not groups:
                logger.warning(
                    "The group '%s' does not exist." % str(group_id),
                )

        elif isinstance(group_id, basestring):
            _id = group_id.lower().strip()
            if not _id:
                continue

            groups = dict(session.query(Group.name, Group.id)
                          .filter(Group.name == _id).all())

            if not groups:
                # No problem; let us use the alias too
                groups = dict(session.query(Group.name, Group.id)
                              .join(GroupAlias)
                              .filter(GroupAlias.name == _id).all())

                if not groups:
                    # Try one last time using normalization
                    _group = NNTPGroup.normalize(group_id)

                    if _group == _id:
                        # We normalized to the same thing the above check did
                        logger.warning(
                            "The group/alias '%s' does not exist." % group_id,
                        )
                        continue

                    # if we reach here, we normalized to something different
                    groups = dict(session.query(Group.name, Group.id)
                                  .filter(Group.name == _group).all())
                    if not groups:
                        # We're definitely out of options at htis point
                        logger.warning(
                            "The group/alias '%s' does not exist." % group_id,
                        )
                        continue

        elif isinstance(group_id, int) and group_id > 0:
            # A id was specified; fetch it
            groups = dict(session.query(Group.name, Group.id)
                          .filter(Group.id == group_id).all())

            if not groups:
                logger.warning("The group id '%d' does not exist." % group_id)
                continue

        else:
            # Not supported - Ignored
            continue

        if groups:
            if results is None:
                # Quick and easy initialization on first match
                results = groups
            else:
                # append to our existing results
                results = dict(results.items() + groups.items())

    return results class Article(ObjectBase):
    """
    An Article class contains a mapping of one Article
    retrieved from usenet (via the xover() option)
    """

    __tablename__ = 'article'

    # Article (Unique) Message-ID
    message_id = Column(String(128), primary_key=True)

    # Article No (unique to usenet server only)
    article_no = Column(Integer, nullable=False, unique=True, index=True)

    # Article Subject
    subject = Column(String(256), nullable=False, index=True)

    # Article Poster
    poster = Column(String(128), nullable=False, index=True)

    # Article Size
    size = Column(Integer, default=0, nullable=False)

    # Article Line Count
    lines = Column(Integer, default=0, nullable=False)

    # Article Post Date
    posted_date = Column(DateTime, index=True)

    # Article Score
    score = Column(Integer, default=0, nullable=False, index=True)

    # There are to just to many articles to delete them individually.
    # The easiest way is to mark them for purging and clear them when
    # the count gets to high.
    deleted = Column(DateTime, default=None, index=True)

    # A flag that can be used to determine if the record should be
    # displayed or not; this over-rides any calculated score.
    hidden = Column(Boolean, default=False, nullable=False, index=True)

    def __init__(self, *args, **kwargs):
        super(Article, self).__init__(*args, **kwargs)

    def __repr__(self):
        return "<Article(message_id=%s)>" % (self.message_id) EOL = '\r\n'
EOD = '\r\n.'
EOL_RE = re.compile(r'([\r]?\n)')
EOD_RE = re.compile(r'(\.([\r]?\n))$')
EOLS_RE = re.compile(r'[\r]?\n')
GZIP_EOL_RE = re.compile(r'\.([\r]?\n)$')
NNTP_FETCH_BY_DATE_MAX_MISSES = 20
NNTP_RESPONSE_RE = re.compile(
    r'^[^0-9]*(?P<code>[1-9][0-9]{2})[\s:]*(?P<desc>.*[^\s])\s*$',
    re.MULTILINE,
)
NNTP_AUTHINFO_CHECK = re.compile(
    r'^AUTHINFO[\s]+(?P<key>[^\s]+)[\s]+(?P<value>.+)[\s]*$',
    re.IGNORECASE,
)
NNTP_GROUP_RESPONSE_RE = re.compile(
    r'(?P<count>[0-9]+)\s+(?P<low>[0-9]+)\s+' +
    r'(?P<high>[0-9]+)\s+(?P<name>.*[^\s]*[^.])(\.|\s)*$',
)
GZIP_COMPRESS_RE = re.compile(
    r'.*COMPRESS\s*=\s*GZIP.*',
    re.IGNORECASE,
)
POSTING_OK_RE = re.compile(
    r'.*POSTING OK.*',
    re.IGNORECASE,
)
NNTP_PORT = 119
NNTP_SSL_PORT = 563
NNTP_RESPONSE_TIMEOUT = 30.0
NNTP_WELCOME_MESSAGE_TIMEOUT = 15.0
NNTP_MIN_READ_BUFFER_SIZE = 1024
SCAN_FROM_HEAD = 40
SCAN_FROM_TAIL = 10
NNTP_XOVER_RETRIES = 5
    MAX_BUFFER_SIZE = 10485760
    READBACK_CHUNK_SIZE = 128
class NNTPConnection(SocketBase):
    def __init__(self, username=None, password=None, secure=False,
                 iostream=NNTPIOStream.RFC3977_GZIP,
                 join_group=False, use_body=False, use_head=True,
                 encoding=None, work_dir=None,
                 filters=None, hooks=None, *args, **kwargs):
    def append(self, connection, *args, **kwargs):
    def connect(self, *args, **kwargs):
    def _connect(self, *args, **kwargs):
    def post(self, payload, update_headers=True, success_only=False):
    def _post(self, article, update_headers=True):
    def group(self, name):
    def groups(self, filters=None, lazy=True):
    def tell(self):
    def seek_by_date(self, refdate, group=None):
    def _seek_by_date(self, refdate, head=None, tail=None):
    def seek(self, index, whence=None):
    def next(self, count=50000):
    def prev(self, count=50000):
    def xover(self, group=None, start=None, end=None,
              sort=XoverGrouping.BY_POSTER_TIME):
    def stat(self, id, full=None, group=None):
    def _stat(self, id, full=None):
    def get(self, id, work_dir=None, decoders=None, group=None, max_bytes=0):
    def _get_wrapper(self, id, work_dir, decoders=None, group=None,
                     max_bytes=0):
    def _get(self, id, work_dir, decoders=None, group=None, max_bytes=0):
    def send(self, command, timeout=None, decoders=None, retries=0):
    def _recv(self, decoders=None, timeout=None):
    def close(self):
    def _soft_reset(self):
    def _hard_reset(self, wait=True):
    def __del__(self):
    def __str__(self):
    def __unicode__(self):
    def __repr__(self): class NNTPGroupDatabase(Database):
    """
    A managment class to handle Group/Article Databases
    """

    def __init__(self, engine=None, reset=None):
        """
        Initialize NNTP Group/Article Database
        """
        super(NNTPGroupDatabase, self).__init__(
            base=ObjectBase,
            vsp=Vsp,
            engine=engine,
            reset=reset,
        ) class NNTPConnection(SocketBase):
    """
        NNTPConnection is a class that wraps and eases the comunication
        to and from an NNTP Server.

        - Usenet Compression supported (GZIP) if the usenet server allows
          for it.

        - SSL Support (you'll ideally want to use the default of TLSv1
          for security reasons; but can support SSLv3 too if the remote
          server can't handle TLSv1).

        - It uses btree+ to store all of it's fetched results using a
          fantastic addon called blist which never maded it into
          python (https://www.python.org/dev/peps/pep-3128/)

        - can seek_by_date() using a recusive binary style searching
          which can narrow in great time!

        - all socket i/o parsed through a BytesIO block to
          save on the constent allocation strings would normally
          have caused. Content is then further parsed and broken into
          an easy to parse dictionary (stored in a btree) and
          returned for the user for parsing.

        - any filters specified must be of type NNTPFilterBase. If a
          list or tuple is passed in, then each element in the list
          must be inheritied from NNTPFilterBase.
    """
    # 10MB Chunk Sizes read from the stream;  If content is compressed
    # it has the potential to get quite larger.
    MAX_BUFFER_SIZE = 10485760

    # The number of characters to read backwards at a time when scanning
    # for a new line character during situations where we've received
    # partial data we don't want to process yet. A new line delimiter
    # found allows us to safely process what is before it.
    READBACK_CHUNK_SIZE = 128

    def __init__(self, username=None, password=None, secure=False,
                 iostream=NNTPIOStream.RFC3977_GZIP,
                 join_group=False, use_body=False, use_head=True,
                 encoding=None, work_dir=None,
                 filters=None, hooks=None, *args, **kwargs):
        """
        Initialize NNTP Connection

        secure
        ------
        Secure connection with NNTP Server using a SSL/TLS Encryption

        iostream
        --------
        NewsReap was developed in order to communicate with an NNTP server.
        The idea behind the IOStream is to allow us to handle other protocols
        too. At this time, there are only really 2 values to specify here:
           - NNTPIOStream.RFC3977_GZIP : Support GZIP Compression wrapped
                                          around the standard RFC3977 (NNTP)
                                          protocol.

           - NNTPIOStream.RFC3977      : Standard RFC3977 (NNTP) Protocol

        join_group
        ----------
        Some NNTP Servers have gone out of their way to make sure that all of
        the Message-ID's associated with an article is unique which eliminates
        the need to have to select the group prior to retrieving the article.

        It's obviously slower to do this; so if you're Usenet provider
        doesn't need this option; then don't specify it. It 'will' slow down
        processing if you set this!

        use_body
        --------
        This actually refers to quering the NNTP server for it's articles
        via the BODY command over the ARTICLE.  If this is set to true, only
        the body's content is downloaded.  This is faster; however the trade
        off is using the ARTICLE option hands down the Header information too.

        The Header information is really useful for scanning more details about
        the content we're retrieving.  Sometimes the header information can
        provide us enough information to no bother downloading the rest of the
        content.  Hence having this set to false can actually be faster in
        some cases. By default this is set to False so we can acquire as much
        information on the article we're retrieving as we can dispite the
        small overhead that it comes with.
        """

        # get connection mode
        if secure:
            kwargs['secure'] = True
            self.protocol = 'nntps'
        else:
            kwargs['secure'] = False
            self.protocol = 'nntp'

        # Default Filters
        self.filters = []

        # Filter Preparation
        if isinstance(filters, NNTPFilterBase):
            self.filters.append(filters)

        elif isinstance(filters, (list, tuple)):
            self.filters.extend([
                x for x in filters if isinstance(x, NNTPFilterBase)
            ])

        if hooks:
            # Force defined hooks into arguments
            kwargs['hooks'] = hooks

        # Initialize the Socket Base Class
        super(NNTPConnection, self).__init__(*args, **kwargs)

        # Store the default encoding
        self.encoding = encoding
        if not self.encoding:
            self.encoding = NNTP_DEFAULT_ENCODING

        # SSL Does not like unicode characters, so we convert the content
        # to a characterset so we don't get any handshaking errors
        if isinstance(username, unicode):
            username = username.encode(self.encoding)
        self.username = username

        # SSL Does not like unicode characters, so we convert the content
        # to a characterset so we don't get any handshaking errors
        if isinstance(password, unicode):
            password = password.encode(self.encoding)
        self.password = password

        # Attempt to gzip the connection
        if isinstance(iostream, basestring):
            try:
                self._iostream = iostream.lower()
                if self._iostream not in NNTP_SUPPORTED_IO_STREAMS:
                    # Default
                    self._iostream = NNTPIOStream.RFC3977_GZIP
                    logger.warning('An unknown iostream was specified; '
                                   "using default: '%s'." % self._iostream)

            except (TypeError, ValueError):
                # Default
                self._iostream = NNTPIOStream.RFC3977_GZIP
                logger.warning('A malformed iostream was specified; '
                               "using default: '%s'." % self._iostream)

        elif iostream:
            # Default
            logger.warning('An invalid iostream was specified; '
                           "using default: '%s'." % self._iostream)
            self._iostream = NNTPIOStream.RFC3977_GZIP

        else:
            # iostream is None (0, or False)
            # used RFC3977 Standards but without Compresssion
            logger.info('An invalid iostream was specified; '
                        "using default: '%s'." % self._iostream)
            self._iostream = NNTPIOStream.RFC3977

        # can_post is a flag that gets set after authenticating
        # with the news server.  The server responds with the
        # posting information which is set in this flag
        self.can_post = False

        # Article Meta information (Extracted from post)
        self.article = {}

        # Article End Of Data
        # This has 3 states:
        #   * False: The end of the article has not been found; scan and
        #            save what we find. We 'always' start in this state
        #   * None:  The end of the article has not been found; but due
        #            to a problem, we don't want anything else that is
        #            returned by the server.  This flag is usually set
        #            during parsing if CRC errors occur as a way of
        #            letting the server complete what it's doing.
        #   * True:  The end of the article has been found; we are in
        #            a somewhat of completed state
        self.article_eod = False

        # Article Filename
        self.article_fname = None

        # Data that has been successfully read from the
        # _buffer is processed (perhaps uncompressed)
        # and placed into the _data stream.
        self._data = BytesIO()

        # A _data length tracker which is used with
        # trying to track of we've processed all of the data yet
        # or not.
        self._data_len = 0

        # Temporary Buffer of read (unprocessed) data
        self._buffer = BytesIO()

        # Our response code (integer) returned by the NNTPServer on the last
        # request made
        self.last_resp_code = None

        # Our response string returned by the NNTPServer on the last request
        # made
        self.last_resp_str = ''

        # A flag set to True if the payload to follow the response is gzipped.
        # GZipped responses require a bit more CPU crunching but much less
        # bandwidth. This value is set to None if there simply wasn't a payload
        # at all (or an initialization value)
        self.payload_gzipped = None

        self.lines = []
        self.line_count = 0

        self.group_name = None
        self.group_count = 0
        self.group_head = 0
        self.group_tail = 0
        self.group_index = 0

        # use HEAD command vs STAT when retrieving information on an article
        # on usenet.
        self.use_head = use_head

        # Use the BODY command vs ARTICLE when retrieving the data itself.
        # The subtle difference is BODY only returns the raw contents where as
        # ARTICLE returns the header information too.  The header information
        # can provide invaluable information on whether the BODY is even worth
        # retrieving in some cases.  It's purely up to the user which they
        # want to use.  It's also possible that some servers won't support
        # one command over the other.
        self.use_body = use_body

        # Some usenet servers do not require the group to be joined prior to
        # fetching it's content.
        self.join_group = join_group

        # A List of backup servers that attempt to fetch missing content; use
        # append() to append to the list.  The entry must be of type
        # NNTPConnection()
        self._backups = []

        # Used to cache group list responses
        self._grouplist = None

        # Default Working Directory
        # All temporary content is downloaded to this location.  If set to
        # None then the defaults are used instead.
        if work_dir is None:
            self.work_dir = DEFAULT_TMP_DIR
        else:
            self.work_dir = abspath(expanduser(work_dir))

    def append(self, connection, *args, **kwargs):
        """
        Add a backup NNTP Server (Block Account) which is only
        queried in the event content can't be pulled from the
        primary server.
        """
        if not isinstance(connection, NNTPConnection):
            return False

        # block/backup connections are established on demand
        self._backups.append(connection)
        return True

    def connect(self, *args, **kwargs):
        """
        Establishes a connection to an NNTP Server
        """
        if self.connected:
            # nothing to see here
            return True

        # Reset tracking items
        self._soft_reset()

        # initialize our return flag
        status = False

        try:
            # pre_connect hook
            response = self.hooks.call('pre_connect', *args, **kwargs)

            # Allow a response over-ride which forces a denied connection
            if next((r for r in response
                     if r is not False), None) is not False:

                # call _connect()
                status = self._connect(*args, **kwargs)

        except SocketRetryLimit:
            # We can not establish a connection and never will
            return False

        finally:
            self.hooks.call(
                'post_connect',
                status=status,
                conection=weakref.proxy(self)
            )

        return status

    def _connect(self, *args, **kwargs):
        """
        _connect() performs the actual connection with little
        error checking.  This is intended so that if we lose the
        connection we can attempt to resume where we left off
        gracefully.
        """

        try:
            if not super(NNTPConnection, self).connect(*args, **kwargs):
                # Return Fail
                return False

        except (SocketException, SignalCaughtException):
            # Return Fail
            return False

        # Receive Initial Welcome Message
        response = self._recv(timeout=NNTP_WELCOME_MESSAGE_TIMEOUT)
        if response.code not in NNTPResponseCode.SUCCESS:
            logger.error(
                'Failed to establish a handshake response from server.',
            )
            # server error
            self.close()
            return False

        # Tests for specific flags that exist in the welcome message that
        # give us indication on whether or not we can post to usenet or
        # not.
        if POSTING_OK_RE.search(response.code_str):
            # Toggle can_post flag
            logger.info('NNTP Posting enabled.')
            self.can_post = True
        else:
            logger.info('NNTP Posting disabled.')

        # Authenticate
        response = self.send('AUTHINFO USER %s' % self.username)
        if response.code not in NNTPResponseCode.PENDING:
            # We got a 400 or 500 error, no good
            logger.error('The specified username was not accepted.')
            self.close()
            return False

        response = self.send('AUTHINFO PASS %s' % self.password)
        if response.code not in NNTPResponseCode.SUCCESS:
            # We failed to get a 200 error; assume an authentication error
            logger.error('The specified password was not accepted.')
            self.close()
            return False

        logger.info('NNTP USER/PASS Handshake was successful.')

        if self._iostream == NNTPIOStream.RFC3977_GZIP:
            # Do Compression
            response = self.send('XFEATURE COMPRESS GZIP')
            if response.code not in NNTPResponseCode.SUCCESS:
                # Not supported; flip flag
                logger.warning('NNTP Compression not supported.')
                self._iostream = NNTPIOStream.RFC3977
            else:
                logger.info('NNTP Compression enabled.')

        if self.join_group and self.group_name:
            # Change to our group
            self.group(self.group_name)

        return True

    def post(self, payload, update_headers=True, success_only=False):
        """
        Allows posting content to a NNTP Server

        If success_only is set to True, then the result set returned only
        contains the entries of successfully posted content.  otherwise
        the result set returns the actual response code for each
        entry posted.

        """

        # A sorted list of all articles pulled down
        results = sortedset(key=lambda x: x.key())

        postable = []
        if isinstance(payload, (set, tuple, sortedset, list)):
            # iterate over all items and append them to our resultset
            for entry in payload:
                _results = self.post(entry, update_headers=True)
                if _results is not None:
                    # Append our results
                    results |= _results

        elif isinstance(payload, NNTPArticle):
            postable = [payload, ]

        elif isinstance(payload, NNTPSegmentedPost):
            postable = iter(payload)

        # Initialize a bound respose object
        response = None

        for content in postable:
            if content not in results:

                try:
                    weak_content = weakref.proxy(content)

                except TypeError:
                    # Some types just can't be converted into a weak reference
                    # no problem...
                    weak_content = content

                try:
                    # pre hook
                    _response = self.hooks.call(
                        'pre_post',
                        content=weak_content,
                    )

                    # Allow a response over-ride which forces a denied
                    # post
                    if next((r for r in _response
                             if r is not False), None) is not False:

                        response = self._post(content, update_headers=True)

                    else:
                        response = NNTPResponse(
                            NNTPResponseCode.HOOK_OVERRIDE,
                            'NNTP POST action blocked by hook',
                        )

                except SocketException:
                    # Connection Lost
                    self.close()

                    # Setup Response
                    response = NNTPResponse(
                        NNTPResponseCode.CONNECTION_LOST,
                        'Connection Lost',
                    )

                finally:
                    # Point our body to our content
                    response.body = weakref.proxy(content)

                    self.hooks.call(
                        'post_post',
                        content=weak_content,
                        response=weakref.proxy(response),
                        status=response in NNTPResponseCode.SUCCESS,
                    )

                if success_only:
                    if response in NNTPResponseCode.SUCCESS:
                        results.add(response)
                else:
                    # Store all response types
                    results.add(response)

        # Return our results
        return results

    def _post(self, article, update_headers=True):
        """
        Takes an NNTPArticle() and posts it to an NNTP Server if it can

        """

        if not self.connected:
            # Attempt to establish a connection
            if not self.connect():
                logger.error('Could not establish a connetion to NNTP Server.')
                return NNTPResponse(
                    NNTPResponseCode.NO_CONNECTION, 'No Connection',
                )

        if not self.can_post:
            # 480 Transfer permission denied
            return NNTPResponse(480, 'Transfer permission denied.')

        if not isinstance(article, NNTPArticle):
            # Not postable content
                return NNTPResponse(
                    NNTPResponseCode.INVALID_INPUT,
                    'You may only post an NNTPArticle.',
                )

        # get an iterator
        post_iter = article.post_iter(update_headers=update_headers)
        if not post_iter:
            return NNTPResponse(
                NNTPResponseCode.INVALID_INPUT,
                'NNTPArticle() is missing data (not postable).',
            )

        # send the POST keyword to the remote server
        response = self.send('POST')
        if response.code not in NNTPResponseCode.PENDING:
            # We got a 400 or 500 error, no good
            logger.error('Could not post content / %s' % response)
            return response

        # If we reach here we're read to go
        xfer_total_bytes = 0
        for chunk in post_iter:
            xfer_bytes = super(NNTPConnection, self).send(chunk)
            if xfer_bytes != len(chunk):
                # uh-oh
                return NNTPResponse(436, 'Transfer failed.')
            xfer_total_bytes += xfer_bytes

        # Send our EOD and capture our response
        response = self.send(NNTP_EOL + NNTP_EOD)

        return response

    def group(self, name):
        """
        Changes to a specific group

        This function returns a tuple in the format of:

            (count, head, tail, group_name)

        If an error occurs, then the following is returned:

            (None, None, None, group_name)

        """

        if not self.connected:
            # Attempt to establish a connection
            if not self.connect():
                logger.error('Could not establish a connetion to NNTP Server.')
                return None

        if isinstance(name, unicode):
            # content should be byte encoded
            name = name.encode(self.encoding)

        response = self.send('GROUP %s' % name)

        self.group_name = name
        self.group_count = 0
        self.group_head = 0
        self.group_tail = 0
        self.group_index = 0

        if response.code in NNTPResponseCode.SUCCESS:
            match = NNTP_GROUP_RESPONSE_RE.match(response.code_str)
            if match:
                # We matched!
                # return is count, first, end, name
                self.group_count = int(match.group('count'))
                self.group_head = int(match.group('low'))
                self.group_tail = int(match.group('high'))
                self.group_name = match.group('name')
                self.group_index = self.group_head

                logger.info('Using Group: %s.' % self.group_name)
                return (
                    self.group_count,
                    self.group_head,
                    self.group_tail,
                    self.group_name,
                )

        logger.warning('Bad Group: %s' % name)
        return (None, None, None, self.group_name)

    def groups(self, filters=None, lazy=True):
        """
        Retrieves a list of groups from the server and returns them in an easy
        to parse dictionary:

              {
                  group: 'alt.binaries.test',
                  head: 0,
                  tail: 100,
                  count: 100,
                  flags: ['y'],
              }

        If filter(s) are specified, then the list is filtered based on
        the specified entries.

        The filters can be in the form of a string or regular expression.
        If multiple strings are specified (in a tuple or list format) then
        they will all be checked for a match. If fiters are specified then
        at least 1 filter must match.

        To fetch just binaries you could type:
            groups(filters='alt.binaries')

        If lazy is set to true; the last returned fetch is used instead
        which greatly increases the speed when you need to make multiple
        calls to this function. By defaut the lazy flag is enabled.
        """

        if not self.connected:
            # Attempt to establish a connection
            if not self.connect():
                logger.error('Could not establish a connetion to NNTP Server.')
                return None

        # Filter Management
        if not filters:
            filters = tuple()

        elif isinstance(filters, basestring):
            filters = (filters,)

        elif isinstance(filters, re._pattern_type):
            filters = (filters,)

        elif not isinstance(filters, (tuple, list)):
            filters = list(filters)

        # Iterate over each item in the list and compile it now
        _filters = []
        for _filter in filters:
            if isinstance(_filter, re._pattern_type):
                _filters.append(_filter)

            elif isinstance(_filter, basestring):
                try:
                    _filter = r'^.*%s.*$' % re.escape(_filter)
                    _filters.append(
                        re.compile(_filter, flags=re.IGNORECASE),
                    )
                    logger.debug('Compiled group regex "%s"' % _filter)

                except:
                    logger.error(
                        'Invalid group regular expression: "%s"' % _filter,
                    )
            else:
                logger.error(
                    'Ignored group expression: "%s"' % _filter,
                )

        if self._grouplist is None or not lazy:
            # Send LIST ACTIVE command
            response = self.send('LIST ACTIVE', decoders=[
                CodecGroups(work_dir=self.work_dir),
            ])
            if not response.is_success(multiline=True):
                logger.error('Failed to interpret NNTP LIST ACTIVE response.')
                # could not retrieve list
                return None

            # Pop the group list off
            grouplist = response.decoded.pop()
            if not isinstance(grouplist, NNTPMetaContent):
                # could not retrieve list
                logger.error('Failed to fetch NNTP LIST ACTIVE content.')
                return None

            # Lazy Caching for speed
            self._grouplist = grouplist.content
            logger.info('Cached %d group(s)' % len(self._grouplist))

        if _filters:
            cleaned = []
            for x in self._grouplist:
                # Apply filter if nessisary
                if not next((True for f in _filters
                             if f.search(x['group']) is not None), False):
                    # Filters were specified and the matched
                    # group did not match what was specified so
                    # we can move onto the next entry
                    continue

                # if we reach here, then we store our group
                cleaned.append(x)

            logger.info('Retrieved %d (and filtered %d) groups(s)' % (
                len(self._grouplist), len(cleaned),
            ))
            return cleaned

        logger.info('Retrieved %d groups(s)' % (
            len(self._grouplist),
        ))
        return self._grouplist

    def tell(self):
        """
        Returns the current index
        """
        return self.group_index

    def seek_by_date(self, refdate, group=None):
        """
        Similar to the seek() function in the sense it changes the
        current index pointer.  However this one attempts to narrow
        into the index location by a date.

        This function returns the first message id to scan from
        based on the time specified.  If no articles are found
        or something bad happens, false is returned.

        refdate must be a datetime object.

        As a user you don't need to set the last 2 flags. In fact
        it might be more advisable not to unless you know what you're
        doing.  Just call the function by setting the date to what you
        want and let the function manage it's own recursive actions.

            from datetime import datetime
            from datetime import timedelta

            NNTPConnection n(username, password, localhost)
            n.connect(localhost)
            # look back 40 days from 'now'
            n.seek_by_date(datetime.now()-timedelta(days=40))

        Seeking takes on average 15 to 16 queries in a group containing
        a size of 33 million records. So it's pretty optimized in that
        reguard. Here is it's Big-O representation:

            O(log n + log**2 k)
        """

        if group is not None and group != self.group_name:
            # allow us to switch groups if nessisary
            if self.group(group)[0] is None:
                # Could not select group
                logger.error(
                    'DATESEEK Could not select group %s' % group)
                return None

        elif group is not None and self.group_name is None:
            logger.error(
                'DATESEEK XOVER failed due to no group selected.')
            return None

        if not self.connected:
            return None

        index = self._seek_by_date(refdate)
        if index < 0:
            logger.warning(
                'No entries found at (or after) date(%s) in group %s' % (
                    refdate.strftime('%Y.%m.%d %H:%M:%S'),
                    group,
                ),
            )
            self.group_index = self.group_head
            return self.group_head

        self.group_index = index
        logger.info('Matched index: %d' % (self.group_index))
        return self.group_index

    def _seek_by_date(self, refdate, head=None, tail=None):
        """

        This ideally shouldn't be called directly; users should
        call the seek_by_date() function instead

        refdate must be a datetime object.

        How it's done:
        Using recursion; we drill down to a specific location but
        since usenet 'sort-of' sorted by time. Once we get within
        X records (defined by a global variable) we need to scan
        within those results to match our time (or as close as we
        can get to it without going over. Since our content is
        stored in a btree; this is relatively speedy.

        """

        if head is None:
            # Initial Entry
            head = self.group_head

        if tail is None:
            # Initial Entry
            tail = self.group_tail

        logger.debug('Scanning for index at %s in %d article(s)' % (
            refdate.strftime('%Y-%m-%d %H:%M:%S'),
            tail - head,
        ))

        # initalize filter count
        filter_count = tail - head

        # Get the middle
        start = head + (filter_count / 2) - (NNTP_FETCH_BY_DATE_MAX_MISSES / 2)
        end = start + min(NNTP_FETCH_BY_DATE_MAX_MISSES, filter_count)

        response = self.xover(
            start=start,
            end=end - 1,
            sort=XoverGrouping.BY_TIME,
        )

        if response is None:
            # Nothing Retrieved
            return -1
            # if response.code != 423:
            #    # 423 means there were no more items to fetch
            #    # this is a common error that even this class produces
            #    # and therefore we do not need to create a verbose
            #    # message from it.
            #    logger.error('NNTP Server responded %s' % response)
            # return -1

        # Deal with our response
        if len(response):
            # Get string equivalent
            _refdate = refdate.strftime('%Y%m%d%H%M%S:0000000000')

            # Get keys
            # (this returns an blist.sortedset() object): Big O(n)
            # This is 'very' fast operation since we're not dealing with a
            # dict()
            _refkeys = response.keys()

            # logger.debug('total=%d, left=%s, right=%s, ref=%s' % (
            #    end-start,
            #    _refkeys[0],
            #    _refkeys[-1],
            #    _refdate,
            # ))

            #
            # Decisions
            #
            if end - start < NNTP_FETCH_BY_DATE_MAX_MISSES and \
               len(_refkeys) < NNTP_FETCH_BY_DATE_MAX_MISSES:
                # We're in great shape if we reach here
                # It means we've narrowed off our search to just
                # the small window for our match
                return _refkeys.bisect_left(_refdate) + start

            elif _refkeys[-1] < _refdate:
                # look closer to the tail (our right)
                logger.debug('%s > %s / JUMP RIGHT' % (
                    _refdate,
                    _refkeys[-1],
                ))

                if tail - end > 0:
                    result = self._seek_by_date(refdate, end - 1, tail)
                else:
                    result = tail

                while result < 0:
                    # SHIFT LEFT
                    logger.debug('SHIFT LEFT')
                    tail -= ((tail - end) / 2)
                    if tail - end > 0:
                        result = self._seek_by_date(refdate, end - 1, tail)
                    else:
                        result = tail
                        break

                return result

            elif _refkeys[0] >= _refdate:
                # look closer to the front
                logger.debug('%s < %s / JUMP LEFT' % (
                    _refdate,
                    _refkeys[0],
                ))

                if start - head > 1:
                    result = self._seek_by_date(refdate, head, start)
                else:
                    result = start

                while result < 0:
                    # SHIFT RIGHT
                    logger.debug('SHIFT RIGHT')
                    head += ((start - head) / 2)
                    if start - head > 1:
                        result = self._seek_by_date(refdate, head, start)
                    else:
                        result = start
                        break

                return result

        # We recursively scanned too far in one
        # direction; we need to
        return -1

    def seek(self, index, whence=None):
        """
        Sets a default nntp index
        TODO: if wence is set then the index is set with respect
              to the results.

        seek() always returns the current index similar to what tell()
        would return if it was called right afterwards
        """
        if index <= self.group_tail:
            if index >= self.group_head:
                self.group_index = index
            else:
                # Can't be less than the head
                self.group_index = self.group_head
        else:
            # Can't be greater than the tail
            self.group_index = self.group_tail

        return self.group_index

    def next(self, count=50000):
        """
        A wrapper to the xover but using a counter offset based
        on a set index value

        use the seek() function to over-ride the default
        index value (which is usually set to the head of
        group.

        """
        # Control Count gracefully
        count = min(count, self.group_tail - self.group_index)

        if count <= 0:
            # 423: Empty Range
            return NNTPResponse(423, 'Empty Range')

        # Fetch our results and adjust our pointer accordingly
        response = self.xover(self.group_index, self.group_index + count - 1)
        if response:
            self.group_index += count

        # Return results
        return response

    def prev(self, count=50000):
        """
        A wrapper to the xover but using a counter offset based
        on a set index value

        use the seek() function to over-ride the default
        index value (which is usually set to the head of
        group.

        """
        # Control Count gracefully
        count = min(
            count, self.group_tail - self.group_index - self.group_head)

        if count <= 0:
            # 423: Empty Range
            return NNTPResponse(423, 'Empty Range')

        # Fetch our results and adjust our pointer accordingly
        response = self.xover(self.group_index - count, self.group_index - 1)
        if response:
            self.group_index -= count

        # Return results
        return response

    def xover(self, group=None, start=None, end=None,
              sort=XoverGrouping.BY_POSTER_TIME):
        """
        xover
        Returns a NNTPRequest object

        RFC: http://tools.ietf.org/html/rfc2980
        list is a list of tuples, one for each article in the range delimited

        If the start or end time are set to `None` then they default to the
        low/high (head or tail) watermark respectively.

        by the start and end article numbers. Each tuple is of the form
        (article number, subject, poster, date, id, references, size, lines).

        If the start or end time is a datetime variable, then additional
        overhead is performed to find these locations in the group identified.

        Content is parsed as follows:
              {
                  id: u'the unique identifier',
                  article_no: 12345678,
                  poster: u'the poster's information',
                  date: datetime() object,
                  subject: u'a subject line in unicode',
                  size: 2135  // the message size in bytes
                  lines: 53   // the number of lines
                  group: u'alt.group.one'
                  xgroups : {
                      // references the Message-ID (id) per cross post
                       u'alt.group.two': 987654321,
                       u'alt.group.three': 12341234,
                  }
              }
        """

        if group is not None and group != self.group_name:
            # allow us to switch groups if nessisary
            if self.group(group)[0] is None:
                # Could not select group
                logger.error('XOVER Could not select group %s' % group)
                return None

        elif group is not None and self.group_name is None:
            logger.error('XOVER failed due to no group selected.')
            return None

        if end is None:
            end = self.group_tail

        elif isinstance(end, datetime):
            # Get the end by date
            end = self.seek_by_date(end)

        # Default Ranges (if not otherwise specified)
        if start is None:
            start = self.group_head

        elif isinstance(start, datetime):
            # Get the start by date
            start = self.seek_by_date(start)

        if start > end:
            # 423: Empty Range
            logger.error('Invalid XOVER start/end (%d/%d) range.' % (
                start,
                end,
            ))
            return None

        elif not (self.group_head and start >= self.group_head):
            # 423: Empty Range
            logger.error('Invalid XOVER head/start (%d/%d) range.' % (
                self.group_head,
                start,
            ))
            return None

        elif not (self.group_tail and end <= self.group_tail):
            # 423: Empty Range
            logger.error('Invalid XOVER tail/end (%d/%d) range.' % (
                self.group_tail,
                end,
            ))
            return None

        try:
            # pre hook
            _response = self.hooks.call(
                'pre_xover',
                group=group,
                start=start,
                end=end,
            )

            # Allow a response over-ride which forces a denied xover
            if next((r for r in _response
                     if r is not False), None) is not False:

                response = self.send(
                    'XOVER %d-%d' % (start, end),
                    decoders=[
                        CodecArticleIndex(
                            filters=self.filters,
                            sort=sort,
                            encoding=self.encoding,
                            work_dir=self.work_dir,
                        ),
                    ],
                    retries=NNTP_XOVER_RETRIES,
                )

            else:
                response = NNTPResponse(
                    NNTPResponseCode.HOOK_OVERRIDE,
                    'NNTP XOVER action blocked by hook',
                )

        finally:

            try:
                weak_response = weakref.proxy(response)

            except TypeError:
                # Some types just can't be converted into a weak reference
                # no problem...
                weak_response = response

            self.hooks.call(
                'post_xover',
                group=group,
                start=start,
                end=end,
                response=weak_response,
                status=response in NNTPResponseCode.SUCCESS_MULTILINE,
            )

        if response.code not in NNTPResponseCode.SUCCESS_MULTILINE:
            logger.error('Failed to interpret NNTP XOVER response.')
            return None

        try:
            # return our decoded content
            return response.decoded.pop().content

        except IndexError:
            # TODO: Debug; wtf is going on
            return None

    def stat(self, id, full=None, group=None):
        """
        A wrapper to _stat that allows us to manage hooks

        """

        if full is None:
            # default
            full = self.use_head

        if self.join_group and group is not None and group != self.group_name:
            # allow us to switch groups if nessisary
            if self.group(group)[0] is None:
                # Could not select group
                logger.error('Could not select group %s' % group)
                return None

        # Initialize a bound object
        response = None

        try:
            _response = self.hooks.call(
                'pre_stat',
                content=id,
                full=True,
                group=self.group_name,
            )

            # Allow a response over-ride which forces a denied stat
            if next((r for r in _response
                     if r is not False), None) is not False:

                response = self._stat(id=id, full=full)

            else:
                response = NNTPResponse(
                    NNTPResponseCode.HOOK_OVERRIDE,
                    'NNTP STAT action blocked by hook',
                )

        finally:
            # Prepare our status for our hook call
            status = False
            if isinstance(response, NNTPHeader):
                status = response.article_exists()

            try:
                weak_response = weakref.proxy(response)

            except TypeError:
                # Some types just can't be converted into a weak reference
                # no problem...
                weak_response = response

            self.hooks.call(
                'post_stat',
                content=id,
                full=full,
                group=self.group_name,
                response=weak_response,
                status=status,
            )

        # Return our response
        return response

    def _stat(self, id, full=None):
        """
        A Simple check to return True of False on whether an article
        exists or not. The function returns:
            False:  if the article does not exist.
            None:   if there was no way to determine the answer.

            Otherwise the function returns a dictionary containing the
            stats the news server had on the file in question.

            if Full is left to None, then the results will be based
            on the defaut use_head function.
        """

        if not full:
            response = self.send('STAT <%s>' % id)
            if response.is_success(multiline=False):
                # we're good to go, return what we do know so it fits
                results = NNTPHeader()
                results['Message-ID'] = id
                return results

        else:
            # Force decoders to just be the header
            response = self.send(
                'HEAD <%s>' % id,
                decoders=[
                    CodecHeader(
                        encoding=self.encoding,
                        work_dir=self.work_dir,
                    ),
                ],
            )

            if response.is_success(multiline=True):
                # Return our content
                return response.decoded.pop()

        if response.code in NNTPResponseCode.NO_ARTICLE:
            if self._backups:
                # Try our backup servers in the sequential order they were
                # added in; if they all fail; then we return None
                logger.warning(
                    'ARTICLE <%s> not found; checking backups.' % id,
                )
                return next((
                    b.article for b in self._backups
                    if b.stat(id=id, full=full, group=self.group_name)
                    not in (None, False)), False)

            logger.warning('ARTICLE <%s> not found.' % id)
            return False

        elif response.code in NNTPResponseCode.SERVER_ERROR:
            # We have a problem
            self.close()
            logger.error('NNTP Error %s' % response)
            if self._backups:
                # Try our backup servers in the sequential order they were
                # added in; if they all fail; then we return None
                return next((
                    b.article for b in self._backups
                    if b.stat(id=id, full=full, group=self.group_name)
                    not in (None, False)), None)
            return None

        # Return
        return True

    def get(self, id, work_dir=None, decoders=None, group=None, max_bytes=0):
        """
        A wrapper to the _get call allowing support for more then one type
        of object (oppose to just _get() which only accepts the message id

        This function returns the articles as a sortedset() containing the
        downloaded content based on what was passed in.

        max_bytes is set to the number of bytes you want to have received
        before you automatically abort the connection and flag the object as
        having only been partially complete  Use this option when you need to
        inspect the first bytes of a binary file. Set this to zero to download
        the entire thing (this is the default value)

        """
        if work_dir is None:
            # Default
            work_dir = self.work_dir

        if isinstance(id, basestring):
            # We're dealing a Message-ID (Article-ID)
            return self._get(
                id=id,
                work_dir=work_dir,
                decoders=decoders,
                group=group,
                max_bytes=max_bytes,
            )

        # A sorted list of all articles pulled down
        results = sortedset(key=lambda x: x.key())

        if isinstance(id, (set, tuple, sortedset, list)):
            # iterate over all items and append them to our resultset
            for entry in id:
                _results = self._get(
                    id=id.id,
                    work_dir=work_dir,
                    decoders=decoders,
                    group=group,
                    max_bytes=max_bytes,
                )
                if _results is not None:
                    # Append our results
                    results |= _results

        elif isinstance(id, NNTPArticle):
            # Support NNTPArticle Objects if they have an id defined
            if id.id:
                return self._get(
                    id=id.id,
                    work_dir=work_dir,
                    decoders=decoders,
                    group=group,
                    max_bytes=max_bytes,
                )

        elif isinstance(id, NNTPSegmentedPost):
            # Support NNTPSegmentedPost() Objects

            for part_no, _article in enumerate(id):
                # We need to download each article define
                if self.join_group:
                    for group in id.groups:
                        # Try each group
                        article = self._get(
                            id=_article,
                            work_dir=work_dir,
                            decoders=decoders,
                            group=group,
                            max_bytes=max_bytes,
                        )

                        if article:
                            # Found
                            break

                    # If we reach here we failed to fetch the item, so
                    # we'll try the next group
                    logger.warning(
                        'Failed to fetch segment #%.2d (%s)' % (
                            _article.no,
                            _article.filename,
                        ),
                    )

                    # mark our article invalid as part of it's response
                    article._is_valid = False

                else:
                    # Fetch our content
                    article = self._get(
                        id=_article,
                        work_dir=work_dir,
                        decoders=decoders,
                        max_bytes=max_bytes,
                    )

                if article:
                    if len(article) == 0:
                        logger.warning(
                            'No content found in segment #%.2d (%s)' % (
                                _article.no, _article.filename,
                            ),
                        )

                else:
                    # Uh oh, we failed to get anything; so just add
                    # our current article generated from the nzb file.
                    # Mark the object invalid and re-add it
                    logger.warning(
                        'Failed to retrieve segment (%s)' % (
                            id.filename,
                        ),
                    )

                    # mark our article invalid as part of it's response
                    article._is_valid = False

                # Store information from our nzb_article over top of
                # the contents in the new article we retrieved
                _article.load(article)

                # Store our article
                results.add(_article)

        elif isinstance(id, NNTPnzb):
            # We're dealing with an NZB File
            if not id.is_valid():
                return None

            # We iterate over each segment defined int our NZBFile and merge
            # them into 1 file. We do this until we've processed all the
            # segments and we return a list of articles
            for segpost_no, segpost in enumerate(id):
                for article in iter(segpost):
                    _results = self.get(article)
                    if _results is not None:
                        # Append our results
                        results |= _results

        else:
            # Unsupported
            return None

        # Return our results
        return results

    def _get_wrapper(self, id, work_dir, decoders=None, group=None,
                     max_bytes=0):
        """
        A wrapper to the __get() function so that we can capture it
        with our hooks

        """
        # Initialize a bound object
        response = None
        try:
            weak_content = weakref.proxy(id)

        except TypeError:
            # Some types just can't be converted into a weak reference
            # no problem...
            weak_content = id

        try:
            _response = self.hooks.call(
                'pre_get',
                work_dir=work_dir,
                group=group,
                max_bytes=max_bytes,
                content=weak_content,
            )

            # Allow a response over-ride which forces a denied stat
            if next((r for r in _response
                     if r is not False), None) is not False:

                response = self._get(
                    id=id,
                    work_dir=work_dir,
                    group=group,
                    max_bytes=max_bytes,
                )

            else:
                response = NNTPResponse(
                    NNTPResponseCode.HOOK_OVERRIDE,
                    'NNTP GET action blocked by hook',
                )

        finally:
            # Prepare our status for our hook call
            status = False
            if isinstance(response, NNTPArticle):
                status = response.header.article_exists()

            try:
                weak_response = weakref.proxy(response)

            except TypeError:
                # Some types just can't be converted into a weak reference
                # no problem...
                weak_response = response

            self.hooks.call(
                'post_get',
                content=id,
                group=self.group_name,
                response=weak_response,
                status=status,
            )

        # Return our response
        return response

    def _get(self, id, work_dir, decoders=None, group=None, max_bytes=0):
        """
        Download a specified message to the work_dir specified. This function
        returns an NNTPArticle() object if it can.

        None is returned if the content could not be retrieved due to an error
        or DMCA.

        This function will always attempt to retrieve missing content from any
        identified backup servers in the order they were saved.

        max_bytes is set to the number of bytes you want to have received
        before you automatically abort the connection and flag the object as
        having only been partially complete  Use this option when you need to
        inspect the first bytes of a binary file. Set this to zero to download
        the entire thing (this is the default value)

        """

        if self.join_group and group is not None and group != self.group_name:
            # allow us to switch groups if nessisary
            if self.group(group)[0] is None:
                # Could not select group
                logger.error('Could not select group %s' % group)
                return None

        if not isdir(work_dir) and not mkdir(work_dir):
            logger.error('Could not create directory %s' % work_dir)
            return None

        if decoders is None:
            # Prepare a default list of decoders
            decoders = list()
            if not self.use_body:
                # BODY calls pull down header information too
                decoders.append(
                    CodecHeader(encoding=self.encoding, work_dir=work_dir),
                )

            decoders.extend([
                # Yenc Encoder/Decoder
                CodecYenc(work_dir=work_dir, max_bytes=max_bytes),
                # UUEncoder/Decoder
                CodecUU(work_dir=work_dir, max_bytes=max_bytes),
            ])

        elif decoders is False:
            # Disable all decoders
            decoders = list()

        elif isinstance(decoders, CodecBase):
            decoders = [decoders, ]

        if self.use_body:
            # Body returns the contents past the header;  Hence there will be
            # no header information to parse if this is the option used.

            # This is faster, but does not allow for some extra checking
            # we can do.
            response = self.send('BODY <%s>' % id, decoders=decoders)
        else:
            # Article retrives the same content as Body plus the Header
            # too.
            response = self.send('ARTICLE <%s>' % id, decoders=decoders)

        if response.is_success(multiline=True):
            # we're good to go!
            pass

        elif response.code in NNTPResponseCode.NO_ARTICLE:
            logger.warning('ARTICLE <%s> not found.' % id)
            if self._backups:
                # Try our backup servers in the sequential order they were
                # added in; if they all fail; then we return None
                return next(
                    (b.article for b in self._backups
                        if b.get(id, work_dir, group) is not None), None)
            return None

        else:  # response.code in NNTPResponseCode.SERVER_ERROR:
            # Close our connection
            self.close()

            logger.error('NNTP Fetch <%s> / %s' % (id, response))

            if self._backups:
                # Try our backup servers in the sequential order they were
                # added in; if they all fail; then we return None
                return next(
                    (b.article for b in self._backups
                        if b.get(id, work_dir, group) is not None), None)
            return None

        # If we reach here, we have data we can work with; build our article
        # using the content retrieved.
        article = NNTPArticle(id=id, work_dir=work_dir)
        article.load(response)

        # Return the content retrieved
        return article

    def send(self, command, timeout=None, decoders=None, retries=0):
        """
        A Simple wrapper for sending NNTP commands to the server

        send() always returns the result of _recv() an internal function
        used to parse the content returned from the NNTP Server.

        article is only used if a list is passed in and the expected
        results are a multi-line response. It is populated with the
        data returned.
        """

        if not self.connected:
            # Attempt to establish a connection
            if not self.connect():
                logger.error('Could not establish a connetion to NNTP Server.')
                return NNTPResponse(
                    NNTPResponseCode.NO_CONNECTION,
                    'No Connection',
                )

        match = NNTP_AUTHINFO_CHECK.match(command)
        if not match:
            logger.debug('send(%s)' % command)

        else:
            logger.debug('send(AUTHINFO %s ********)' % (
                match.group('key'),
            ))

        total_retries = retries
        while True:
            # Soft reset in preparation for returned results
            self._soft_reset()

            if not super(NNTPConnection, self).send(command + EOL):
                break

            try:
                response = self._recv(timeout=0, decoders=decoders)

            except SocketException:
                # Connection Lost
                self.close()

                # Setup Response
                response = NNTPResponse(
                    NNTPResponseCode.CONNECTION_LOST,
                    'Connection Lost',
                )

            if retries > 0:
                # We have a retry left; depending on the severity of our return
                # code, we may try again
                if response.code in (
                        NNTPResponseCode.FETCH_ERROR,
                        NNTPResponseCode.BAD_RESPONSE,
                        NNTPResponseCode.NO_CONNECTION,
                        NNTPResponseCode.CONNECTION_LOST):
                    # If we reach here; our return code is considered
                    # recoverable;

                    # decrement our retry counter
                    retries -= 1

                    # Reset our decoders for re-use
                    for decoder in decoders:
                        decoder.reset()

                    logger.warning(
                        'Received NNTP error %d; retrying... (%d/%d)' % (
                            response.code,
                            total_retries - retries,
                            total_retries,
                        ))
                    continue

            return response

        return NNTPResponse(
            NNTPResponseCode.BAD_RESPONSE,
            'Invalid Command: "%s"' % command,
        )

    def _recv(self, decoders=None, timeout=None):
        """ Receive data, return #bytes, done, skip

        The response always returns the content in the format of
         (code, data)

        if response is set to an NNTPResponse object, it's populated
        with the data retrieved.
        """

        if not self.connected:
            logger.debug('_recv() %d: %s' % (
                NNTPResponseCode.NO_CONNECTION, 'No Connection',
            ))
            return NNTPResponse(
                NNTPResponseCode.NO_CONNECTION, 'No Connection',
            )

        if self.article_eod is True:
            # We've completed
            return NNTPResponse(self.last_resp_code, self.last_resp_str)

        # Pointers used to help identify the start and end
        # of the datablock read from the News Server
        head_ptr = 0

        # Pointer into the _buffer at the last new line
        # as we don't want to grab content from a line that hasn't
        # completely downloaded yet
        tail_ptr = 0

        # Tracks the size of the buffer
        total_bytes = 0

        # Our response object
        response = None

        # Our decoded result
        decoded = None

        # The maximum allowable bytes we can parse before we abort
        # this is calculated from the decoders
        max_bytes = 0

        if not decoders:
            decoders = []

        elif isinstance(decoders, CodecBase):
            decoders = [decoders, ]

        if len(decoders):
            # If at least 1 decoder is defined, we scan it for it's
            # max_bytes() value which tells us the maximum # of bytes we're
            # expecting before completing the transaction (early).  max_bytes()
            # is set from get() and _get() calls.

            # If we determine the end user has specified a max_bytes value,
            # then we adjust the total buffer read size.  This allows us to
            # force an earlier processing.  The minimum buffer size can never
            # be less than 2048
            max_bytes = max([d.max_bytes() for d in decoders])
            if max_bytes > 0:
                if max_bytes < NNTP_MIN_READ_BUFFER_SIZE:
                    self.MAX_BUFFER_SIZE = NNTP_MIN_READ_BUFFER_SIZE
                else:
                    self.MAX_BUFFER_SIZE = min(
                        [max_bytes, self.MAX_BUFFER_SIZE])

        logger.debug('Read Buffer set to %d bytes' % self.MAX_BUFFER_SIZE)

        #  We track the last codec activated using the codec_active
        #  variable.
        codec_active = None

        while not self.article_eod:
            # This loop really only takes effect if we aren't forced
            # to just wait for a server command line. Otherwise
            # we break out the first second we have data to process

            self._buffer.seek(0, SEEK_END)

            try:
                self._buffer.write(self.read(
                    max_bytes=self.MAX_BUFFER_SIZE - total_bytes,
                    timeout=timeout, retry_wait=None,
                ))

            except (SocketException, SignalCaughtException):
                logger.debug('_recv() Connection Lost')
                return NNTPResponse(
                    NNTPResponseCode.CONNECTION_LOST,
                    'Connection Lost',
                )

            # Some Stats (TODO)
            _bytes = self._buffer.tell() - total_bytes
            total_bytes += _bytes
            logger.debug('_recv() %d byte(s) read.' % (_bytes))

            # # DEBUG START
            # self._buffer.seek(head_ptr, SEEK_SET)
            # logger.debug('Characters "%s"' % \
            #     ", ".join(['0x%0x' % ord(b) for b in self._buffer.read()]))
            # # DEBUG END

            ##################################################################
            #                                                                #
            #  Step: 1: Get the status code from the NNTP Server if we don't #
            #           already have it.                                     #
            #                                                                #
            ##################################################################
            if not self.last_resp_code:

                # Reset our payload flag
                self.payload_gzipped = None

                # Seek to head of buffer
                self._buffer.seek(head_ptr, SEEK_SET)

                # Extract Header Response
                data = self._buffer.readline()

                # Only the first time do we remove the first entry
                # because this is our header
                match = NNTP_RESPONSE_RE.match(data.strip())
                if match:
                    # Generate Reponse code
                    self.last_resp_code = int(match.group('code'))
                    self.last_resp_str = match.group('desc')

                    # Adjust head ptr to end of line
                    head_ptr = self._buffer.tell()

                else:
                    if self.connected:
                        # Generate erroneous reponse code
                        self.last_resp_code = NNTPResponseCode.FETCH_ERROR
                        if len(data):
                            self.last_resp_str = 'Bad Response: (len=%d)' % (
                                len(data),
                            )
                        else:
                            self.last_resp_str = 'Bad Response: <nothing>'

                        # Drop connection
                        self.close()

                    else:
                        self.last_resp_code = NNTPResponseCode.CONNECTION_LOST
                        if len(data):
                            self.last_resp_str = \
                                'Connection Lost: (len=%d)' % (len(data))
                        else:
                            self.last_resp_str = 'Connection Lost: <nothing>'

                    # Toggle End of Article and Flush Results
                    self.article_eod = True

                    logger.debug('_recv() %d: %s' % (
                        self.last_resp_code, self.last_resp_str))

                    # Return our response
                    return NNTPResponse(
                        self.last_resp_code, self.last_resp_str,
                    )

                logger.debug('_recv() %d: %s' % (
                    self.last_resp_code, self.last_resp_str))

                # Detect Compression from NNTP response if possible
                if GZIP_COMPRESS_RE.search(self.last_resp_str):
                    logger.debug('_recv() Detected compressed payload.')
                    # GZIP response type; toggle our flag
                    self.payload_gzipped = True

                else:
                    # Uncompressed response type; toggle our flag
                    self.payload_gzipped = False

            # Non-Multiline responses mean were done now.
            if self.last_resp_code not in NNTPResponseCode.SUCCESS_MULTILINE:
                # We're done; flush results and move on
                self.article_eod = True

                # Safety: just clean anything remaining
                while self.can_read():
                    try:
                        self.read(timeout=timeout, retry_wait=None)

                    except (SocketException, SignalCaughtException):
                        # Connection lost
                        return NNTPResponse(
                            NNTPResponseCode.NO_CONNECTION,
                            'Connection Lost',
                        )

                # Return our response
                return NNTPResponse(self.last_resp_code, self.last_resp_str)

            # Store our can_read() flag
            can_read = self.can_read(3)

            if response is None:
                # Initialize our response object on first iteration
                # through loop
                response = NNTPResponse(
                    self.last_resp_code,
                    self.last_resp_str,
                    work_dir=self.work_dir,
                )

            # We have multi-line code to store fill our buffer before
            # proceeding.
            if can_read and total_bytes < self.MAX_BUFFER_SIZE:
                # Keep storing content until we've either reached the end
                # or filled our buffer
                # logger.debug('_recv() Data pending on server...')
                continue

            ##################################################################
            #                                                                #
            #  Step: 2: Check for END OF DATA (EOD) and END OF LINE (EOL).   #
            #           If we find any; trim them and mark flags accordingly #
            #                                                                #
            ##################################################################

            # Initialize our EOL Flag
            eol = False

            if not can_read and total_bytes > 0:
                # Offset
                offset = min(5, total_bytes)

                # The below accomplishes the same results as abs()
                # but does it at a much faster speed (screw looking pythonic)
                if offset > 0:
                    offset = -offset

                self._buffer.seek(offset, SEEK_END)

                # Now we read what we can from the buffer
                data = self._buffer.read()

                if offset == -5:
                    # Adjust offset if nessisary
                    offset += 1

                # Check for the End of Data
                eod_results = EOD_RE.search(data)
                if eod_results:
                    # We can truncate here to trim the EOD off
                    self._buffer.truncate(
                        total_bytes - len(eod_results.group(1)),
                    )

                    # Correct Total Length
                    total_bytes = self._buffer.seek(0, SEEK_END)

                    # Toggle flag so we can break out
                    self.article_eod = True

                    # Toggle our end of line flag incase it doesn't
                    # match below (this is okay)
                    eol = True

                    # Update Offset
                    offset = min(2, total_bytes)

                    # The below accomplishes the same results as abs()
                    # does but at a much faster speed (screw looking
                    # pythonic)
                    if offset > 0:
                        offset = -offset

                    self._buffer.seek(offset, SEEK_END)

                    # Now we read what we can from the buffer
                    data = self._buffer.read()

                # Check for end of line
                eol_results = EOL_RE.search(data)
                if eol_results:
                    # We can truncate here to trim the EOL off
                    self._buffer.truncate(
                        total_bytes - len(eol_results.group(1)),
                    )

                    # Correct Total Length
                    total_bytes = self._buffer.seek(0, SEEK_END)

                    # We have our end of line
                    eol = True

                    if self.last_resp_code in NNTPResponseCode.SUCCESS \
                       and not self.can_read():
                        # If we're still not seeing more data pending on the
                        # server; then we can safely assume we're done. This
                        # handles the situations where we retrieved a
                        # multi-line response code yet we didn't get one.
                        #
                        # the GROUP NNTP server command is known to return 211
                        # which just returns 1 line, yet the command LISTGROUP
                        # also returns 211 too and returns a listing
                        self.article_eod = True

            ##################################################################
            #                                                                #
            #  Step: 3: We now have to handle situations where we never      #
            #           found the EOD or EOL markers. This is a perfectly    #
            #           okay situation so long as our buffer is full.        #
            #                                                                #
            #           If our buffer is full, then we need to mark the tail #
            #           of our buffer because we don't want to handle the    #
            #           content that hasn't completely downloaded yet.       #
            #                                                                #
            ##################################################################

            # the tail_ptr is used to track the last officially completed line
            # as we don't want to process a line that hasn't completely
            # downloaded yet.
            tail_ptr = total_bytes

            if not self.payload_gzipped and \
               total_bytes >= self.MAX_BUFFER_SIZE and not eol:
                # We have a full buffer and there is most likely
                # a lot more content to still download; we need to find
                # the last new line

                # Number of characters to look back (chunk):
                chunk_size = self.READBACK_CHUNK_SIZE

                # Current Matched offset (<0 means no match yet)
                offset = -1

                # Our reference pointer
                ref_ptr = tail_ptr - head_ptr
                while offset < 0:
                    # Keep iterating backwards by chunk sizes
                    # until we get a match

                    # a few checks to make sure we don't look past
                    # our head_ptr
                    chunk_size = min(ref_ptr - head_ptr, chunk_size)
                    ref_ptr -= (chunk_size + 1)

                    if ref_ptr < head_ptr:
                        # Nothing more to look back at; we reached
                        # the head of our buffer. do nothing; just let
                        # process all of the data we received.
                        break

                    chunk_ptr = self._buffer.seek(ref_ptr, SEEK_SET)
                    offset = self._buffer.read(chunk_size).rfind('\n')
                    if offset >= 0:
                        # We found the new line; safe our offset and break
                        tail_ptr = offset + chunk_ptr
                        break

            if not self.payload_gzipped and (tail_ptr - head_ptr) > 0 \
                    and total_bytes < self.MAX_BUFFER_SIZE \
                    and tail_ptr < self.MAX_BUFFER_SIZE and self.can_read(1):
                # Astraweb is absolutely terrible for sending a little
                # bit more data a few seconds later. This is a final
                # call to try to handle these stalls just before the last
                # few bytes are sent.
                continue

            # Ensure we're pointing at the head of our buffer
            self._buffer.seek(head_ptr, SEEK_SET)

            # Compression Support
            if (tail_ptr - head_ptr) > 0:
                if self.payload_gzipped is True:

                    dc_obj = decompressobj()
                    try:
                        self._data.write(dc_obj.decompress(
                            self._buffer.read(tail_ptr - head_ptr),
                        ))
                        logger.debug("NNTP ZLIB decompression successful.")

                    except ZlibException:
                        # Decompression error; since compression is only used
                        # when retrieving server-side listings; it's best to
                        # just alert the end user and move along
                        logger.error(
                            '_recv() %d byte(s) ZLIB decompression failure.'
                            % (_bytes),
                        )

                        # Convert our response to that of an response Fetch
                        # Error
                        return NNTPResponse(
                            NNTPResponseCode.FETCH_ERROR,
                            'Fetch Error',
                        )
                else:
                    # No compression
                    self._data.write(self._buffer.read(tail_ptr - head_ptr))

                # Truncate our original buffer by striping out what was already
                # processed from it
                self._buffer = BytesIO(self._buffer.read())

                # Correct Total Length
                total_bytes = self._buffer.tell()

            ##################################################################
            #                                                                #
            #  Step: 4: This step processes the extra content founds from    #
            #           the NNTP stream based on the decoders passed in.     #
            #                                                                #
            ##################################################################

            self._data_len = self._data.seek(0, SEEK_END)
            d_head = self._data.seek(0, SEEK_SET)
            while d_head < self._data_len and self.connected:

                if codec_active is None:
                    # Scan our decoders (sequentially) and detect our match
                    # If we get a match, then we want to save it in the
                    # codec_active variable. We use this to track the data
                    # found for processing.

                    # Get our data
                    d_head = self._data.tell()
                    data = self._data.readline()

                    # This line scans the line of data we read and determines
                    # what kind of data it is (yEnc, Headers, etc) based on the
                    # decoders passed into _recv()
                    codec_active = next(
                        (d for d in decoders
                            if d.detect(data) is not None), None)
                    if codec_active:
                        logger.debug('Decoding using %s' % type(codec_active))

                # Based on previous check; we may actually have an active codec
                # now if we don't have one yet; well want to store the content
                # into our body and move along
                if codec_active is None:
                    # All data matched that no decoder took ownership off is
                    # saved into our body
                    response.body.write(data)

                    # Update d_head value
                    d_head = self._data.tell()

                    # Track lines processed
                    self.line_count += 1
                    continue

                # Adjust pointer for processing
                self._data.seek(d_head, SEEK_SET)

                # Begin decoding content
                decoded = codec_active.decode(self._data)

                # Adjust our pointer
                d_head = self._data.tell()

                # If we're at the end of our buffer; we can go ahead and clear
                # it
                if d_head >= self._data_len:
                    # Reset our data object once we're done parsing
                    self._data.truncate(0)
                    # Adjust pointer for processing
                    self._data.seek(0, SEEK_SET)

                # A little Decoder 101 for anyone reading my code; the below
                # identifies the possible return types from a Codec
                # (specifically the Decoder):
                #
                #     - NNTPContent:  We're done with the decoder, everything
                #                     was correctly processed. we're handled
                #                     and NNTPContent object.
                #
                #     - True:         We're still expecting more content
                #                     before we're finished, don't adjust
                #                     this from being the 'active' decoder!
                #
                #     - False:        Oh boy; we had a problem and we
                #                     couldn't deal with it. We're finished
                #                     with the decoder; the data is bad.
                #
                #     - None:         A graceful way of saying that we're done
                #                     with the decoder. Like an abort if you
                #                     will
                #
                if decoded is None:
                    # The Codec has completed and has nothing to return for
                    # storing. We gracefully move along at this point.
                    logger.debug(
                        'Decoding complete (no results) / %s' % codec_active,
                    )

                    # we're done; do nothing more
                    codec_active = None

                    continue

                elif decoded is True:
                    # We're expecting more data a long as the End of Data
                    # (EOD) flag hasn't been picked up.
                    if not self.article_eod:
                        logger.debug(
                            'Expecting more data to build results with...',
                        )
                        continue

                    # If we reach here, we've reached the end of the line
                    # a half (or possibly full block of data), but we just
                    # need to close off what we have. fall through and
                    # handle our codec; the HEAD call for example doesn't
                    # return an empty line after the the header like expected
                    # so it's normal to reach here.  Corrupted stuff still
                    # needs to be saved; and we can hope the par files can
                    # rebuild it if it is infact damaged.
                    if isinstance(codec_active, CodecBase):
                        # Store our decoded content (complete or not)
                        decoded = codec_active.decoded

                # If we got here, our content was good; we can safely
                # toggle our codec_active back to off since we're going to
                # be expecting more data now
                logger.debug('Decoding completed. %s' % codec_active)
                codec_active = None

                if not isinstance(decoded, NNTPContent):
                    # We ignore any other return type, Decoders should always
                    # return an NNTPContent type; anything else is considered
                    # moot
                    continue

                # Add to our NNTPContent() to our decoded set associated with
                # our NNTPResponse() object
                response.decoded.add(decoded)

                if not isinstance(decoded, NNTPMetaContent):
                    # Print a representative string into the body to identify
                    # the content parsed out (and decoded)

                    if max_bytes > 0:
                        # We're instructed to only retrieve 'some' content and
                        # abort for peaking purposes; so we're done now.
                        self.close()
                        return response

        # Track lines processed
        self.line_count += len(self.lines)

        logger.debug('Returning Response %s' % response)
        return response

    def close(self):
        """
        Drop the connection
        """

        # Reset our other flags
        self.group_name = None
        self.group_count = 0
        self.group_head = 0
        self.group_tail = 0
        self.group_index = 0

        # Reset Can Post Flag (if set)
        self.can_post = False

        if self.connected:
            logger.debug('Closing Connection')
            try:
                # Prevent Recursion by calling parent send()
                super(NNTPConnection, self).send('QUIT' + EOL)
            except:
                pass

        # SocketBase close() tries to grab the last
        # content on the socket before closing it
        return super(NNTPConnection, self).close()

    def _soft_reset(self):
        """
        Reset tracking items
        """
        self.article = {}
        self.article_eod = False
        self.article_fname = None

        # Reset our Buffers
        self._buffer.truncate(0)
        self._data.truncate(0)
        self._data_len = 0

        # Our response code (integer) returned by the NNTPServer on the last
        # request made
        self.last_resp_code = None

        # Our response string returned by the NNTPServer on the last request
        # made
        self.last_resp_str = ''

        # A flag set to True if the payload to follow the response is gzipped.
        # GZipped responses require a bit more CPU crunching but much less
        # bandwidth. This value is set to None if there simply wasn't a payload
        # at all (or an initialization value)
        self.payload_gzipped = None

        self.lines = []
        self.line_count = 0

        # Empty the receive buffer if anything is pending
        while self.can_read():
            try:
                self.read()

            except (SocketException, SignalCaughtException):
                pass

    def _hard_reset(self, wait=True):
        """
        Drop the connection and re-establish it
        """
        self.close()
        self.connect()

    def __del__(self):
        """
        Handle Deconstruction
        """
        self.close()

    def __str__(self):
        return '%s://%s@%s:%d' % (
            self.protocol, self.username, self.host, self.port)

    def __unicode__(self):
        return u'%s://%s@%s:%d' % (
            self.protocol, self.username, self.host, self.port)

    def __repr__(self):
        """
        Return a printable object
        """
        return '<NNTPConnection id=%d url="%s://%s@%s:%d" />' % (
            id(self),
            self.protocol,
            self.username,
            self.host,
            self.port,
        ) SQLITE_DATABASE_EXTENSION = '.db' def mkdir(name, perm=0775):
    """
    A more contained wrapper to directory management
    """
    attempt = 3
    if isdir(name):
        return True

    while attempt > 0:
        try:
            makedirs(name, perm)
            logger.debug('Created directory: %s' % name)
            return True

        except OSError, e:
            if e[0] == errno.EEXIST:
                # directory exists; this is okay
                return isdir(name)

            logger.debug('Created directory %s exception: %s' % (
                name, e,
            ))

        # racing condition; just try again
        attempt -= 1

    # To many attempts... fail
    # ... fall through...
    return False
import logging
import click
import sys
from os.path import join
from os.path import isdir
from os.path import exists
from os.path import dirname
from os.path import abspath
from os import rename
from os import unlink
from os import access
from os import W_OK
from datetime import datetime
from datetime import timedelta
from dateutil.parser import parse
from shutil import copyfile as copy
from shutil import move
    from newsreap.Logging import NEWSREAP_CLI
    from newsreap.Logging import NEWSREAP_CLI
from newsreap.objects.nntp.Group import Group
from newsreap.objects.nntp.GroupAlias import GroupAlias
from newsreap.objects.nntp.GroupTrack import GroupTrack
from newsreap.objects.nntp.Server import Server
from newsreap.objects.nntp.Common import get_groups
from newsreap.objects.group.Article import Article
from sqlalchemy.exc import OperationalError
from sqlalchemy.exc import InvalidRequestError
from sqlalchemy.exc import IntegrityError
from newsreap.NNTPConnection import XoverGrouping
from newsreap.NNTPGroupDatabase import NNTPGroupDatabase
from newsreap.NNTPConnection import NNTPConnection
from newsreap.NNTPSettings import SQLITE_DATABASE_EXTENSION
from newsreap.Utils import mkdir
@click.option('--date-to', '-t', help='Date To')
@click.option('--watched', '-w', is_flag=True, help='All watched groups.')
@click.pass_obj
def update_search(ctx, groups, date_from, date_to, watched):
    """
    Cache specified articles.

    Articles are cached into their own database due to the sheer size of
    the content within each group.

    """
    # TODO: Support loading by date ranges (from and to)
    # TODO: Support loading by X articles from front or X articles from back
    # TODO: Support loading data by X days back
    # TODO: Support loading data by X days back
    # TODO: Support specifing how many entries to process. Hence if someone
    #       only does a --count=1 (or -c), then only 1 batch is processed.
    #       Support specifying the batch sizes otherwise we use the config
    #       file which is already in place --batch (or -b).
    # TODO: GroupTrack needs to be smarter and not block until the fetch
    #       is repeated on a failure. Each batch loaded should update the
    #       main database and track it's successful fetch. If it's fetch
    #       runs into another, then the 2 tables can be combined into a larger
    #       one.
    #
    #       GroupIndex example:
    #           The below is what the table might look like; you can see we
    #           successfully loaded 100 to 199, and 300 to 399
    #           a.b.test.group:
    #               <id>  <low>   <high>
    #                 1    100      199
    #                 1    300      399
    #
    #           If we fill the void (200 to 299), the table should restructure
    #           itself to look like this:
    #               <id>  <low>   <high>
    #                 1    100      399
    #
    #           Basically the more entries in this table, the more holes/gaps
    #           we have, but we can use this to adjust our batches when we
    #           collide with content that is already fetched.  If a reset
    #           switch is specified (or a reset is detected because the
    #           database is missing, then this table should be included in
    #           the reset too!!)
    #
    # TODO: Support a --force (-f) switch which forces a re-fetch of the
    #       specified ranges defined that override the GroupIndex table.
    #
    # Use our Database first if it exists
    session = ctx['NNTPSettings'].session()
    if not session:
        logger.error('Could not acquire a database connection.')
        exit(1)

    if not len(ctx['NNTPSettings'].nntp_servers) > 0:
        logger.error("There are no servers defined.")
        exit(1)

    if date_from:
        try:
            date_from = parse(date_from, fuzzy=True)

        except TypeError:
            logger.error(
                "An invalid from date/time was specified: %s" % str(date_from),
            )
            exit(1)

    if date_to:
        try:
            date_to = parse(date_to, fuzzy=True)

        except TypeError:
            logger.error(
                "An invalid to date/time was specified: %s" % str(date_to),
            )
            exit(1)

    if date_to and date_from and date_from > date_to:
        logger.error(
            "The from date can not be larger then the to date.",
        )
        exit(1)

    # Store Primary server
    s = ctx['NNTPSettings'].nntp_servers[0]
    try:
        _server = session.query(Server)\
            .filter(Server.host == s['host'])\
            .filter(Server.port == s['port']).first()

    except (InvalidRequestError, OperationalError):
        # Database isn't set up
        logger.error("The database is not correctly configured.")
        exit(1)

    if not _server:
        logger.error("Server entry is not in the database.")
        exit(1)

    groups = get_groups(session=session, lookup=groups, watched=watched)
    if not groups:
        logger.error("There were not groups identified for indexing.")
        exit(1)

    # Get our RamDisk if we got one
    ramdisk = ctx['NNTPSettings'].nntp_processing.get('ramdisk')
    if ramdisk:
        if not (isdir(ramdisk) and access(ramdisk, W_OK)):
            logger.warning('Ramdisk "%s" is not accessible.' % (ramdisk))
            # Turn it off so we don't reference it
            ramdisk = None
        else:
            logger.info('Using ramdisk: %s' % (ramdisk))

    for name, _id in groups.iteritems():

        db_path = join(ctx['NNTPSettings'].cfg_path, 'cache', 'search')
        db_file = '%s%s' % (
            join(db_path, name),

            SQLITE_DATABASE_EXTENSION,