perceval/backends/core/mbox.py
def _fetch_and_parse_messages(self, mailing_list, from_date):
        """Fetch and parse the messages from a mailing list"""

        from_date = datetime_to_utc(from_date)

        nmsgs, imsgs, tmsgs = (0, 0, 0)

        for mbox in mailing_list.mboxes:
            tmp_path = None

            try:
                tmp_path = self._copy_mbox(mbox)

                for message in self.parse_mbox(tmp_path):
                    tmsgs += 1

                    if not self._validate_message(message):
                        imsgs += 1
                        continue

                    # Ignore those messages sent before the given date
                    dt = str_to_datetime(message[MBox.DATE_FIELD])

                    if dt < from_date:
                        logger.debug("Message %s sent before %s; skipped",
                                     message['unixfrom'], str(from_date))
                        tmsgs -= 1
                        continue

                    # Convert 'CaseInsensitiveDict' to dict
                    message = self._casedict_to_dict(message)

                    nmsgs += 1
                    logger.debug("Message %s parsed", message['unixfrom'])

                    yield message
            except (OSError, EOFError) as e:
                logger.warning("Ignoring %s mbox due to: %s", mbox.filepath, str(e))
            except Exception as e:
                if tmp_path and os.path.exists(tmp_path):
                    os.remove(tmp_path)
                raise e
            finally:
                if tmp_path and os.path.exists(tmp_path):
                    os.remove(tmp_path)

        logger.info("Done. %s/%s messages fetched; %s ignored",
                    nmsgs, tmsgs, imsgs)
def _fetch_and_parse_messages(self, mailing_list, from_date):
        from_date = datetime_to_utc(from_date)

        nmsgs, imsgs, tmsgs = (0, 0, 0)

        for mbox in mailing_list.mboxes:
            tmp_path = None

            try:
                tmp_path = self._copy_mbox(mbox)

                for message in self.parse_mbox(tmp_path):
                    tmsgs += 1

                    if not self._validate_message(message):
                        imsgs += 1
                        continue

                    # Ignore those messages sent before the given date
                    dt = str_to_datetime(message[MBox.DATE_FIELD])

                    if dt < from_date:
                        logger.debug("Message %s sent before %s; skipped",
                                     message['unixfrom'], str(from_date))
                        tmsgs -= 1
                        continue

                    # Convert 'CaseInsensitiveDict' to dict
                    message = self._casedict_to_dict(message)

                    nmsgs += 1
                    logger.debug("Message %s parsed", message['unixfrom'])

                    yield message
            except (OSError, EOFError) as e:
                logger.warning("Ignoring %s mbox due to: %s", mbox.filepath, str(e))
            except Exception as e:
                if tmp_path and os.path.exists(tmp_path):
                    os.remove(tmp_path)
                raise e
            finally:
                if tmp_path and os.path.exists(tmp_path):
                    os.remove(tmp_path)

        logger.info("Done. %s/%s messages fetched; %s ignored",
                    nmsgs, tmsgs, imsgs)
def _fetch_and_parse_messages ( self , mailing_list , from_date ) : from_date = datetime_to_utc ( from_date ) nmsgs , imsgs , tmsgs = ( 0 , 0 , 0 ) for mbox in mailing_list . mboxes : tmp_path = None try : tmp_path = self . _copy_mbox ( mbox ) for message in self . parse_mbox ( tmp_path ) : tmsgs += 1 if not self . _validate_message ( message ) : imsgs += 1 continue # Ignore those messages sent before the given date dt = str_to_datetime ( message [ MBox . DATE_FIELD ] ) if dt < from_date : logger . debug ( "Message %s sent before %s; skipped" , message [ 'unixfrom' ] , str ( from_date ) ) tmsgs -= 1 continue # Convert 'CaseInsensitiveDict' to dict message = self . _casedict_to_dict ( message ) nmsgs += 1 logger . debug ( "Message %s parsed" , message [ 'unixfrom' ] ) yield message except ( OSError , EOFError ) as e : logger . warning ( "Ignoring %s mbox due to: %s" , mbox . filepath , str ( e ) ) except Exception as e : if tmp_path and os . path . exists ( tmp_path ) : os . remove ( tmp_path ) raise e finally : if tmp_path and os . path . exists ( tmp_path ) : os . remove ( tmp_path ) logger . info ( "Done. %s/%s messages fetched; %s ignored" , nmsgs , tmsgs , imsgs )
Fetch and parse the messages from a mailing list
Fetch and parse the messages from a mailing list