arthur/jobs.py
def run(self, backend_args, archive_args=None, resume=False):
        """Run the backend with the given parameters.

        The method will run the backend assigned to this job,
        storing the fetched items in a Redis queue. The ongoing
        status of the job, can be accessed through the property
        `result`. When `resume` is set, the job will start from
        the last execution, overwriting 'from_date' and 'offset'
        parameters, if needed.

        Setting to `True` the parameter `fetch_from_archive`, items can
        be fetched from the archive assigned to this job.

        Any exception during the execution of the process will
        be raised.

        :param backend_args: parameters used to un the backend
        :param archive_args: archive arguments
        :param resume: fetch items starting where the last
            execution stopped
        """
        args = backend_args.copy()

        if archive_args:
            self.initialize_archive_manager(archive_args['archive_path'])

        if not resume:
            max_date = backend_args.get('from_date', None)
            offset = backend_args.get('offset', None)

            if max_date:
                max_date = datetime_to_utc(max_date).timestamp()

            self._result = JobResult(self.job_id, self.task_id, self.backend, self.category,
                                     None, max_date, 0, offset=offset,
                                     nresumed=0)
        else:
            if self.result.max_date:
                args['from_date'] = unixtime_to_datetime(self.result.max_date)
            if self.result.offset:
                args['offset'] = self.result.offset
            self._result.nresumed += 1

        for item in self._execute(args, archive_args):
            self.conn.rpush(self.qitems, pickle.dumps(item))

            self._result.nitems += 1
            self._result.last_uuid = item['uuid']

            if not self.result.max_date or self.result.max_date < item['updated_on']:
                self._result.max_date = item['updated_on']
            if 'offset' in item:
                self._result.offset = item['offset']
def run(self, backend_args, archive_args=None, resume=False):
        args = backend_args.copy()

        if archive_args:
            self.initialize_archive_manager(archive_args['archive_path'])

        if not resume:
            max_date = backend_args.get('from_date', None)
            offset = backend_args.get('offset', None)

            if max_date:
                max_date = datetime_to_utc(max_date).timestamp()

            self._result = JobResult(self.job_id, self.task_id, self.backend, self.category,
                                     None, max_date, 0, offset=offset,
                                     nresumed=0)
        else:
            if self.result.max_date:
                args['from_date'] = unixtime_to_datetime(self.result.max_date)
            if self.result.offset:
                args['offset'] = self.result.offset
            self._result.nresumed += 1

        for item in self._execute(args, archive_args):
            self.conn.rpush(self.qitems, pickle.dumps(item))

            self._result.nitems += 1
            self._result.last_uuid = item['uuid']

            if not self.result.max_date or self.result.max_date < item['updated_on']:
                self._result.max_date = item['updated_on']
            if 'offset' in item:
                self._result.offset = item['offset']
def run ( self , backend_args , archive_args = None , resume = False ) : args = backend_args . copy ( ) if archive_args : self . initialize_archive_manager ( archive_args [ 'archive_path' ] ) if not resume : max_date = backend_args . get ( 'from_date' , None ) offset = backend_args . get ( 'offset' , None ) if max_date : max_date = datetime_to_utc ( max_date ) . timestamp ( ) self . _result = JobResult ( self . job_id , self . task_id , self . backend , self . category , None , max_date , 0 , offset = offset , nresumed = 0 ) else : if self . result . max_date : args [ 'from_date' ] = unixtime_to_datetime ( self . result . max_date ) if self . result . offset : args [ 'offset' ] = self . result . offset self . _result . nresumed += 1 for item in self . _execute ( args , archive_args ) : self . conn . rpush ( self . qitems , pickle . dumps ( item ) ) self . _result . nitems += 1 self . _result . last_uuid = item [ 'uuid' ] if not self . result . max_date or self . result . max_date < item [ 'updated_on' ] : self . _result . max_date = item [ 'updated_on' ] if 'offset' in item : self . _result . offset = item [ 'offset' ]
Run the backend with the given parameters.

        The method will run the backend assigned to this job,
        storing the fetched items in a Redis queue. The ongoing
        status of the job, can be accessed through the property
        `result`. When `resume` is set, the job will start from
        the last execution, overwriting 'from_date' and 'offset'
        parameters, if needed.

        Setting to `True` the parameter `fetch_from_archive`, items can
        be fetched from the archive assigned to this job.

        Any exception during the execution of the process will
        be raised.

        :param backend_args: parameters used to un the backend
        :param archive_args: archive arguments
        :param resume: fetch items starting where the last
            execution stopped
Run the backend with the given parameters .