connectordb/logger.py
def sync(self):
        """Attempt to sync with the ConnectorDB server"""
        logging.debug("Logger: Syncing...")
        failed = False
        try:
            # Get the connectordb object
            cdb = self.connectordb

            # Ping the database - most connection errors will happen here
            cdb.ping()

            with self.synclock:
                c = self.database.cursor()
                for stream in self.streams:
                    s = cdb[stream]

                    c.execute(
                        "SELECT * FROM cache WHERE stream=? ORDER BY timestamp ASC;",
                        (stream, ))
                    datapointArray = []
                    for dp in c.fetchall():
                        datapointArray.append(
                            {"t": dp[1],
                             "d": json.loads(dp[2])})

                    # First, check if the data already inserted has newer timestamps,
                    # and in that case, assume that there was an error, and remove the datapoints
                    # with an older timestamp, so that we don't have an error when syncing
                    if len(s) > 0:
                        newtime = s[-1]["t"]
                        while (len(datapointArray) > 0 and datapointArray[0]["t"] < newtime):
                            logging.debug("Datapoint exists with older timestamp. Removing the datapoint.")
                            datapointArray = datapointArray[1:]

                    if len(datapointArray) > 0:
                        logging.debug("%s: syncing %i datapoints" %
                                      (stream, len(datapointArray)))

                        while (len(datapointArray) > DATAPOINT_INSERT_LIMIT):
                            # We insert datapoints in chunks of a couple
                            # thousand so that they fit in the insert size
                            # limit of ConnectorDB
                            s.insert_array(
                                datapointArray[:DATAPOINT_INSERT_LIMIT])

                            # Clear the written datapoints
                            datapointArray = datapointArray[
                                DATAPOINT_INSERT_LIMIT:]

                            # If there was no error inserting, delete the
                            # datapoints from the cache
                            c.execute(
                                "DELETE FROM cache WHERE stream=? AND timestamp <?",
                                (stream, datapointArray[0]["t"]))

                        s.insert_array(datapointArray)

                        # If there was no error inserting, delete the
                        # datapoints from the cache
                        c.execute(
                            "DELETE FROM cache WHERE stream=? AND timestamp <=?",
                            (stream, datapointArray[-1]["t"]))
                self.lastsynctime = time.time()

                if self.onsync is not None:
                    self.onsync()
        except Exception as e:
            # Handle the sync failure callback
            falied = True
            reraise = self.syncraise
            if self.onsyncfail is not None:
                reraise = self.onsyncfail(e)
            if reraise:
                raise
def sync(self):
        logging.debug("Logger: Syncing...")
        failed = False
        try:
            # Get the connectordb object
            cdb = self.connectordb

            # Ping the database - most connection errors will happen here
            cdb.ping()

            with self.synclock:
                c = self.database.cursor()
                for stream in self.streams:
                    s = cdb[stream]

                    c.execute(
                        "SELECT * FROM cache WHERE stream=? ORDER BY timestamp ASC;",
                        (stream, ))
                    datapointArray = []
                    for dp in c.fetchall():
                        datapointArray.append(
                            {"t": dp[1],
                             "d": json.loads(dp[2])})

                    # First, check if the data already inserted has newer timestamps,
                    # and in that case, assume that there was an error, and remove the datapoints
                    # with an older timestamp, so that we don't have an error when syncing
                    if len(s) > 0:
                        newtime = s[-1]["t"]
                        while (len(datapointArray) > 0 and datapointArray[0]["t"] < newtime):
                            logging.debug("Datapoint exists with older timestamp. Removing the datapoint.")
                            datapointArray = datapointArray[1:]

                    if len(datapointArray) > 0:
                        logging.debug("%s: syncing %i datapoints" %
                                      (stream, len(datapointArray)))

                        while (len(datapointArray) > DATAPOINT_INSERT_LIMIT):
                            # We insert datapoints in chunks of a couple
                            # thousand so that they fit in the insert size
                            # limit of ConnectorDB
                            s.insert_array(
                                datapointArray[:DATAPOINT_INSERT_LIMIT])

                            # Clear the written datapoints
                            datapointArray = datapointArray[
                                DATAPOINT_INSERT_LIMIT:]

                            # If there was no error inserting, delete the
                            # datapoints from the cache
                            c.execute(
                                "DELETE FROM cache WHERE stream=? AND timestamp <?",
                                (stream, datapointArray[0]["t"]))

                        s.insert_array(datapointArray)

                        # If there was no error inserting, delete the
                        # datapoints from the cache
                        c.execute(
                            "DELETE FROM cache WHERE stream=? AND timestamp <=?",
                            (stream, datapointArray[-1]["t"]))
                self.lastsynctime = time.time()

                if self.onsync is not None:
                    self.onsync()
        except Exception as e:
            # Handle the sync failure callback
            falied = True
            reraise = self.syncraise
            if self.onsyncfail is not None:
                reraise = self.onsyncfail(e)
            if reraise:
                raise
def sync ( self ) : logging . debug ( "Logger: Syncing..." ) failed = False try : # Get the connectordb object cdb = self . connectordb # Ping the database - most connection errors will happen here cdb . ping ( ) with self . synclock : c = self . database . cursor ( ) for stream in self . streams : s = cdb [ stream ] c . execute ( "SELECT * FROM cache WHERE stream=? ORDER BY timestamp ASC;" , ( stream , ) ) datapointArray = [ ] for dp in c . fetchall ( ) : datapointArray . append ( { "t" : dp [ 1 ] , "d" : json . loads ( dp [ 2 ] ) } ) # First, check if the data already inserted has newer timestamps, # and in that case, assume that there was an error, and remove the datapoints # with an older timestamp, so that we don't have an error when syncing if len ( s ) > 0 : newtime = s [ - 1 ] [ "t" ] while ( len ( datapointArray ) > 0 and datapointArray [ 0 ] [ "t" ] < newtime ) : logging . debug ( "Datapoint exists with older timestamp. Removing the datapoint." ) datapointArray = datapointArray [ 1 : ] if len ( datapointArray ) > 0 : logging . debug ( "%s: syncing %i datapoints" % ( stream , len ( datapointArray ) ) ) while ( len ( datapointArray ) > DATAPOINT_INSERT_LIMIT ) : # We insert datapoints in chunks of a couple # thousand so that they fit in the insert size # limit of ConnectorDB s . insert_array ( datapointArray [ : DATAPOINT_INSERT_LIMIT ] ) # Clear the written datapoints datapointArray = datapointArray [ DATAPOINT_INSERT_LIMIT : ] # If there was no error inserting, delete the # datapoints from the cache c . execute ( "DELETE FROM cache WHERE stream=? AND timestamp <?" , ( stream , datapointArray [ 0 ] [ "t" ] ) ) s . insert_array ( datapointArray ) # If there was no error inserting, delete the # datapoints from the cache c . execute ( "DELETE FROM cache WHERE stream=? AND timestamp <=?" , ( stream , datapointArray [ - 1 ] [ "t" ] ) ) self . lastsynctime = time . time ( ) if self . onsync is not None : self . onsync ( ) except Exception as e : # Handle the sync failure callback falied = True reraise = self . syncraise if self . onsyncfail is not None : reraise = self . onsyncfail ( e ) if reraise : raise
Attempt to sync with the ConnectorDB server
Attempt to sync with the ConnectorDB server