<reponame>simonsobs/ocs<filename>ocs/checkdata.py
#!/usr/bin/env python3
# <NAME>

import os
import time

from progress.bar import Bar

from so3g import hk
from ocs.ocs_feed import Feed

from colorama import init, Fore, Style
init()

def _build_file_list(target):
    """Build list of files to scan.

    Parameters
    ----------
    target : str
        File or directory to scan.

    Returns
    -------
    list
        List of full paths to files for scanning.

    """
    _file_list = []
    if os.path.isfile(target):
        _file_list.append(target)
    elif os.path.isdir(target):
        a = os.walk(target)
        for root, _, _file in a:
            for g3 in _file:
                if g3[-2:] == "g3":
                    _file_list.append(os.path.join(root, g3))

    return _file_list

class DataChecker:
    """Check data for latest feeds and fields.

    Parameters
    ----------
    target : str
        File or directory to scan.
    verbose : bool
        Verbose output flag

    Attributes
    ----------
    hkas : so3g.hk.HKArchiveScanner
        HKArchiveScanner for reading in the data
    cat :
        Finalized HKArchiveScanner
    target : str
        File or directory to scan.
    verbose : bool
        Verbose output flag
    fields : dict
        fields returned from a cat.get_fields call
    timelines : dict
        timelines returned from a cat.get_fields call
    instances : dict
        Agent/feed/field information for each instance-id, format described in
        _populate_instances docstring

    """
    def __init__(self, target, verbose=False):
        self.target = target
        self.verbose = verbose

        self.hkas = hk.HKArchiveScanner()
        self.cat = None

        self.fields = None
        self.timelines = None

        self.instances = {}

        # Private attributes
        self._file_list = _build_file_list(target)
        self._field_count = 0

    def scan_files(self):
        """Scan all the files with the HKArchiveScanner for later processing."""
        _bar = Bar('Scanning', max=len(self._file_list))
        for _file in self._file_list:
            try:
                self.hkas.process_file(_file)
            except Exception as e:
                print(e)
            _bar.next()
        _bar.finish()

        self.cat = self.hkas.finalize()

    def _populate_instances(self):
        """Populate the instances dictionary with information about each agent,
        the agent's feeds, and the associated fields.

        For each instance ID there is a dictionary like:
        {INSTANCE_ID:
            {FEED1:
                {"fields":
                   {FIELD1:
                       {'full_name': 'observatory.INSTANCE_ID.feeds.FEED.FIELD',
                        't_last': float,
                        'v_last': float},
                    FIELD2:
                       {'full_name': 'observatory.INSTANCE_ID.feeds.FEED.FIELD',
                        't_last': float,
                        'v_last': float}
                    },
                 "t_last" : lowest t_last in all fields
                }
             FEED2: {},
             FEED3: {},
             ...
            }
        }

        """

        for field_name in sorted(self.fields):
            site, instance_id, _, feed, field = field_name.split(".")

            if instance_id not in self.instances:
                self.instances[instance_id] = {}

            if feed not in self.instances[instance_id]:
                self.instances[instance_id][feed] = {"fields": {},
                                                     "t_last": None}

            if field not in self.instances[instance_id][feed]:
                self.instances[instance_id][feed]["fields"][field] = \
                    {'full_name': field_name,
                     't_last': None,
                     'v_last': None}
                self._field_count += 1

    def process_files(self):
        """Process all files, resquesting simple data points for each dataset
        and determining the last time we saw the field and the last value that
        field held.

        """
        _bar = Bar('Processing', max=self._field_count)
        for instance_id, feeds in self.instances.items():
            for feed, fields in feeds.items():
                for field, d_info in fields['fields'].items():
                    t, x = self.cat.simple(d_info['full_name'])
                    d_info['t_last'] = t[-1]
                    d_info['v_last'] = x[-1]

                    if fields['t_last'] is None:
                        fields['t_last'] = t[-1]
                    elif t[-1] < fields['t_last']:
                        fields['t_last'] = t[-1]
                    else:
                        pass

                    _bar.next()
        _bar.finish()
        print()

    def run(self):
        """Run data checker, scan and process all files in target."""
        self.scan_files()
        self.fields, self.timelines = self.cat.get_fields()
        self._populate_instances()
        self.process_files()

    def __str__(self):
        """Print informative string representing the data checker results."""
        description_string = ""

        for instance_id, feeds in self.instances.items():
            description_string += f"{instance_id}\n"
            for feed, fields in feeds.items():
                if not self.verbose:
                    field_t_diff = time.time() - fields['t_last']
                    if field_t_diff > 600:
                        description_string += f"  {feed}: " + Fore.RED + f"{field_t_diff:.1f} s old\n" + Style.RESET_ALL
                    else:
                        description_string += f"  {feed}: {field_t_diff:.1f} s old\n"
                else:
                    # Determine width of "Field" column
                    field_str_len = 20
                    for _f in fields['fields']:
                        if len(_f) > field_str_len:
                            field_str_len = len(_f)

                    description_string += f"  {feed}\n"
                    # 20 per fixed field, 9 for dividers, and field_str_len
                    description_string += "  " + "-"*(69+field_str_len) + "\n"

                    # Header string
                    desc_substring = "  "

                    # Field
                    _field_string = "Field".rjust(field_str_len)
                    desc_substring += _field_string + " | "

                    # Last Seen [s ago]
                    _t_diff_string = "{:>20}".format("Last Seen [s ago]")
                    desc_substring += _t_diff_string + " | "

                    # Seen At [ctime]
                    _t_last_string = "{:>20}".format("Seen At [ctime]")
                    desc_substring += _t_last_string + " | "

                    # Value
                    _v_last_string = "{:>20}".format("Value")
                    desc_substring += _v_last_string

                    description_string += desc_substring + "\n"

                    description_string += "  " + "-"*(69+field_str_len) + "\n"

                    for field, d_info in fields['fields'].items():
                        t_diff = time.time() - d_info['t_last']
                        desc_substring = "  "

                        # Field
                        try:
                            valid_field = Feed.verify_data_field_string(field)
                        except ValueError:
                            valid_field = False
                        _field_string = field.rjust(field_str_len)
                        if not valid_field:
                            _field_string = Fore.YELLOW + _field_string + Style.RESET_ALL
                        desc_substring += _field_string + " | "

                        # Last Seen [s ago]
                        _t_diff_string = "{:>20.1f}".format(t_diff)
                        if t_diff > 600:
                            _t_diff_string = Fore.RED + _t_diff_string + Style.RESET_ALL
                        desc_substring += _t_diff_string + " | "

                        # Seen At [ctime]
                        _t_last_string = "{:>20}".format(d_info['t_last'])
                        desc_substring += _t_last_string + " | "

                        # Value
                        _v_last_string = "{:>20}".format(d_info['v_last'])
                        desc_substring += _v_last_string

                        description_string += desc_substring + "\n"

            description_string += "\n"

        return description_string


def main(target, verbose=False):
    checker = DataChecker(target, verbose)
    checker.run()
    print(checker)

#!/usr/bin/env python3
# <NAME>

import os
import time

from progress.bar import Bar

from so3g import hk
from ocs.ocs_feed import Feed

from colorama import init, Fore, Style
init()

def _build_file_list(target):
    """Build list of files to scan.

    Parameters
    ----------
    target : str
        File or directory to scan.

    Returns
    -------
    list
        List of full paths to files for scanning.

    """
    _file_list = []
    if os.path.isfile(target):
        _file_list.append(target)
    elif os.path.isdir(target):
        a = os.walk(target)
        for root, _, _file in a:
            for g3 in _file:
                if g3[-2:] == "g3":
                    _file_list.append(os.path.join(root, g3))

    return _file_list

class DataChecker:
    """Check data for latest feeds and fields.

    Parameters
    ----------
    target : str
        File or directory to scan.
    verbose : bool
        Verbose output flag

    Attributes
    ----------
    hkas : so3g.hk.HKArchiveScanner
        HKArchiveScanner for reading in the data
    cat :
        Finalized HKArchiveScanner
    target : str
        File or directory to scan.
    verbose : bool
        Verbose output flag
    fields : dict
        fields returned from a cat.get_fields call
    timelines : dict
        timelines returned from a cat.get_fields call
    instances : dict
        Agent/feed/field information for each instance-id, format described in
        _populate_instances docstring

    """
    def __init__(self, target, verbose=False):
        self.target = target
        self.verbose = verbose

        self.hkas = hk.HKArchiveScanner()
        self.cat = None

        self.fields = None
        self.timelines = None

        self.instances = {}

        # Private attributes
        self._file_list = _build_file_list(target)
        self._field_count = 0

    def scan_files(self):
        """Scan all the files with the HKArchiveScanner for later processing."""
        _bar = Bar('Scanning', max=len(self._file_list))
        for _file in self._file_list:
            try:
                self.hkas.process_file(_file)
            except Exception as e:
                print(e)
            _bar.next()
        _bar.finish()

        self.cat = self.hkas.finalize()

    def _populate_instances(self):
        """Populate the instances dictionary with information about each agent,
        the agent's feeds, and the associated fields.

        For each instance ID there is a dictionary like:
        {INSTANCE_ID:
            {FEED1:
                {"fields":
                   {FIELD1:
                       {'full_name': 'observatory.INSTANCE_ID.feeds.FEED.FIELD',
                        't_last': float,
                        'v_last': float},
                    FIELD2:
                       {'full_name': 'observatory.INSTANCE_ID.feeds.FEED.FIELD',
                        't_last': float,
                        'v_last': float}
                    },
                 "t_last" : lowest t_last in all fields
                }
             FEED2: {},
             FEED3: {},
             ...
            }
        }

        """

        for field_name in sorted(self.fields):
            site, instance_id, _, feed, field = field_name.split(".")

            if instance_id not in self.instances:
                self.instances[instance_id] = {}

            if feed not in self.instances[instance_id]:
                self.instances[instance_id][feed] = {"fields": {},
                                                     "t_last": None}

            if field not in self.instances[instance_id][feed]:
                self.instances[instance_id][feed]["fields"][field] = \
                    {'full_name': field_name,
                     't_last': None,
                     'v_last': None}
                self._field_count += 1

    def process_files(self):
        """Process all files, resquesting simple data points for each dataset
        and determining the last time we saw the field and the last value that
        field held.

        """
        _bar = Bar('Processing', max=self._field_count)
        for instance_id, feeds in self.instances.items():
            for feed, fields in feeds.items():
                for field, d_info in fields['fields'].items():
                    t, x = self.cat.simple(d_info['full_name'])
                    d_info['t_last'] = t[-1]
                    d_info['v_last'] = x[-1]

                    if fields['t_last'] is None:
                        fields['t_last'] = t[-1]
                    elif t[-1] < fields['t_last']:
                        fields['t_last'] = t[-1]
                    else:
                        pass

                    _bar.next()
        _bar.finish()
        print()

    def run(self):
        """Run data checker, scan and process all files in target."""
        self.scan_files()
        self.fields, self.timelines = self.cat.get_fields()
        self._populate_instances()
        self.process_files()

    def __str__(self):
        """Print informative string representing the data checker results."""
        description_string = ""

        for instance_id, feeds in self.instances.items():
            description_string += f"{instance_id}\n"
            for feed, fields in feeds.items():
                if not self.verbose:
                    field_t_diff = time.time() - fields['t_last']
                    if field_t_diff > 600:
                        description_string += f"  {feed}: " + Fore.RED + f"{field_t_diff:.1f} s old\n" + Style.RESET_ALL
                    else:
                        description_string += f"  {feed}: {field_t_diff:.1f} s old\n"
                else:
                    # Determine width of "Field" column
                    field_str_len = 20
                    for _f in fields['fields']:
                        if len(_f) > field_str_len:
                            field_str_len = len(_f)

                    description_string += f"  {feed}\n"
                    # 20 per fixed field, 9 for dividers, and field_str_len
                    description_string += "  " + "-"*(69+field_str_len) + "\n"

                    # Header string
                    desc_substring = "  "

                    # Field
                    _field_string = "Field".rjust(field_str_len)
                    desc_substring += _field_string + " | "

                    # Last Seen [s ago]
                    _t_diff_string = "{:>20}".format("Last Seen [s ago]")
                    desc_substring += _t_diff_string + " | "

                    # Seen At [ctime]
                    _t_last_string = "{:>20}".format("Seen At [ctime]")
                    desc_substring += _t_last_string + " | "

                    # Value
                    _v_last_string = "{:>20}".format("Value")
                    desc_substring += _v_last_string

                    description_string += desc_substring + "\n"

                    description_string += "  " + "-"*(69+field_str_len) + "\n"

                    for field, d_info in fields['fields'].items():
                        t_diff = time.time() - d_info['t_last']
                        desc_substring = "  "

                        # Field
                        try:
                            valid_field = Feed.verify_data_field_string(field)
                        except ValueError:
                            valid_field = False
                        _field_string = field.rjust(field_str_len)
                        if not valid_field:
                            _field_string = Fore.YELLOW + _field_string + Style.RESET_ALL
                        desc_substring += _field_string + " | "

                        # Last Seen [s ago]
                        _t_diff_string = "{:>20.1f}".format(t_diff)
                        if t_diff > 600:
                            _t_diff_string = Fore.RED + _t_diff_string + Style.RESET_ALL
                        desc_substring += _t_diff_string + " | "

                        # Seen At [ctime]
                        _t_last_string = "{:>20}".format(d_info['t_last'])
                        desc_substring += _t_last_string + " | "

                        # Value
                        _v_last_string = "{:>20}".format(d_info['v_last'])
                        desc_substring += _v_last_string

                        description_string += desc_substring + "\n"

            description_string += "\n"

        return description_string


def main(target, verbose=False):
    checker = DataChecker(target, verbose)
    checker.run()
    print(checker)
#!/usr/bin/env python3 # <NAME> Build list of files to scan.      Parameters     ----------     target : str         File or directory to scan.      Returns     -------     list         List of full paths to files for scanning. Check data for latest feeds and fields.      Parameters     ----------     target : str         File or directory to scan.     verbose : bool         Verbose output flag      Attributes     ----------     hkas : so3g.hk.HKArchiveScanner         HKArchiveScanner for reading in the data     cat :         Finalized HKArchiveScanner     target : str         File or directory to scan.     verbose : bool         Verbose output flag     fields : dict         fields returned from a cat.get_fields call     timelines : dict         timelines returned from a cat.get_fields call     instances : dict         Agent/feed/field information for each instance-id, format described in         _populate_instances docstring # Private attributes Scan all the files with the HKArchiveScanner for later processing. Populate the instances dictionary with information about each agent,         the agent's feeds, and the associated fields.          For each instance ID there is a dictionary like:         {INSTANCE_ID:             {FEED1:                 {"fields":                    {FIELD1:                        {'full_name': 'observatory.INSTANCE_ID.feeds.FEED.FIELD',                         't_last': float,                         'v_last': float},                     FIELD2:                        {'full_name': 'observatory.INSTANCE_ID.feeds.FEED.FIELD',                         't_last': float,                         'v_last': float}                     },                  "t_last" : lowest t_last in all fields                 }              FEED2: {},              FEED3: {},              ...             }         } Process all files, resquesting simple data points for each dataset         and determining the last time we saw the field and the last value that         field held. Run data checker, scan and process all files in target. Print informative string representing the data checker results. # Determine width of "Field" column # 20 per fixed field, 9 for dividers, and field_str_len # Header string # Field # Last Seen [s ago] # Seen At [ctime] # Value # Field # Last Seen [s ago] # Seen At [ctime] # Value