class OSV(object):
    """
    interface for management of S1 Orbit State Vector (OSV) files

    input is a directory which is supposed to contain, or already contains, OSV files.
    Two subdirectories are expected and created otherwise:
    one for Precise Orbit Ephemerides (POE) named POEORB and one for Restituted Orbit (RES) files named RESORB

    Using method :meth:`match` the corresponding POE (priority) or RES file is returned for a timestamp.
    Timestamps are always handled in the format YYYYmmddTHHMMSS.

    Parameters
    ----------
    osvdir: str
        the directory to write the orbit files to
    timeout: int or tuple or None
        the timeout in seconds for downloading OSV files as provided to :func:`requests.get`
    
    See Also
    --------
    `requests timeouts <https://requests.readthedocs.io/en/master/user/advanced/#timeouts>`_
    """
    
    def __init__(self, osvdir=None, timeout=300):
        self.timeout = timeout
        if osvdir is None:
            try:
                auxdatapath = ExamineSnap().auxdatapath
            except AttributeError:
                auxdatapath = os.path.join(os.path.expanduser('~'), '.snap', 'auxdata')
            osvdir = os.path.join(auxdatapath, 'Orbits', 'Sentinel-1')
        self.outdir_poe = os.path.join(osvdir, 'POEORB')
        self.outdir_res = os.path.join(osvdir, 'RESORB')
        self.pattern = r'S1[AB]_OPER_AUX_(?:POE|RES)ORB_OPOD_[0-9TV_]{48}\.EOF'
        self.pattern_fine = r'(?P<sensor>S1[AB])_OPER_AUX_' \
                            r'(?P<type>(?:POE|RES)ORB)_OPOD_' \
                            r'(?P<publish>[0-9]{8}T[0-9]{6})_V' \
                            r'(?P<start>[0-9]{8}T[0-9]{6})_' \
                            r'(?P<stop>[0-9]{8}T[0-9]{6})\.EOF'
        self._init_dir()
        self._reorganize()
    
    def __enter__(self):
        return self
    
    def __exit__(self, exc_type, exc_val, exc_tb):
        return
    
    def _init_dir(self):
        """
        create directories if they don't exist yet
        """
        for dir in [self.outdir_poe, self.outdir_res]:
            if not os.path.isdir(dir):
                os.makedirs(dir)
    
    def _parse(self, file):
        basename = os.path.basename(file)
        groups = re.match(self.pattern_fine, basename).groupdict()
        return groups
    
    def _reorganize(self):
        """
        compress and move EOF files into subdirectories

        Returns
        -------

        """
        message = True
        for subdir in [self.outdir_poe, self.outdir_res]:
            if not os.path.isdir(subdir):
                continue
            files = finder(subdir, [self.pattern], recursive=False, regex=True)
            for eof in files:
                base = os.path.basename(eof)
                target = os.path.join(self._subdir(eof), base + '.zip')
                os.makedirs(os.path.dirname(target), exist_ok=True)
                if not os.path.isfile(target):
                    if message:
                        log.info('compressing and reorganizing EOF files')
                        message = False
                    with zf.ZipFile(file=target,
                                    mode='w',
                                    compression=zf.ZIP_DEFLATED) as zip:
                        zip.write(filename=eof,
                                  arcname=base)
                os.remove(eof)
    
    def _typeEvaluate(self, osvtype):
        """
        evaluate the 'osvtype' method argument and return the corresponding remote repository and local directory

        Parameters
        ----------
        osvtype: str
            the type of orbit files required; either 'POE' or 'RES'

        Returns
        -------
        tuple of str
            the remote repository and local directory of the osv type
        """
        if osvtype not in ['POE', 'RES']:
            raise IOError('type must be either "POE" or "RES"')
        if osvtype == 'POE':
            return self.outdir_poe
        else:
            return self.outdir_res
    
    def __catch_aux_sentinel(self, sensor, start, stop, osvtype='POE'):
        url = 'http://aux.sentinel1.eo.esa.int'
        skeleton = '{url}/{osvtype}ORB/{year}/{month:02d}/{day:02d}/'
        
        files = []
        date_search = start
        busy = True
        while busy:
            url_sub = skeleton.format(url=url,
                                      osvtype=osvtype,
                                      year=date_search.year,
                                      month=date_search.month,
                                      day=date_search.day)
            response = requests.get(url_sub, timeout=self.timeout)
            response.raise_for_status()
            result = response.text
            files_sub = list(set(re.findall(self.pattern, result)))
            if len(files_sub) == 0:
                break
            for file in files_sub:
                match = re.match(self.pattern_fine, file)
                start2 = datetime.strptime(match.group('start'), '%Y%m%dT%H%M%S')
                stop2 = datetime.strptime(match.group('stop'), '%Y%m%dT%H%M%S')
                if sensor == match.group('sensor'):
                    if start2 < stop and stop2 > start:
                        log.info(url_sub)
                        files.append({'filename': file,
                                      'href': url_sub + '/' + file,
                                      'auth': None})
                if start2 >= stop:
                    busy = False
            date_search += timedelta(days=1)
        
        return files
    
    def __catch_step_auxdata(self, sensor, start, stop, osvtype='POE'):
        url = 'https://step.esa.int/auxdata/orbits/Sentinel-1'
        skeleton = '{url}/{osvtype}ORB/{sensor}/{year}/{month:02d}/'
        
        if isinstance(sensor, str):
            sensor = [sensor]
        
        files = []
        for sens in sensor:
            date_search = datetime(year=start.year,
                                   month=start.month,
                                   day=1)
            busy = True
            while busy:
                url_sub = skeleton.format(url=url,
                                          osvtype=osvtype,
                                          sensor=sens,
                                          year=date_search.year,
                                          month=date_search.month)
                log.info(url_sub)
                response = requests.get(url_sub, timeout=self.timeout)
                response.raise_for_status()
                result = response.text
                files_sub = list(set(re.findall(self.pattern, result)))
                if len(files_sub) == 0:
                    break
                for file in files_sub:
                    match = re.match(self.pattern_fine, file)
                    start2 = datetime.strptime(match.group('start'), '%Y%m%dT%H%M%S')
                    stop2 = datetime.strptime(match.group('stop'), '%Y%m%dT%H%M%S')
                    if start2 < stop and stop2 > start:
                        files.append({'filename': file,
                                      'href': url_sub + '/' + file + '.zip',
                                      'auth': None})
                    if start2 >= stop:
                        busy = False
                if date_search.month < 12:
                    date_search = date_search.replace(month=date_search.month + 1)
                else:
                    date_search = date_search.replace(year=date_search.year + 1, month=1)
        
        return files
    
    def __catch_gnss(self, sensor, start, stop, osvtype='POE'):
        url = 'https://scihub.copernicus.eu/gnss'
        redirect = 'https://dhusfeed.dhus.onda-dias.net/gnss'
        auth = ('gnssguest', 'gnssguest')
        # a dictionary for storing the url arguments
        query = {}
        
        if osvtype == 'POE':
            query['producttype'] = 'AUX_POEORB'
        elif osvtype == 'RES':
            query['producttype'] = 'AUX_RESORB'
        else:
            raise RuntimeError("osvtype must be either 'POE' or 'RES'")
        
        if sensor in ['S1A', 'S1B']:
            query['platformname'] = 'Sentinel-1'
            # filename starts w/ sensor
            query['filename'] = '{}*'.format(sensor)
        elif sorted(sensor) == ['S1A', 'S1B']:
            query['platformname'] = 'Sentinel-1'
        else:
            raise RuntimeError('unsupported input for parameter sensor')
        
        # the collection of files to be returned
        collection = []
        
        date_start = start.strftime('%Y-%m-%dT%H:%M:%SZ')
        date_stop = stop.strftime('%Y-%m-%dT%H:%M:%SZ')
        
        # append the time frame to the query dictionary
        query['beginPosition'] = '[{} TO {}]'.format(date_start, date_stop)
        query['endPosition'] = '[{} TO {}]'.format(date_start, date_stop)
        query_list = []
        for keyword, value in query.items():
            query_elem = '{}:{}'.format(keyword, value)
            query_list.append(query_elem)
        query_str = ' '.join(query_list)
        target = '{}/search?q={}&format=json'.format(url, query_str)
        log.info(target)
        
        def _parse_gnsssearch_json(search_dict):
            parsed_dict = {}
            # Will return ['entry'] as dict if only one item
            # If so just make a list
            if isinstance(search_dict, dict):
                search_dict = [search_dict]
            for entry in search_dict:
                id = entry['id']
                entry_dict = {}
                
                for key, value in entry.items():
                    if key == 'title':
                        entry_dict[key] = value
                    elif key == 'id':
                        entry_dict[key] = value
                    elif key == 'ondemand':
                        if value.lower() == 'true':
                            entry_dict[key] = True
                        else:
                            entry_dict[key] = False
                    elif key == 'str':
                        for elem in value:
                            entry_dict[elem['name']] = elem['content']
                    elif key == 'link':
                        for elem in value:
                            if 'rel' in elem.keys():
                                href_key = 'href_' + elem['rel']
                                entry_dict[href_key] = elem['href']
                            else:
                                entry_dict['href'] = elem['href']
                    elif key == 'date':
                        for elem in value:
                            entry_dict[elem['name']] = dateutil_parser.parse(elem['content'])
                
                parsed_dict[id] = entry_dict
            return parsed_dict
        
        def _parse_gnsssearch_response(response_json):
            if 'entry' in response_json.keys():
                search_dict = response_json['entry']
                parsed_dict = _parse_gnsssearch_json(search_dict)
            else:
                parsed_dict = {}
            return parsed_dict
        
        response = requests.get(target, auth=auth, timeout=self.timeout)
        response.raise_for_status()
        response_json = response.json()['feed']
        total_results = response_json['opensearch:totalResults']
        subquery = [link['href'] for link in response_json['link'] if link['rel'] == 'self'][0]
        subquery = subquery.replace(redirect, url.strip())
        if int(total_results) > 10:
            subquery = subquery.replace('rows=10', 'rows=100')
        while subquery:
            subquery_response = requests.get(subquery, auth=auth, timeout=self.timeout)
            subquery_response.raise_for_status()
            subquery_json = subquery_response.json()['feed']
            subquery_products = _parse_gnsssearch_response(subquery_json)
            items = list(subquery_products.values())
            for item in items:
                item['auth'] = auth
            collection += list(subquery_products.values())
            if 'next' in [link['rel'] for link in subquery_json['link']]:
                subquery = [link['href'] for link in subquery_json['link'] if link['rel'] == 'next'][0]
                subquery = subquery.replace(redirect, url.strip())
            else:
                subquery = None
        if osvtype == 'RES' and self.maxdate('POE', 'stop') is not None:
            collection = [x for x in collection
                          if self.date(x['filename'], 'start') > self.maxdate('POE', 'stop')]
        for item in collection:
            item['href'] = item['href'].replace(redirect, url)
        return collection
    
    def catch(self, sensor, osvtype='POE', start=None, stop=None, url_option=1):
        """
        check a server for files

        Parameters
        ----------
        sensor: str or list
            The S1 mission(s):
            
             - 'S1A'
             - 'S1B'
             - ['S1A', 'S1B']
        osvtype: {'POE', 'RES'}
            the type of orbit files required
        start: str
            the date to start searching for files in format YYYYmmddTHHMMSS
        stop: str
            the date to stop searching for files in format YYYYmmddTHHMMSS
        url_option: int
            the URL to query for OSV files
            
             - 1: https://scihub.copernicus.eu/gnss
             - 2: https://step.esa.int/auxdata/orbits/Sentinel-1

        Returns
        -------
        list[dict]
            the product dictionary of the remote OSV files, with href
        """
        
        log.info('searching for new {} files'.format(osvtype))
        
        if start is not None:
            start = datetime.strptime(start, '%Y%m%dT%H%M%S')
        else:
            start = datetime.strptime('2014-07-31', '%Y-%m-%d')
        # set the defined date or the current date otherwise
        if stop is not None:
            stop = datetime.strptime(stop, '%Y%m%dT%H%M%S')
        else:
            stop = datetime.now()
        
        if url_option == 1:
            items = self.__catch_gnss(sensor, start, stop, osvtype)
        elif url_option == 2:
            items = self.__catch_step_auxdata(sensor, start, stop, osvtype)
        else:
            raise ValueError("'url_option' must be either 1 or 2")
        
        if osvtype == 'RES' and self.maxdate('POE', 'stop') is not None:
            items = [x for x in items
                     if self.date(x['filename'], 'start') > self.maxdate('POE', 'stop')]
        log.info('found {} results'.format(len(items)))
        
        return items
    
    def date(self, file, datetype):
        """
        extract a date from an OSV file name

        Parameters
        ----------
        file: str
            the OSV file
        datetype: {'publish', 'start', 'stop'}
            one of three possible date types contained in the OSV filename

        Returns
        -------
        str
            a time stamp in the format YYYYmmddTHHMMSS
        """
        return self._parse(file)[datetype]
    
    def clean_res(self):
        """
        delete all RES files for whose date a POE file exists
        """
        maxdate_poe = self.maxdate('POE', 'stop')
        if maxdate_poe is not None:
            deprecated = [x for x in self.getLocals('RES') if self.date(x, 'stop') < maxdate_poe]
            log.info('deleting {} RES file{}'.format(len(deprecated), '' if len(deprecated) == 1 else 's'))
            for item in deprecated:
                os.remove(item)
    
    def getLocals(self, osvtype='POE'):
        """
        get a list of local files of specific type

        Parameters
        ----------
        osvtype: {'POE', 'RES'}
            the type of orbit files required

        Returns
        -------
        list
            a selection of local OSV files
        """
        directory = self._typeEvaluate(osvtype)
        return finder(directory, [self.pattern], regex=True)
    
    def maxdate(self, osvtype='POE', datetype='stop'):
        """
        return the latest date of locally existing POE/RES files

        Parameters
        ----------
        osvtype: {'POE', 'RES'}
            the type of orbit files required
        datetype: {'publish', 'start', 'stop'}
            one of three possible date types contained in the OSV filename

        Returns
        -------
        str
            a timestamp in format YYYYmmddTHHMMSS
        """
        directory = self._typeEvaluate(osvtype)
        files = finder(directory, [self.pattern], regex=True)
        return max([self.date(x, datetype) for x in files]) if len(files) > 0 else None
    
    def mindate(self, osvtype='POE', datetype='start'):
        """
        return the earliest date of locally existing POE/RES files

        Parameters
        ----------
        osvtype: {'POE', 'RES'}
            the type of orbit files required
        datetype: {'publish', 'start', 'stop'}
            one of three possible date types contained in the OSV filename

        Returns
        -------
        str
            a timestamp in format YYYYmmddTHHMMSS
        """
        directory = self._typeEvaluate(osvtype)
        files = finder(directory, [self.pattern], regex=True)
        return min([self.date(x, datetype) for x in files]) if len(files) > 0 else None
    
    def match(self, sensor, timestamp, osvtype='POE'):
        """
        return the corresponding OSV file for the provided sensor and time stamp.
        The file returned is one which covers the acquisition time and, if multiple exist,
        the one which was published last.
        In case a list of options is provided as osvtype, the file of higher accuracy (i.e. POE over RES) is returned.

        Parameters
        ----------
        sensor: str
            The S1 mission:
            
             - 'S1A'
             - 'S1B'
        timestamp: str
            the time stamp in the format 'YYYmmddTHHMMSS'
        osvtype: str or list
            the type of orbit files required; either 'POE', 'RES' or a list of both

        Returns
        -------
        str
            the best matching orbit file (overlapping time plus latest publication date)
        """
        # list all locally existing files of the defined type
        if osvtype in ['POE', 'RES']:
            locals = self.getLocals(osvtype)
            # filter the files to those which contain data for the defined time stamp
            files = [x for x in locals if self.date(x, 'start') <= timestamp <= self.date(x, 'stop')]
            files = [x for x in files if os.path.basename(x).startswith(sensor)]
            if len(files) > 0:
                # select the file which was published last
                best = self.sortByDate(files, 'publish')[-1]
                return best
            elif len(files) == 1:
                return files[0]
            return None
        elif sorted(osvtype) == ['POE', 'RES']:
            best = self.match(sensor=sensor, timestamp=timestamp, osvtype='POE')
            if not best:
                best = self.match(sensor=sensor, timestamp=timestamp, osvtype='RES')
            return best
    
    def retrieve(self, products, pbar=False):
        """
        download a list of product dictionaries into the respective subdirectories, i.e. POEORB or RESORB

        Parameters
        ----------
        files: list
            a list of remotely existing OSV product dictionaries as returned by method :meth:`catch`
        pbar: bool
            add a progressbar?

        Returns
        -------
        """
        downloads = []
        for product in products:
            if all(key not in ['filename', 'href'] for key in product.keys()):
                raise RuntimeError("product dictionaries must contain 'filename' and 'href' keys")
            basename = product['filename']
            remote = product['href']
            auth = product['auth']
            
            outdir = self._subdir(basename)
            os.makedirs(outdir, exist_ok=True)
            local = os.path.join(outdir, basename) + '.zip'
            if not os.path.isfile(local):
                downloads.append((remote, local, basename, auth))
        if len(downloads) == 0:
            return
        log.info('downloading {} file{}'.format(len(downloads), '' if len(downloads) == 1 else 's'))
        if pbar:
            progress = pb.ProgressBar(max_value=len(downloads))
        else:
            progress = None
        i = 0
        for remote, local, basename, auth in downloads:
            response = requests.get(remote, auth=auth, timeout=self.timeout)
            response.raise_for_status()
            infile = response.content

            # use a tempfile to allow atomic writes in the case of
            # parallel executions dependent on the same orbit files
            fd, tmp_path = tempfile.mkstemp(prefix=os.path.basename(local), dir=os.path.dirname(local))
            os.close(fd)
            try:
                if remote.endswith('.zip'):
                    with zf.ZipFile(file=BytesIO(infile)) as tmp:
                        members = tmp.namelist()
                        target = [x for x in members if re.search(basename, x)][0]
                        with zf.ZipFile(tmp_path, 'w') as outfile:
                            outfile.write(filename=tmp.extract(target),
                                          arcname=basename)
                else:
                    with zf.ZipFile(file=tmp_path,
                                    mode='w',
                                    compression=zf.ZIP_DEFLATED) \
                            as outfile:
                        outfile.writestr(zinfo_or_arcname=basename,
                                         data=infile)
                os.rename(tmp_path, local)
            except Exception as e:
                os.unlink(tmp_path)
                raise

            if pbar:
                i += 1
                progress.update(i)
        if pbar:
            progress.finish()
        self.clean_res()
    
    def sortByDate(self, files, datetype='start'):
        """
        sort a list of OSV files by a specific date type

        Parameters
        ----------
        files: list[str]
            some OSV files
        datetype: {'publish', 'start', 'stop'}
            one of three possible date types contained in the OSV filename

        Returns
        -------
        list[str]
            the input OSV files sorted by the defined date
        """
        return sorted(files, key=lambda x: self.date(x, datetype))
    
    def _subdir(self, file):
        """
        | return the subdirectory in which to store the EOF file,
        | i.e. basedir/{type}ORB/{sensor}/{year}/{month}
        | e.g. basedir/POEORB/S1A/2018/12

        Parameters
        ----------
        file: str
            the EOF filename

        Returns
        -------
        str
            the target directory
        """
        attr = self._parse(file)
        outdir = self._typeEvaluate(attr['type'][:3])
        start = self.date(file, datetype='start')
        start = datetime.strptime(start, '%Y%m%dT%H%M%S')
        month = '{:02d}'.format(start.month)
        outdir = os.path.join(outdir, attr['sensor'],
                              str(start.year), month)
        return outdir class ID(object):
    """
    Abstract class for SAR meta data handlers
    """
    
    def __init__(self, metadict):
        """
        to be called by the __init__methods of the format drivers
        scans a metadata dictionary and registers entries with a standardized name as object attributes
        see __LOCAL__ for standard names. It must be ensured that each of these is actually read by the individual SAR format driver.

        :param metadict: a dictionary containing the metadata attributes of a SAR scene
        """
        self.locals = __LOCAL__
        for item in self.locals:
            setattr(self, item, metadict[item])
    
    def __getattr__(self, item):
        raise AttributeError("object has no attribute '{}'".format(item))
    
    def __str__(self):
        lines = ['pyroSAR ID object of type {}'.format(self.__class__.__name__)]
        for item in sorted(self.locals):
            value = getattr(self, item)
            if item == 'projection':
                value = crsConvert(value, 'proj4')
            line = '{0}: {1}'.format(item, value)
            lines.append(line)
        return '\n'.join(lines)
    
    def bbox(self, outname=None, driver=None, overwrite=True):
        """
        get the bounding box of a scene either as a vector object or written to a file

        Parameters
        ----------
        outname: str
            the name of the shapefile to be written
        driver: str
            the output file format; needs to be defined if the format cannot
            be auto-detected from the filename extension
        overwrite: bool
            overwrite an existing shapefile?

        Returns
        -------
        ~spatialist.vector.Vector or None
            the vector object if `outname` is None, None otherwise
        """
        if outname is None:
            return bbox(self.getCorners(), self.projection)
        else:
            bbox(self.getCorners(), self.projection, outname=outname, driver=driver,
                 overwrite=overwrite)
    
    def geometry(self, outname=None, driver=None, overwrite=True):
        """
        get the footprint geometry of a scene either as a vector object or written to a file

        Parameters
        ----------
        outname: str
            the name of the shapefile to be written
        driver: str
            the output file format; needs to be defined if the format cannot
            be auto-detected from the filename extension
        overwrite: bool
            overwrite an existing shapefile?

        Returns
        -------
        ~spatialist.vector.Vector or None
            the vector object if `outname` is None, None otherwise
        """
        srs = crsConvert(self.projection, 'osr')
        ring = ogr.Geometry(ogr.wkbLinearRing)
        for coordinate in self.meta['coordinates']:
            ring.AddPoint(*coordinate)
        ring.CloseRings()
        
        geom = ogr.Geometry(ogr.wkbPolygon)
        geom.AddGeometry(ring)
        
        geom.FlattenTo2D()
        
        bbox = Vector(driver='Memory')
        bbox.addlayer('geometry', srs, geom.GetGeometryType())
        bbox.addfield('area', ogr.OFTReal)
        bbox.addfeature(geom, fields={'area': geom.Area()})
        geom = None
        if outname is None:
            return bbox
        else:
            bbox.write(outfile=outname, driver=driver, overwrite=overwrite)
    
    @property
    def compression(self):
        """
        check whether a scene is compressed into an tarfile or zipfile or not at all

        Returns
        -------
        str or None
            either 'zip', 'tar' or None
        """
        if os.path.isdir(self.scene):
            return None
        elif zf.is_zipfile(self.scene):
            return 'zip'
        elif tf.is_tarfile(self.scene):
            return 'tar'
        else:
            return None
    
    def export2dict(self):
        """
        Return the uuid and the metadata that is defined in self.locals as a dictionary
        """
        metadata = {item: self.meta[item] for item in self.locals}
        sq_file = os.path.basename(self.file)
        title = os.path.splitext(sq_file)[0]
        metadata['uuid'] = title
        return metadata
    
    def export2sqlite(self, dbfile):
        """
        Export relevant metadata to an SQLite database

        Parameters
        ----------
        dbfile: str
            the database file

        """
        with Archive(dbfile) as archive:
            archive.insert(self)
    
    def examine(self, include_folders=False):
        """
        check whether any items in the SAR scene structure (i.e. files/folders) match the regular expression pattern
        defined by the class. On success the item is registered in the object as attribute `file`.

        Parameters
        ----------
        include_folders: bool
            also match folder (or just files)?

        Returns
        -------

        Raises
        -------
        RuntimeError
        """
        files = self.findfiles(self.pattern, include_folders=include_folders)
        if len(files) == 1:
            self.file = files[0]
        elif len(files) == 0:
            raise RuntimeError('scene does not match {} naming convention'.format(type(self).__name__))
        else:
            raise RuntimeError('file ambiguity detected:\n{}'.format('\n'.join(files)))
    
    def findfiles(self, pattern, include_folders=False):
        """
        find files in the scene archive, which match a pattern.

        Parameters
        ----------
        pattern: str
            the regular expression to match
        include_folders: bool
             also match folders (or just files)?
        Returns
        -------
        list[str]
            the matched file names
        
        See Also
        --------
        :func:`spatialist.ancillary.finder`
        """
        foldermode = 1 if include_folders else 0
        
        files = finder(target=self.scene, matchlist=[pattern],
                       foldermode=foldermode, regex=True)
        
        if os.path.isdir(self.scene) \
                and re.search(pattern, os.path.basename(self.scene)) \
                and include_folders:
            files.append(self.scene)
        
        return files
    
    def gdalinfo(self):
        """
        read metadata directly from the GDAL SAR image drivers

        Returns
        -------
        dict
            the metadata attributes
        """
        files = self.findfiles(r'(?:\.[NE][12]$|DAT_01\.001$|product\.xml|manifest\.safe$)')
        
        if len(files) == 1:
            prefix = {'zip': '/vsizip/', 'tar': '/vsitar/', None: ''}[self.compression]
            header = files[0]
        elif len(files) > 1:
            raise RuntimeError('file ambiguity detected')
        else:
            raise RuntimeError('file type not supported')
        
        meta = {}
        
        ext_lookup = {'.N1': 'ASAR', '.E1': 'ERS1', '.E2': 'ERS2'}
        extension = os.path.splitext(header)[1]
        if extension in ext_lookup:
            meta['sensor'] = ext_lookup[extension]
        
        img = gdal.Open(prefix + header, GA_ReadOnly)
        gdalmeta = img.GetMetadata()
        meta['samples'], meta['lines'], meta['bands'] = img.RasterXSize, img.RasterYSize, img.RasterCount
        meta['projection'] = img.GetGCPProjection()
        meta['gcps'] = [((x.GCPPixel, x.GCPLine), (x.GCPX, x.GCPY, x.GCPZ)) for x in img.GetGCPs()]
        img = None
        
        for item in gdalmeta:
            entry = [item, parse_literal(gdalmeta[item].strip())]
            
            try:
                entry[1] = self.parse_date(str(entry[1]))
            except ValueError:
                pass
            
            if re.search('LAT|LONG', entry[0]):
                entry[1] /= 1000000.
            meta[entry[0]] = entry[1]
        return meta
    
    @abc.abstractmethod
    def getCorners(self):
        """
        derive the corner coordinates from a SAR scene

        Returns
        -------
        dict
            dictionary with keys `xmin`, `xmax`, `ymin` and `ymax`
        """
        raise NotImplementedError
    
    def getFileObj(self, filename):
        """
        Load a file into a readable file object.

        Parameters
        ----------
        filename: str
            the name of a file in the scene archive, easiest to get with method :meth:`~ID.findfiles`

        Returns
        -------
        ~io.BytesIO
            a file pointer object
        """
        return getFileObj(self.scene, filename)
    
    def getGammaImages(self, directory=None):
        """
        list all files processed by GAMMA

        Parameters
        ----------
        directory: str or None
            the directory to be scanned; if left empty the object attribute `gammadir` is scanned

        Returns
        -------
        list
            the file names of the images processed by GAMMA

        Raises
        -------
        RuntimeError
        """
        if directory is None:
            if hasattr(self, 'gammadir'):
                directory = self.gammadir
            else:
                raise RuntimeError(
                    'directory missing; please provide directory to function or define object attribute "gammadir"')
        return [x for x in finder(directory, [self.outname_base()], regex=True) if
                not re.search(r'\.(?:par|hdr|aux\.xml|swp|sh)$', x)]
    
    def getHGT(self):
        """
        get the names of all SRTM HGT tiles overlapping with the SAR scene

        Returns
        -------
        list
            names of the SRTM HGT tiles
        """
        
        corners = self.getCorners()
        
        # generate sequence of integer coordinates marking the tie points of the overlapping hgt tiles
        lat = range(int(float(corners['ymin']) // 1), int(float(corners['ymax']) // 1) + 1)
        lon = range(int(float(corners['xmin']) // 1), int(float(corners['xmax']) // 1) + 1)
        
        # convert coordinates to string with leading zeros and hemisphere identification letter
        lat = [str(x).zfill(2 + len(str(x)) - len(str(x).strip('-'))) for x in lat]
        lat = [x.replace('-', 'S') if '-' in x else 'N' + x for x in lat]
        
        lon = [str(x).zfill(3 + len(str(x)) - len(str(x).strip('-'))) for x in lon]
        lon = [x.replace('-', 'W') if '-' in x else 'E' + x for x in lon]
        
        # concatenate all formatted latitudes and longitudes with each other as final product
        return [x + y + '.hgt' for x in lat for y in lon]
    
    def is_processed(self, outdir, recursive=False):
        """
        check whether a scene has already been processed and stored in the defined output directory
        (and subdirectories if scanned recursively)

        Parameters
        ----------
        outdir: str
            the directory to be checked

        Returns
        -------
        bool
            does an image matching the scene pattern exist?
        """
        if os.path.isdir(outdir):
            # '{}.*tif$'.format(self.outname_base())
            return len(finder(outdir, [self.outname_base()], regex=True, recursive=recursive)) != 0
        else:
            return False
    
    def outname_base(self, extensions=None):
        """
        parse a string containing basic information about the scene in standardized format.
        Currently this id contains the sensor (4 digits), acquisition mode (4 digits), orbit (1 digit)
        and acquisition start time (15 digits)., e.g. `S1A__IW___A_20150523T122350`
        
        Parameters
        ----------
        extensions: list of str
            the names of additional parameters to append to the basename, e.g. ``['orbitNumber_rel']``
        Returns
        -------
        str
            a standardized name unique to the scene
            
        """
        
        fields = ('{:_<4}'.format(self.sensor),
                  '{:_<4}'.format(self.acquisition_mode),
                  self.orbit,
                  self.start)
        out = '_'.join(fields)
        if isinstance(extensions, list) and len(extensions) is not None:
            ext = '_'.join([str(getattr(self, key)) for key in extensions])
            out += '_' + ext
        return out
    
    @staticmethod
    def parse_date(x):
        """
        this function gathers known time formats provided in the different SAR products and converts them to a common
        standard of the form YYYYMMDDTHHMMSS.

        Parameters
        ----------
        x: str
            the time stamp

        Returns
        -------
        str
            the converted time stamp in format YYYYmmddTHHMMSS
        """
        return parse_date(x)
    
    @abc.abstractmethod
    def quicklook(self, outname, format='kmz'):
        """
        export a quick look image of the scene

        Parameters
        ----------
        outname: str
            the name of the output file
        format: str
            the format of the file to write;
            currently only kmz is supported

        Returns
        -------

        Examples
        --------

        >>> from pyroSAR import identify
        >>> scene = identify('S1A_IW_GRDH_1SDV_20180101T170648_20180101T170713_019964_021FFD_DA78.zip')
        >>> scene.quicklook('S1A__IW___A_20180101T170648.kmz')
        """
        raise NotImplementedError
    
    def summary(self):
        """
        print the set of standardized scene metadata attributes

        Returns
        -------

        """
        print(self.__str__())
    
    @abc.abstractmethod
    def scanMetadata(self):
        """
        scan SAR scenes for metadata attributes.
        The returned dictionary is registered as attribute `meta` by the class upon object initialization.
        This dictionary furthermore needs to return a set of standardized attribute keys,
        which are directly registered as object attributes.

        Returns
        -------
        dict
            the derived attributes

        """
        raise NotImplementedError
    
    @abc.abstractmethod
    def unpack(self, directory, overwrite=False, exist_ok=False):
        """
        Unpack the SAR scene into a defined directory.

        Parameters
        ----------
        directory: str
            the base directory into which the scene is unpacked
        overwrite: bool
            overwrite an existing unpacked scene?
        exist_ok: bool
            allow existing output files and do not create new ones?

        Returns
        -------

        """
        raise NotImplementedError
    
    def _unpack(self, directory, offset=None, overwrite=False, exist_ok=False):
        """
        general function for unpacking scene archives; to be called by implementations of ID.unpack.
        Will reset object attributes `scene` and `file` to point to the locations of the unpacked scene
        
        Parameters
        ----------
        directory: str
            the name of the directory in which the files are written
        offset: str
            an archive directory offset; to be defined if only a subdirectory is to be unpacked (see e.g. TSX.unpack)
        overwrite: bool
            should an existing directory be overwritten?
        exist_ok: bool
            do not attempt unpacking if the target directory already exists? Ignored if ``overwrite==True``
        
        Returns
        -------
        
        """
        do_unpack = True
        if os.path.isdir(directory):
            if overwrite:
                shutil.rmtree(directory)
            else:
                if exist_ok:
                    do_unpack = False
                else:
                    raise RuntimeError('target scene directory already exists: {}'.format(directory))
        os.makedirs(directory, exist_ok=True)
        
        if do_unpack:
            if tf.is_tarfile(self.scene):
                archive = tf.open(self.scene, 'r')
                names = archive.getnames()
                if offset is not None:
                    names = [x for x in names if x.startswith(offset)]
                header = os.path.commonprefix(names)
                
                if header in names:
                    if archive.getmember(header).isdir():
                        for item in sorted(names):
                            if item != header:
                                member = archive.getmember(item)
                                if offset is not None:
                                    member.name = member.name.replace(offset + '/', '')
                                archive.extract(member, directory)
                        archive.close()
                    else:
                        archive.extractall(directory)
                        archive.close()
            
            elif zf.is_zipfile(self.scene):
                archive = zf.ZipFile(self.scene, 'r')
                names = archive.namelist()
                header = os.path.commonprefix(names)
                if header.endswith('/'):
                    for item in sorted(names):
                        if item != header:
                            repl = item.replace(header, '', 1)
                            outname = os.path.join(directory, repl)
                            outname = outname.replace('/', os.path.sep)
                            if item.endswith('/'):
                                os.makedirs(outname)
                            else:
                                try:
                                    with open(outname, 'wb') as outfile:
                                        outfile.write(archive.read(item))
                                except zf.BadZipfile:
                                    log.info('corrupt archive, unpacking failed')
                                    continue
                    archive.close()
                else:
                    archive.extractall(directory)
                    archive.close()
            else:
                log.info('unpacking is only supported for TAR and ZIP archives')
                return
        
        self.scene = directory
        main = os.path.join(self.scene, os.path.basename(self.file))
        self.file = main if os.path.isfile(main) else self.scene def identify(scene):
    """
    identify a SAR scene and return the appropriate metadata handler object

    Parameters
    ----------
    scene: str
        a file or directory name

    Returns
    -------
    a subclass object of :class:`~pyroSAR.drivers.ID`
        a pyroSAR metadata handler
    
    Examples
    --------

    >>> from pyroSAR import identify
    >>> filename = 'S1A_IW_GRDH_1SDV_20180829T170656_20180829T170721_023464_028DE0_F7BD.zip'
    >>> scene = identify(filename)
    >>> print(scene)
    pyroSAR ID object of type SAFE
    acquisition_mode: IW
    cycleNumber: 148
    frameNumber: 167392
    lines: 16703
    orbit: A
    orbitNumber_abs: 23464
    orbitNumber_rel: 117
    polarizations: ['VV', 'VH']
    product: GRD
    projection: +proj=longlat +datum=WGS84 +no_defs
    samples: 26056
    sensor: S1A
    spacing: (10.0, 10.0)
    start: 20180829T170656
    stop: 20180829T170721
    """
    if not os.path.exists(scene):
        raise OSError("No such file or directory: '{}'".format(scene))
    
    for handler in ID.__subclasses__():
        try:
            return handler(scene)
        except (RuntimeError, KeyError):
            pass
    raise RuntimeError('data format not supported') def identify_many(scenes, pbar=False, sortkey=None):
    """
    wrapper function for returning metadata handlers of all valid scenes in a list, similar to function
    :func:`~pyroSAR.drivers.identify`.

    Parameters
    ----------
    scenes: list[str]
        the file names of the scenes to be identified
    pbar: bool
        adds a progressbar if True
    sortkey: str
        sort the handler object list by an attribute
    Returns
    -------
    list[ID]
        a list of pyroSAR metadata handlers
    
    Examples
    --------
    >>> from pyroSAR import identify_many
    >>> files = finder('/path', ['S1*.zip'])
    >>> ids = identify_many(files, pbar=False, sortkey='start')
    """
    idlist = []
    if pbar:
        progress = pb.ProgressBar(max_value=len(scenes)).start()
    else:
        progress = None
    for i, scene in enumerate(scenes):
        if isinstance(scene, ID):
            idlist.append(scene)
        else:
            try:
                id = identify(scene)
                idlist.append(id)
            except RuntimeError:
                continue
            except PermissionError:
                log.warning("Permission denied: '{}'".format(scene))
        if progress is not None:
            progress.update(i + 1)
    if progress is not None:
        progress.finish()
    if sortkey is not None:
        idlist.sort(key=operator.attrgetter(sortkey))
    return idlist class ISPPar(object):
    """
    Reader for ISP parameter files of the GAMMA software package

    This class allows to read all information from files in GAMMA's parameter file format.
    Each key-value pair is parsed and added as attribute. For instance if the parameter file
    contains the pair 'sensor:    TSX-1' an attribute named 'sensor' with the value 'TSX-1' will be available.

    The values are converted to native Python types, while unit identifiers like 'dB' or 'Hz' are removed.
    Please see the GAMMA reference manual for further information on the actual file format.
    
    Parameters
    ----------
    filename: str
        the GAMMA parameter file
    
    Examples
    --------
    >>> from pyroSAR.gamma import ISPPar
    >>> with ISPPar('S1A__IW___A_20141115T181801_VH_grd.par') as par:
    ...     print(par) # print an overview of all available metadata
    ...     print(par.keys) # print all parameter names
    ...     for key, value in par.envidict().items():
    ...         print('{0}: {1}'.format(key, value)) # print the ENVI HDR compliant metadata
    
    Attributes
    ----------
    keys: list
        the names of all parameters
    """
    
    _re_kv_pair = re.compile(r'^(\w+):\s*(.+)\s*')
    _re_float_literal = re.compile(r'^[+-]?(?:(\d*\.\d+)|(\d+\.?))(?:[Ee][+-]?\d+)?')
    
    def __init__(self, filename):
        """Parses an ISP parameter file from disk.

        Args:
            filename: The filename or file object representing the ISP parameter file.
        """
        if isinstance(filename, str):
            par_file = open(filename, 'r')
        else:
            par_file = filename
        
        self.keys = ['filetype']
        
        try:
            content = par_file.read().split('\n')
        except UnicodeDecodeError:
            par_file = codecs.open(filename, 'r', encoding='utf-8', errors='ignore')
            content = par_file.read()
            printable = set(string.printable)
            content = filter(lambda x: x in printable, content)
            content = ''.join(list(content)).split('\n')
        finally:
            par_file.close()
        
        if 'Image Parameter File' in content[0]:
            setattr(self, 'filetype', 'isp')
        elif 'DEM/MAP parameter file' in content[0]:
            setattr(self, 'filetype', 'dem')
        else:
            setattr(self, 'filetype', 'unknown')
        
        for line in content:
            match = ISPPar._re_kv_pair.match(line)
            if not match:
                continue  # Skip malformed lines with no key-value pairs
            key = match.group(1)
            items = match.group(2).split()
            if len(items) == 0:
                value = None
            elif len(items) == 1:
                value = parse_literal(items[0])
            else:
                if not ISPPar._re_float_literal.match(items[0]):
                    # Value is a string literal containing whitespace characters
                    value = match.group(2)
                else:
                    # Evaluate each item and stop at the first non-float literal
                    value = []
                    for i in items:
                        match = ISPPar._re_float_literal.match(i)
                        if match:
                            value.append(parse_literal(match.group()))
                        else:
                            # If the first float literal is immediately followed by a non-float literal handle the
                            # first one as singular value, e.g. in '20.0970 dB'
                            if len(value) == 1:
                                value = value[0]
                            break
            self.keys.append(key)
            setattr(self, key, value)

        if hasattr(self, 'date'):
            try:
                self.date = '{}-{:02d}-{:02d}T{:02d}:{:02d}:{:02f}'.format(*self.date)
            except:
                # if only date available
                self.date = '{}-{:02d}-{:02d}'.format(*self.date)
    
    def __enter__(self):
        return self
    
    def __exit__(self, exc_type, exc_val, exc_tb):
        return
    
    def __getattr__(self, item):
        # will only be run if object has no attribute item
        raise AttributeError("parameter file has no attribute '{}'".format(item))
    
    def __str__(self):
        maxlen = len(max(self.keys, key=len)) + 1
        return '\n'.join(['{key}:{sep}{value}'.format(key=key,
                                                      sep=(maxlen - len(key)) * ' ',
                                                      value=getattr(self, key)) for key in self.keys])
    
    def envidict(self, nodata=None):
        """
        export relevant metadata to an ENVI HDR file compliant format
        
        Parameters
        ----------
        nodata: int, float or None
            a no data value to write to the HDR file via attribute 'data ignore value'
        
        Returns
        -------
        dict
            a dictionary containing attributes translated to ENVI HDR naming
        """
        out = dict(bands=1,
                   header_offset=0,
                   file_type='ENVI Standard',
                   interleave='bsq',
                   sensor_type='Unknown',
                   byte_order=1,
                   wavelength_units='Unknown')
        
        if hasattr(self, 'date'):
            out['acquisition_time'] = self.date + 'Z'
        
        out['samples'] = getattr(self, union(['width', 'range_samples', 'samples'], self.keys)[0])
        out['lines'] = getattr(self, union(['nlines', 'azimuth_lines', 'lines'], self.keys)[0])
        
        dtypes_lookup = {'FCOMPLEX': 6, 'FLOAT': 4, 'REAL*4': 4, 'INTEGER*2': 2, 'SHORT': 12}
        dtype = getattr(self, union(['data_format', 'image_format'], self.keys)[0])
        
        if dtype not in dtypes_lookup.keys():
            raise TypeError('unsupported data type: {}'.format(dtype))
        
        out['data_type'] = dtypes_lookup[dtype]
        
        if nodata is not None:
            out['data_ignore_value'] = nodata
        
        if out['data_type'] == 6:
            out['complex_function'] = 'Power'
        # projections = ['AEAC', 'EQA', 'LCC', 'LCC2', 'OMCH', 'PC', 'PS', 'SCH', 'TM', 'UTM']
        # the corner coordinates are shifted by 1/2 pixel to the Northwest since GAMMA pixel
        # coordinates are defined for the pixel center while in ENVI it is the upper left
        if hasattr(self, 'DEM_projection'):
            if self.DEM_projection == 'UTM':
                hem = 'North' if float(self.false_northing) == 0 else 'South'
                out['map_info'] = ['UTM', '1.0000', '1.0000',
                                   self.corner_east - (abs(self.post_east) / 2),
                                   self.corner_north + (abs(self.post_north) / 2),
                                   str(abs(float(self.post_east))),
                                   str(abs(float(self.post_north))),
                                   self.projection_zone, hem, 'WGS-84', 'units=Meters']
            elif self.DEM_projection == 'EQA':
                out['map_info'] = ['Geographic Lat/Lon', '1.0000', '1.0000',
                                   self.corner_lon - (abs(self.post_lon) / 2),
                                   self.corner_lat + (abs(self.post_lat) / 2),
                                   str(abs(float(self.post_lon))),
                                   str(abs(float(self.post_lat))),
                                   'WGS-84', 'units=Degrees']
            else:
                raise RuntimeError('unsupported projection: {}'.format(self.DEM_projection))
        return out class Namespace(object):
    def __init__(self, directory, basename):
        self.__base = basename
        self.__outdir = directory
        self.__reg = []
    
    def __getitem__(self, item):
        item = str(item).replace('.', '_')
        return self.get(item)
    
    def __getattr__(self, item):
        # will only be run if object has no attribute item
        return '-'
    
    def appreciate(self, keys):
        for key in keys:
            setattr(self, key.replace('.', '_'), os.path.join(self.__outdir, self.__base + '_' + key))
            if key not in self.__reg:
                self.__reg.append(key.replace('.', '_'))
    
    def depreciate(self, keys):
        for key in keys:
            setattr(self, key.replace('.', '_'), '-')
            if key not in self.__reg:
                self.__reg.append(key.replace('.', '_'))
    
    def getall(self):
        out = {}
        for key in self.__reg:
            out[key] = getattr(self, key)
        return out
    
    def select(self, selection):
        return [getattr(self, key) for key in selection]
    
    def isregistered(self, key):
        return key in self.__reg
    
    def isappreciated(self, key):
        if self.isregistered(key):
            if self.get(key) != '-':
                return True
        return False
    
    def isfile(self, key):
        return hasattr(self, key) and os.path.isfile(getattr(self, key))
    
    def get(self, key):
        return getattr(self, key) def par2hdr(parfile, hdrfile, modifications=None, nodata=None):
    """
    Create an ENVI HDR file from a GAMMA PAR file
    
    Parameters
    ----------
    parfile: str
        the GAMMA parfile
    hdrfile: str
        the ENVI HDR file
    modifications: dict or None
        a dictionary containing value deviations to write to the HDR file
    nodata: int, float or None
        a no data value to write to the HDR file via attribute 'data ignore value'

    Returns
    -------
    
    Examples
    --------
    >>> from pyroSAR.gamma.auxil import par2hdr
    >>> par2hdr('dem_seg.par', 'inc.hdr')
    # write a HDR file for byte data based on a parfile of float data
    >>> par2hdr('dem_seg.par', 'ls_map.hdr', modifications={'data_type': 1})
    
    See Also
    --------
    :class:`spatialist.envi.HDRobject`
    :func:`spatialist.envi.hdr`
    """
    
    with ISPPar(parfile) as par:
        items = par.envidict(nodata)
        if modifications is not None:
            items.update(modifications)
        hdr(items, hdrfile) def multilook_factors(source_rg, source_az, target, geometry, incidence):
    """
    compute multi-looking factors to approximate a square pixel with defined target ground range pixel spacing.
    
    Parameters
    ----------
    source_rg: int or float
        the range pixel spacing
    source_az: int or float
        the azimuth pixel spacing
    target: int or float
        the target pixel spacing of an approximately square pixel
    geometry: str
        the imaging geometry; either 'SLANT_RANGE' or 'GROUND_RANGE'
    incidence: int or float
        the angle of incidence

    Returns
    -------
    tuple[int]
        the multi-looking factors as (range looks, azimuth looks)
    
    Examples
    --------
    >>> from pyroSAR.ancillary import multilook_factors
    >>> rlks, azlks = multilook_factors(source_rg=2, source_az=13, target=10,
    >>>                                 geometry='SLANT_RANGE', incidence=39)
    >>> print(rlks, azlks)
    4 1
    """
    azlks = int(round(float(target) / source_az))
    azlks = azlks if azlks > 0 else 1
    if geometry == 'SLANT_RANGE':
        rlks = float(azlks) * source_az * sin(radians(incidence)) / source_rg
    elif geometry == 'GROUND_RANGE':
        rlks = float(azlks) * source_az / source_rg
    else:
        raise ValueError("parameter 'geometry' must be either 'SLANT_RANGE' or 'GROUND_RANGE'")
    
    rlks = int(round(rlks))
    return rlks, azlks def hasarg(func, arg):
    """
    simple check whether a function takes a parameter as input
    
    Parameters
    ----------
    func: function
        the function to be checked
    arg: str
        the argument name to be found

    Returns
    -------
    bool
        does the function take this as argument?
    """
    return arg in getargs(func) def groupby(images, attribute):
    """
    group a list of images by a metadata attribute
    
    Parameters
    ----------
    images: list of str
        the names of the images to be sorted
    attribute: str
        the name of the attribute used for sorting;
        see :func:`parse_datasetname` for options
    
    Returns
    -------
    list of lists
        a list containing a list with image names ofr each group
    """
    images_sort = sorted(images, key=lambda x: re.search(product_pattern, x).group(attribute))
    out_meta = [[parse_datasetname(images_sort.pop(0))]]
    while len(images_sort) > 0:
        filename = images_sort.pop(0)
        meta = parse_datasetname(filename)
        
        if out_meta[-1][0][attribute] == meta[attribute]:
            out_meta[-1].append(meta)
        else:
            out_meta.append([meta])
    out = [[x['filename'] for x in y] for y in out_meta]
    return out class ExamineSnap(object):
    """
    Class to check if ESA SNAP is installed.
    Upon initialization, this class searches for relevant binaries and the accompanying
    relative directory structure, which uniquely identify an ESA SNAP installation on a system.
    First, all relevant file and folder names are read from the pyroSAR config file if it exists
    and their existence is verified.
    If this fails, a system check is performed to find relevant binaries in the system PATH variable and
    additional files and folders relative to them.
    Furthermore, a snap.auxdata.properties file is scanned for auxiliary data URLs and local storage location.
    This is used by SNAP to manage data from e.g. the SRTM mission. In case SNAP is not installed, the respective
    information is read from a default file delivered with pyroSAR. This has the advantage of using the SNAP download
    URLs and local directory structure without having SNAP installed such that it can be adapted by other SAR software.
    """
    
    def __init__(self):
        
        # define some attributes which identify SNAP
        self.identifiers = ['path', 'gpt', 'etc', 'auxdata']
        
        # a list of relevant sections
        self.sections = ['SNAP', 'OUTPUT', 'SNAP_SUFFIX']
        
        # try reading all necessary attributes from the config file
        self.__read_config()
        
        # if SNAP could not be identified from the config attributes, do a system search for it
        if not self.__is_identified():
            log.debug('identifying SNAP')
            self.__identify_snap()
        
        # if the auxdatapath attribute was not yet set, create a default directory
        if not hasattr(self, 'auxdatapath'):
            self.auxdatapath = os.path.join(os.path.expanduser('~'), '.snap', 'auxdata')
            os.makedirs(self.auxdatapath, exist_ok=True)
        
        # if the SNAP auxdata properties attribute was not yet identified,
        # point it to the default file delivered with pyroSAR
        if not hasattr(self, 'properties'):
            # log.info('using default properties file..')
            template = os.path.join('snap', 'data', 'snap.auxdata.properties')
            self.properties = pkg_resources.resource_filename(__name__, template)
        
        # if the SNAP suffices attribute was not yet identified,
        # point it to the default file delivered with pyroSAR
        if not hasattr(self, 'suffices'):
            template = os.path.join('snap', 'data', 'snap.suffices.properties')
            fname_suffices = pkg_resources.resource_filename(__name__, template)
            with open(fname_suffices, 'r') as infile:
                content = infile.read().split('\n')
            self.__suffices = {k: v for k, v in [x.split('=') for x in content]}
        
        # update the snap properties; this reads the 'properties' file and looks for any changes,
        # which are then updated for the object
        self.__update_snap_properties()
        
        # update the config file: this scans for config changes and re-writes the config file if any are found
        self.__update_config()
    
    def __getattr__(self, item):
        raise AttributeError("'ExamineSnap' object has no attribute '{}'".format(item))
    
    def __is_identified(self):
        """
        Check if SNAP has been properly identified, i.e. all paths in self.identifiers
        have been detected and confirmed.
        
        Returns
        -------
        bool
        """
        return sum([hasattr(self, x) for x in self.identifiers]) == len(self.identifiers)
    
    def __identify_snap(self):
        """
        do a comprehensive search for an ESA SNAP installation
        
        Returns
        -------
        bool
            has the SNAP properties file been changed?
        """
        # create a list of possible SNAP executables
        defaults = ['snap64.exe', 'snap32.exe', 'snap.exe', 'snap']
        paths = os.environ['PATH'].split(os.path.pathsep)
        options = [os.path.join(path, option) for path in paths for option in defaults]
        options = [x for x in options if os.path.isfile(x)]
        
        if not hasattr(self, 'path') or not os.path.isfile(self.path):
            executables = options
        else:
            executables = [self.path] + options
        
        if len(executables) == 0:
            log.debug("could not detect any potential 'snap' executables")
        
        # for each possible SNAP executable, check whether additional files and directories exist relative to it
        # to confirm whether it actually is a ESA SNAP installation or something else like e.g. the Ubuntu App Manager
        for path in executables:
            log.debug('checking candidate {}'.format(path))
            if os.path.islink(path):
                path = os.path.realpath(path)
            
            # check whether a directory etc exists relative to the SNAP executable
            etc = os.path.join(os.path.dirname(os.path.dirname(path)), 'etc')
            if not os.path.isdir(etc):
                log.debug("could not find the 'etc' directory")
                continue
            
            # check the content of the etc directory
            auxdata = os.listdir(etc)
            if 'snap.auxdata.properties' not in auxdata:
                log.debug("could not find the 'snap.auxdata.properties' file")
                continue
            else:
                auxdata_properties = os.path.join(etc, 'snap.auxdata.properties')
            
            # identify the gpt executable
            gpt_candidates = finder(os.path.dirname(path), ['gpt', 'gpt.exe'])
            if len(gpt_candidates) == 0:
                log.debug("could not find the 'gpt' executable")
                continue
            else:
                gpt = gpt_candidates[0]
            
            self.path = path
            self.etc = etc
            self.gpt = gpt
            self.auxdata = auxdata
            self.properties = auxdata_properties
            return
        
        log.warning('SNAP could not be identified. If you have installed it please add the path to the SNAP '
                    'executables (bin subdirectory) to the PATH environment. '
                    'E.g. in the Linux .bashrc file add the following line:\nexport PATH=$PATH:path/to/snap/bin"')
    
    def __read_config(self):
        """
        This method reads the config.ini to examine the snap paths.
        If the snap paths are not in the config.ini or the paths are wrong they will be automatically created.

        Returns
        -------

        """
        for attr in self.identifiers + ['auxdatapath', 'properties']:
            self.__read_config_attr(attr, 'SNAP')
        
        snap_properties = {}
        if 'OUTPUT' in config.sections:
            snap_properties = config['OUTPUT']
        if len(snap_properties.keys()) > 0:
            self.snap_properties = snap_properties
        
        suffices = {}
        if 'SNAP_SUFFIX' in config.sections:
            suffices = config['SNAP_SUFFIX']
        if len(suffices.keys()) > 0:
            self.__suffices = suffices
    
    def __read_config_attr(self, attr, section):
        """
        read an attribute from the config file and set it as an object attribute
        
        Parameters
        ----------
        attr: str
            the attribute name
        section: str
            the config section to read the attribute from
        
        Returns
        -------
        
        """
        if section in config.sections:
            if attr in config[section].keys():
                val = config[section][attr]
                if attr in ['path', 'gpt', 'properties']:
                    exist = os.path.isfile(val)
                elif attr == 'auxdata':
                    val = ast.literal_eval(val)
                    exist = isinstance(val, list)
                else:
                    exist = os.path.isdir(val)
                if exist:
                    # log.info('setting attribute {}'.format(attr))
                    setattr(self, attr, val)
    
    def __update_config(self):
        for section in self.sections:
            if section not in config.sections:
                # log.info('creating section {}..'.format(section))
                config.add_section(section)
        
        for key in self.identifiers + ['auxdatapath', 'properties']:
            if hasattr(self, key):
                self.__update_config_attr(key, getattr(self, key), 'SNAP')
        
        for key, value in self.snap_properties.items():
            self.__update_config_attr(key, value, 'OUTPUT')
        
        for key in sorted(self.__suffices.keys()):
            self.__update_config_attr(key, self.__suffices[key], 'SNAP_SUFFIX')
    
    @staticmethod
    def __update_config_attr(attr, value, section):
        if isinstance(value, list):
            value = json.dumps(value)
        
        if attr not in config[section].keys() or config[section][attr] != value:
            # log.info('updating attribute {0}:{1}..'.format(section, attr))
            # log.info('  {0} -> {1}'.format(repr(config[section][attr]), repr(value)))
            config.set(section, key=attr, value=value, overwrite=True)
    
    def __update_snap_properties(self):
        """
        Read the snap.auxdata.properties file entries to object attributes

        Returns
        -------

        """
        pattern = r'^(?P<key>[\w\.]*)\s*=\s*(?P<value>.*)\n'
        
        if not hasattr(self, 'snap_properties'):
            self.snap_properties = {}
        
        demPath = os.path.join(self.auxdatapath, 'dem')
        landCoverPath = os.path.join(self.auxdatapath, 'LandCover')
        
        with open(self.properties, 'r') as prop:
            for line in prop:
                if re.search(pattern, line):
                    key, value = re.match(re.compile(pattern), line).groups()
                    value = value \
                        .replace('${AuxDataPath}', self.auxdatapath) \
                        .replace('${demPath}', demPath) \
                        .replace('${landCoverPath}', landCoverPath) \
                        .replace('\\', '/')
                    if key not in self.snap_properties.keys() or self.snap_properties[key] != value:
                        self.snap_properties[key] = value
    
    def get_suffix(self, operator):
        """
        get the file name suffix for an operator
        
        Parameters
        ----------
        operator: str
            the name of the operator

        Returns
        -------
        str
            the file suffix
        
        Examples
        --------
        >>> from pyroSAR.examine import ExamineSnap
        >>> config = ExamineSnap()
        >>> print(config.get_suffix('Terrain-Flattening'))
        'TF'
        """
        if operator in self.__suffices.keys():
            return self.__suffices[operator]
        else:
            return None
    
    def get_version(self, module):
        """
        Read the version and date of different SNAP modules.
        This scans a file 'messages.log', which is re-written every time SNAP is started.
        
        Parameters
        ----------
        module: str
            one of the following
            
            - core
            - desktop
            - rstbx
            - s1tbx
            - s2tbx
            - s3tbx

        Returns
        -------
        dict
            a dictionary with keys 'version' and 'date'
        """
        # base search patterns for finding the right lines
        patterns = {'core': r'org\.esa\.snap\.snap\.core',
                    'desktop': r'org\.esa\.snap\.snap\.ui',
                    'rstb': r'org\.csa\.rstb\.rstb\.kit',
                    's1tbx': r'org\.esa\.s1tbx\.s1tbx\.kit',
                    's2tbx': r'org\.esa\.s2tbx\.s2tbx\.kit',
                    's3tbx': r'org\.esa\.s3tbx\.s3tbx\.kit'}
        
        if module in patterns.keys():
            pattern = patterns[module]
            pattern += r' \[(?P<version>[0-9.]+) [0-9.]+ (?P<date>[0-9]{12})'
        else:
            raise RuntimeError('module not supported')
        
        system = platform.system()
        if system in ['Linux', 'Darwin']:
            path = os.path.join(os.path.expanduser('~'), '.snap', 'system')
        elif system == 'Windows':
            path = os.path.join(os.environ['APPDATA'], 'SNAP')
        else:
            raise RuntimeError('operating system not supported')
        
        conda_env_path = os.environ.get('CONDA_PREFIX')
        if conda_env_path is not None and conda_env_path in self.gpt:
            fname = os.path.join(conda_env_path, 'snap', '.snap', 'system', 'var', 'log', 'messages.log')
        else:
            fname = os.path.join(path, 'var', 'log', 'messages.log')
        
        if not os.path.isfile(fname):
            try:
                # This will start SNAP and immediately stop it because of the invalid argument.
                # Currently this seems to be the only way to create the messages.log file if it does not exist.
                sp.check_call([self.path, '--nosplash', '--dummytest', '--console', 'suppress'])
            except sp.CalledProcessError:
                pass
        
        if not os.path.isfile(fname):
            raise RuntimeError("cannot find 'messages.log' to read SNAP module versions from.")
        
        with open(fname, 'r') as m:
            content = m.read()
        match = re.search(pattern, content)
        if match is None:
            raise RuntimeError('cannot read version information from {}.\nPlease restart SNAP.'.format(fname))
        return match.groupdict() def do_execute(par, ids, exist_ok):
    """
    small helper function to assess whether a GAMMA command shall be executed.

    Parameters
    ----------
    par: dict
        a dictionary containing all arguments for the command
    ids: list
        the IDs of the output files
    exist_ok: bool
        allow existing output files?

    Returns
    -------
    bool
        execute the command because (a) not all output files exist or (b) existing files are not allowed
    """
    all_exist = all([os.path.isfile(par[x]) for x in ids if par[x] != '-'])
    return (exist_ok and not all_exist) or not exist_ok
import os
import re
import shutil
import zipfile as zf
import logging
from datetime import datetime
from urllib.error import URLError
from spatialist import haversine
from spatialist.ancillary import union, finder
from ..S1 import OSV
from ..drivers import ID, identify, identify_many
from . import ISPPar, Namespace, par2hdr
from ..ancillary import multilook_factors, hasarg, groupby
from pyroSAR.examine import ExamineSnap
from .auxil import do_execute
    from .api import diff, disp, isp, lat
        an SAR scene object of type pyroSAR.ID or any subclass
    directory: str
        the directory to search for GAMMA calibration candidates
    return_fnames: bool
        return the names of the output image files? Default: False.
    logpath: str or None
        a directory to write command logfiles to
    outdir: str or None
        the directory to execute the command in
    shellscript: str or None
        a file to write the GAMMA commands to in shell format

    Returns
    -------

    """
    cname = type(id).__name__
    new = []
    if cname == 'CEOS_PSR':
        for image in id.getGammaImages(directory):
            if image.endswith('_slc'):
                isp.radcal_SLC(SLC=image,
                               SLC_par=image + '.par',
                               CSLC=image + '_cal',
                               CSLC_par=image + '_cal.par',
                               K_dB=id.meta['k_dB'],
                               logpath=logpath,
                               outdir=outdir,
                               shellscript=shellscript)
                par2hdr(image + '_cal.par', image + '_cal.hdr')
                new.append(image + '_cal')
    
    elif cname == 'EORC_PSR':
        for image in id.getGammaImages(directory):
            if image.endswith('_mli'):
                isp.radcal_MLI(MLI=image,
                               MLI_par=image + '.par',
                               OFF_par='-',
                               CMLI=image + '_cal',
                               antenna='-',
                               rloss_flag=0,
                               ant_flag=0,
                               refarea_flag=1,
                               sc_dB=0,
                               K_dB=id.meta['k_dB'],
                               pix_area=image + '_cal_pix_ell',
                               logpath=logpath,
                               outdir=outdir,
                               shellscript=shellscript)
                par2hdr(image + '.par', image + '_cal.hdr')
                par2hdr(image + '.par', image + '_cal_pix_ell' + '.hdr')
                # rename parameter file 
                os.rename(image + '.par', image + '_cal.par')
                new.append(image + '_cal')
    
    elif cname == 'ESA':
        k_db = {'ASAR': 55., 'ERS1': 58.24, 'ERS2': 59.75}[id.sensor]
        inc_ref = 90. if id.sensor == 'ASAR' else 23.
        candidates = [x for x in id.getGammaImages(directory) if re.search('_pri$', x)]
        for image in candidates:
            out = image.replace('pri', 'grd')
            isp.radcal_PRI(PRI=image,
                           PRI_par=image + '.par',
                           GRD=out,
                           GRD_par=out + '.par',
                           K_dB=k_db,
                           inc_ref=inc_ref,
                           logpath=logpath,
                           outdir=outdir,
                           shellscript=shellscript)
            par2hdr(out + '.par', out + '.hdr')
            new.append(out)
    
    elif cname == 'SAFE':
        log.info('calibration already performed during import')
    
    else:
        raise NotImplementedError('calibration for class {} is not implemented yet'.format(cname))
    
    if return_fnames and len(new) > 0:
        return new


def convert2gamma(id, directory, S1_tnr=True, S1_bnr=True,
                  basename_extensions=None, exist_ok=False,
                  return_fnames=False,
                  logpath=None, outdir=None, shellscript=None):
    """
    general function for converting SAR images to GAMMA format

    Parameters
    ----------
    id: ~pyroSAR.drivers.ID
        an SAR scene object of type pyroSAR.ID or any subclass
    directory: str
        the output directory for the converted images
    S1_tnr: bool
        only Sentinel-1: should thermal noise removal be applied to the image?
    S1_bnr: bool
        only Sentinel-1 GRD: should border noise removal be applied to the image?
        This is available since version 20191203, for older versions this argument is ignored.
    basename_extensions: list of str
        names of additional parameters to append to the basename, e.g. ['orbitNumber_rel']
    exist_ok: bool
        allow existing output files and do not create new ones?
    return_fnames: bool
        return the names of the output image files? Default: False.
    logpath: str or None
        a directory to write command logfiles to
    outdir: str or None
        the directory to execute the command in
    shellscript: str or None
        a file to write the GAMMA commands to in shell format

    Returns
    -------
    list or None
        the sorted image file names if ``return_fnames=True`` and None otherwise
    """
    

    if not isinstance(id, ID):