#!/usr/bin/python
# 
# Copyright 2017 University of Southern California
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#    http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Client for generating pyramidal tiles.
"""

import os
import subprocess
import json
import urlparse
import sys
import traceback
import time
import shutil
import smtplib
from email.mime.text import MIMEText
import socket
import czifile
from dateutil.parser import parse
from bioformats import BioformatsClient
from lxml.etree import XMLSyntaxError
from socket import gaierror, EAI_AGAIN

from deriva.core import PollingErmrestCatalog, HatracStore, urlquote

mail_footer = 'Do not reply to this message.  This is an automated message generated by the system, which does not receive email messages.'

class PyramidalClient (object):
    """Network client for generating pyramidal tiles.
    """
    ## Derived from the ermrest iobox service client

    def __init__(self, **kwargs):
        self.metadata = kwargs.get("metadata")
        self.baseuri = kwargs.get("baseuri")
        o = urlparse.urlparse(self.baseuri)
        self.scheme = o[0]
        host_port = o[1].split(":")
        self.host = host_port[0]
        self.path = o.path
        self.port = None
        if len(host_port) > 1:
            self.port = host_port[1]
        self.dzi = kwargs.get("dzi")
        self.thumbnails = kwargs.get("thumbnails")
        self.czi2dzi = kwargs.get("czi2dzi")
        self.viewer = kwargs.get("viewer")
        self.czirules = kwargs.get("czirules")
        self.showinf = kwargs.get("showinf")
        self.data_scratch = kwargs.get("data_scratch")
        self.cookie = kwargs.get("cookie")
        self.store = HatracStore(
            self.scheme, 
            self.host,
            {'cookie': self.cookie}
        )
        self.catalog = PollingErmrestCatalog(
            self.scheme, 
            self.host,
            self.path.split('/')[-1],
            {'cookie': self.cookie}
        )
        self.mail_server = kwargs.get("mail_server")
        self.mail_sender = kwargs.get("mail_sender")
        self.mail_receiver = kwargs.get("mail_receiver")
        self.logger = kwargs.get("logger")
        self.logger.debug('Client initialized.')

    """
    Send email notification
    """
    def sendMail(self, subject, text):
        if self.mail_server and self.mail_sender and self.mail_receiver:
            retry = 0
            ready = False
            while not ready:
                try:
                    msg = MIMEText('%s\n\n%s' % (text, mail_footer), 'plain')
                    msg['Subject'] = subject
                    msg['From'] = self.mail_sender
                    msg['To'] = self.mail_receiver
                    s = smtplib.SMTP(self.mail_server)
                    s.sendmail(self.mail_sender, self.mail_receiver.split(','), msg.as_string())
                    s.quit()
                    self.logger.debug('Sent email notification.')
                    ready = True
                except socket.gaierror as e:
                    if e.errno == socket.EAI_AGAIN:
                        time.sleep(100)
                        retry = retry + 1
                        ready = retry > 10
                    else:
                        ready = True
                    if ready:
                        et, ev, tb = sys.exc_info()
                        self.logger.error('got exception "%s"' % str(ev))
                        self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                except:
                    et, ev, tb = sys.exc_info()
                    self.logger.error('got exception "%s"' % str(ev))
                    self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                    ready = True

    """
    Start the process for generating pyramidal tiles
    """
    def start(self):
        try:
            self.processHistologicalImages()
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: unexpected exception', '%s\nThe process might have been stopped\n' % str(traceback.format_exception(et, ev, tb)))
            raise
        
    def processHistologicalImages(self):
        """
        Query for detecting new slides - the most recently first
        """
        url = '/entity/Histological_Images:HE_Slide/!File_Bytes::null::&Pyramid_URL::null::/Processing_Status=in%%20progress;Processing_Status::null::@sort(%s::desc::)' % (urlquote('RCT'))
        
        resp = self.catalog.get(url)
        resp.raise_for_status()
        slides = resp.json()
        slideids = []
        for slide in slides:
            slideids.append((slide['ID'], slide['Filename'], slide['File_URL'], slide['RCT'], slide['File_MD5'], slide['Name'], slide['RID']))
                
        self.logger.debug('Processing %d HistologicalImages slides(s).' % (len(slideids))) 
                
        for slideId,filename,file_url,creation_time,md5,name,rid in slideids:
            self.logger.debug('Generating pyramidal tiles for the file "%s"' % (filename))
            
            """
            Extract the file from hatrac
            """
            f = self.getHatracFile(filename, file_url)
            
            if f == None:
                continue
            
            """
            Create the directory for the tiles
            """
            year = parse(creation_time).strftime("%Y")
            outdir = '%s/%s/%s' % (self.dzi, year, md5)
            if not os.path.exists(outdir):
                os.makedirs(outdir)
            
            """
            Convert the file to DZI
            """
            returncode = self.convert2dzi(f, outdir)
            
            if returncode != 0:
                """
                Update the slide table with the failure result.
                """
                self.updateAttributes('Histological_Images:HE_Slide',
                                     rid,
                                     ["Thumbnail", "Processing_Status"],
                                     {'RID': rid,
                                      'Thumbnail': '/thumbnails/generic/generic_genetic.png',
                                      'Processing_Status': 'czi2dzi error'
                                      })
                continue
            
            """
            Generate the thumbnail
            """
            thumbnail,urls = self.writeThumbnailImage(f, year, md5)
            
            if thumbnail == None:
                """
                Update the slide table with the failure result.
                """
                self.updateAttributes('Histological_Images:HE_Slide',
                                     rid,
                                     ["Thumbnail", "Processing_Status"],
                                     {'RID': rid,
                                      'Thumbnail': '/thumbnails/generic/generic_genetic.png',
                                      'Processing_Status': 'DZI failure'
                                      })
                continue
                
            """
            Extract the metadata
            """
            self.logger.debug('Extracting metadata for filename "%s"' % (filename)) 
            bioformatsClient = BioformatsClient(showinf=self.showinf, \
                                                czirules=self.czirules, \
                                                cziFile=f, \
                                                logger=self.logger)
            try:
                metadata = bioformatsClient.getMetadata()
                if metadata == None:
                    metadata = {}
                self.logger.debug('Metadata: "%s"' % str(metadata)) 
                os.remove('temp.xml')
            except XMLSyntaxError:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: XMLSyntaxError', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                metadata = {}
                    
            os.remove(f)
            
            """
            Update the slide table with the success result.
            """
            self.updateAttributes('Histological_Images:HE_Slide',
                                 rid,
                                 ["Thumbnail","Pyramid_URL","Processing_Status","uri"],
                                 {'RID': rid,
                                   'Thumbnail': thumbnail,
                                   'Pyramid_URL': '/%s?%s' % (self.viewer, urls),
                                   'uri': '/%s?%s' % (self.viewer, urls),
                                   "Processing_Status": 'success'
                                  })
            
            self.logger.debug('SUCCEEDED created the tiles directory for the file "%s".' % (filename)) 
            
            """
            Update/Create the image entry with the metadata
            """
            obj = {}
            obj['ID'] = slideId
            obj['Name'] = name
            obj['url'] = '/chaise/viewer/#2/Histological_Images:HE_Slide/ID=%d' % slideId
            columns = ['ID', 'Name', 'url']
            for col in self.metadata:
                if col in metadata and metadata[col] != None:
                    columns.append(col)
                    obj[col] = metadata[col]
                    
            """
            Check if we have an update or create
            """
            rid = self.getRID('Histological_Images:HE_Image', 'ID=%d' % slideId)
            if rid != None:
                obj['RID'] = rid
                self.updateAttributes('Histological_Images:HE_Image',
                                     rid,
                                     columns,
                                     obj
                                      )
            else:
                self.createEntity('Histological_Images:HE_Image', obj)
                
            self.logger.debug('SUCCEEDED created the image entry for the file "%s".' % (filename)) 
            
        self.logger.debug('Ended HistologicalImages Slides Processing.') 
        
    """
    Extract the file from hatrac
    """
    def getHatracFile(self, filename, file_url):
        try:
            hatracFile = '%s/%s' % (self.data_scratch, filename)
            self.store.get_obj(file_url, destfilename=hatracFile)
            self.logger.debug('File "%s", %d bytes.' % (hatracFile, os.stat(hatracFile).st_size)) 
            return hatracFile
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: write thumbnail ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            return None

    """
    Generate the thumbnail
    """
    def writeThumbnailImage(self, filename, year, md5):
        try:
            scanDir='%s/%s/%s' % (self.dzi, year, md5)
            channels = []
            for channel in os.listdir(scanDir):
                if os.path.isdir('%s%s%s' % (scanDir, os.sep, channel)):
                   channels.append( channel)
            outdir = '%s/%s' % (self.thumbnails, year)
            if not os.path.exists(outdir):
                os.makedirs(outdir)
            shutil.copyfile('%s/%s/%s/%s/0/0_0.jpg' % (self.dzi, year, md5, channels[0]), '%s/%s.jpg' % (outdir, md5))
            thumbnail = '/thumbnails/%s/%s.jpg' % (urlquote(year), urlquote(md5))
            urls = []
            for channel in channels:
                urls.append('url=/data/%s/%s/%s/ImageProperties.xml' % (year, md5, channel))
            return (thumbnail, '&'.join(urls))
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: write thumbnail ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            os.remove(filename)
            return (None, None)
            
    """
    Convert the input file to DZI
    """
    def convert2dzi(self, filename, outdir):
        try:
            currentDirectory=os.getcwd()
            os.chdir(self.dzi)
            args = [self.czi2dzi, filename, outdir]
            p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            stdoutdata, stderrdata = p.communicate()
            returncode = p.returncode
            os.chdir(currentDirectory)
            
            if returncode != 0:
                self.logger.error('Can not convert czi to dzi for file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata)) 
                self.sendMail('FAILURE Tiles', 'Can not convert czi to dzi for file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata))
                os.remove(filename)
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: czi2dzi ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            os.chdir(currentDirectory)
            self.logger.error('Can not generate pyramidal tiles for the file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata)) 
            self.sendMail('FAILURE Tiles', 'Can not generate pyramidal tiles for the file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata))
            returncode = 1
            
        return returncode
            
        
    """
    Update the ermrest attributes
    """
    def updateAttributes (self, path, rid, columns, row):
        """
        Update the ermrest attributes with the row values.
        """
        try:
            columns = ','.join([urlquote(col) for col in columns])
            url = '/attributegroup/%s/RID;%s' % (path, columns)
            resp = self.catalog.put(
                url,
                json=[row]
            )
            resp.raise_for_status()
            self.logger.debug('SUCCEEDED updated the table "%s" for the RID "%s"  with "%s".' % (path, rid, json.dumps(row, indent=4))) 
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            
    """
    Insert a row in a table
    """
    def createEntity (self, path, row):
        """
        Insert the row in the table.
        """
        try:
            url = '/entity/%s' % (path)
            resp = self.catalog.post(
                url,
                json=[row]
            )
            resp.raise_for_status()
            self.logger.debug('SUCCEEDED created in the table "%s" the entry "%s".' % (path, json.dumps(row, indent=4))) 
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))

    """
    Check if an entry exist in the table
    Return the RID if it exists, and None otherwise
    """
    def getRID (self, path, predicate):
        """
        Get the RID of the row.
        """
        try:
            RID = None
            url = '/entity/%s/%s' % (path, predicate)
            resp = self.catalog.get(url)
            resp.raise_for_status()
            rows = resp.json()
            if len(rows) == 1:
                RID = rows[0]['RID']
            self.logger.debug('RID for the url = "%s" is "%s".' % (url, RID)) 
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
        
        return RID


#!/usr/bin/python
# 
# Copyright 2017 University of Southern California
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#    http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Client for generating pyramidal tiles.
"""

import os
import subprocess
import json
import urlparse
import sys
import traceback
import time
import shutil
import smtplib
from email.mime.text import MIMEText
import socket
import czifile
from dateutil.parser import parse
from bioformats import BioformatsClient
from lxml.etree import XMLSyntaxError
from socket import gaierror, EAI_AGAIN

from deriva.core import PollingErmrestCatalog, HatracStore, urlquote

mail_footer = 'Do not reply to this message.  This is an automated message generated by the system, which does not receive email messages.'

class PyramidalClient (object):
    """Network client for generating pyramidal tiles.
    """
    ## Derived from the ermrest iobox service client

    def __init__(self, **kwargs):
        self.metadata = kwargs.get("metadata")
        self.baseuri = kwargs.get("baseuri")
        o = urlparse.urlparse(self.baseuri)
        self.scheme = o[0]
        host_port = o[1].split(":")
        self.host = host_port[0]
        self.path = o.path
        self.port = None
        if len(host_port) > 1:
            self.port = host_port[1]
        self.dzi = kwargs.get("dzi")
        self.thumbnails = kwargs.get("thumbnails")
        self.czi2dzi = kwargs.get("czi2dzi")
        self.viewer = kwargs.get("viewer")
        self.czirules = kwargs.get("czirules")
        self.showinf = kwargs.get("showinf")
        self.data_scratch = kwargs.get("data_scratch")
        self.cookie = kwargs.get("cookie")
        self.store = HatracStore(
            self.scheme, 
            self.host,
            {'cookie': self.cookie}
        )
        self.catalog = PollingErmrestCatalog(
            self.scheme, 
            self.host,
            self.path.split('/')[-1],
            {'cookie': self.cookie}
        )
        self.mail_server = kwargs.get("mail_server")
        self.mail_sender = kwargs.get("mail_sender")
        self.mail_receiver = kwargs.get("mail_receiver")
        self.logger = kwargs.get("logger")
        self.logger.debug('Client initialized.')

    """
    Send email notification
    """
    def sendMail(self, subject, text):
        if self.mail_server and self.mail_sender and self.mail_receiver:
            retry = 0
            ready = False
            while not ready:
                try:
                    msg = MIMEText('%s\n\n%s' % (text, mail_footer), 'plain')
                    msg['Subject'] = subject
                    msg['From'] = self.mail_sender
                    msg['To'] = self.mail_receiver
                    s = smtplib.SMTP(self.mail_server)
                    s.sendmail(self.mail_sender, self.mail_receiver.split(','), msg.as_string())
                    s.quit()
                    self.logger.debug('Sent email notification.')
                    ready = True
                except socket.gaierror as e:
                    if e.errno == socket.EAI_AGAIN:
                        time.sleep(100)
                        retry = retry + 1
                        ready = retry > 10
                    else:
                        ready = True
                    if ready:
                        et, ev, tb = sys.exc_info()
                        self.logger.error('got exception "%s"' % str(ev))
                        self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                except:
                    et, ev, tb = sys.exc_info()
                    self.logger.error('got exception "%s"' % str(ev))
                    self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                    ready = True

    """
    Start the process for generating pyramidal tiles
    """
    def start(self):
        try:
            self.processHistologicalImages()
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: unexpected exception', '%s\nThe process might have been stopped\n' % str(traceback.format_exception(et, ev, tb)))
            raise
        
    def processHistologicalImages(self):
        """
        Query for detecting new slides - the most recently first
        """
        url = '/entity/Histological_Images:HE_Slide/!File_Bytes::null::&Pyramid_URL::null::/Processing_Status=in%%20progress;Processing_Status::null::@sort(%s::desc::)' % (urlquote('RCT'))
        
        resp = self.catalog.get(url)
        resp.raise_for_status()
        slides = resp.json()
        slideids = []
        for slide in slides:
            slideids.append((slide['ID'], slide['Filename'], slide['File_URL'], slide['RCT'], slide['File_MD5'], slide['Name'], slide['RID']))
                
        self.logger.debug('Processing %d HistologicalImages slides(s).' % (len(slideids))) 
                
        for slideId,filename,file_url,creation_time,md5,name,rid in slideids:
            self.logger.debug('Generating pyramidal tiles for the file "%s"' % (filename))
            
            """
            Extract the file from hatrac
            """
            f = self.getHatracFile(filename, file_url)
            
            if f == None:
                continue
            
            """
            Create the directory for the tiles
            """
            year = parse(creation_time).strftime("%Y")
            outdir = '%s/%s/%s' % (self.dzi, year, md5)
            if not os.path.exists(outdir):
                os.makedirs(outdir)
            
            """
            Convert the file to DZI
            """
            returncode = self.convert2dzi(f, outdir)
            
            if returncode != 0:
                """
                Update the slide table with the failure result.
                """
                self.updateAttributes('Histological_Images:HE_Slide',
                                     rid,
                                     ["Thumbnail", "Processing_Status"],
                                     {'RID': rid,
                                      'Thumbnail': '/thumbnails/generic/generic_genetic.png',
                                      'Processing_Status': 'czi2dzi error'
                                      })
                continue
            
            """
            Generate the thumbnail
            """
            thumbnail,urls = self.writeThumbnailImage(f, year, md5)
            
            if thumbnail == None:
                """
                Update the slide table with the failure result.
                """
                self.updateAttributes('Histological_Images:HE_Slide',
                                     rid,
                                     ["Thumbnail", "Processing_Status"],
                                     {'RID': rid,
                                      'Thumbnail': '/thumbnails/generic/generic_genetic.png',
                                      'Processing_Status': 'DZI failure'
                                      })
                continue
                
            """
            Extract the metadata
            """
            self.logger.debug('Extracting metadata for filename "%s"' % (filename)) 
            bioformatsClient = BioformatsClient(showinf=self.showinf, \
                                                czirules=self.czirules, \
                                                cziFile=f, \
                                                logger=self.logger)
            try:
                metadata = bioformatsClient.getMetadata()
                if metadata == None:
                    metadata = {}
                self.logger.debug('Metadata: "%s"' % str(metadata)) 
                os.remove('temp.xml')
            except XMLSyntaxError:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: XMLSyntaxError', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                metadata = {}
                    
            os.remove(f)
            
            """
            Update the slide table with the success result.
            """
            self.updateAttributes('Histological_Images:HE_Slide',
                                 rid,
                                 ["Thumbnail","Pyramid_URL","Processing_Status","uri"],
                                 {'RID': rid,
                                   'Thumbnail': thumbnail,
                                   'Pyramid_URL': '/%s?%s' % (self.viewer, urls),
                                   'uri': '/%s?%s' % (self.viewer, urls),
                                   "Processing_Status": 'success'
                                  })
            
            self.logger.debug('SUCCEEDED created the tiles directory for the file "%s".' % (filename)) 
            
            """
            Update/Create the image entry with the metadata
            """
            obj = {}
            obj['ID'] = slideId
            obj['Name'] = name
            obj['url'] = '/chaise/viewer/#2/Histological_Images:HE_Slide/ID=%d' % slideId
            columns = ['ID', 'Name', 'url']
            for col in self.metadata:
                if col in metadata and metadata[col] != None:
                    columns.append(col)
                    obj[col] = metadata[col]
                    
            """
            Check if we have an update or create
            """
            rid = self.getRID('Histological_Images:HE_Image', 'ID=%d' % slideId)
            if rid != None:
                obj['RID'] = rid
                self.updateAttributes('Histological_Images:HE_Image',
                                     rid,
                                     columns,
                                     obj
                                      )
            else:
                self.createEntity('Histological_Images:HE_Image', obj)
                
            self.logger.debug('SUCCEEDED created the image entry for the file "%s".' % (filename)) 
            
        self.logger.debug('Ended HistologicalImages Slides Processing.') 
        
    """
    Extract the file from hatrac
    """
    def getHatracFile(self, filename, file_url):
        try:
            hatracFile = '%s/%s' % (self.data_scratch, filename)
            self.store.get_obj(file_url, destfilename=hatracFile)
            self.logger.debug('File "%s", %d bytes.' % (hatracFile, os.stat(hatracFile).st_size)) 
            return hatracFile
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: write thumbnail ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            return None

    """
    Generate the thumbnail
    """
    def writeThumbnailImage(self, filename, year, md5):
        try:
            scanDir='%s/%s/%s' % (self.dzi, year, md5)
            channels = []
            for channel in os.listdir(scanDir):
                if os.path.isdir('%s%s%s' % (scanDir, os.sep, channel)):
                   channels.append( channel)
            outdir = '%s/%s' % (self.thumbnails, year)
            if not os.path.exists(outdir):
                os.makedirs(outdir)
            shutil.copyfile('%s/%s/%s/%s/0/0_0.jpg' % (self.dzi, year, md5, channels[0]), '%s/%s.jpg' % (outdir, md5))
            thumbnail = '/thumbnails/%s/%s.jpg' % (urlquote(year), urlquote(md5))
            urls = []
            for channel in channels:
                urls.append('url=/data/%s/%s/%s/ImageProperties.xml' % (year, md5, channel))
            return (thumbnail, '&'.join(urls))
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: write thumbnail ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            os.remove(filename)
            return (None, None)
            
    """
    Convert the input file to DZI
    """
    def convert2dzi(self, filename, outdir):
        try:
            currentDirectory=os.getcwd()
            os.chdir(self.dzi)
            args = [self.czi2dzi, filename, outdir]
            p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            stdoutdata, stderrdata = p.communicate()
            returncode = p.returncode
            os.chdir(currentDirectory)
            
            if returncode != 0:
                self.logger.error('Can not convert czi to dzi for file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata)) 
                self.sendMail('FAILURE Tiles', 'Can not convert czi to dzi for file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata))
                os.remove(filename)
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: czi2dzi ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            os.chdir(currentDirectory)
            self.logger.error('Can not generate pyramidal tiles for the file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata)) 
            self.sendMail('FAILURE Tiles', 'Can not generate pyramidal tiles for the file "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (filename, stdoutdata, stderrdata))
            returncode = 1
            
        return returncode
            
        
    """
    Update the ermrest attributes
    """
    def updateAttributes (self, path, rid, columns, row):
        """
        Update the ermrest attributes with the row values.
        """
        try:
            columns = ','.join([urlquote(col) for col in columns])
            url = '/attributegroup/%s/RID;%s' % (path, columns)
            resp = self.catalog.put(
                url,
                json=[row]
            )
            resp.raise_for_status()
            self.logger.debug('SUCCEEDED updated the table "%s" for the RID "%s"  with "%s".' % (path, rid, json.dumps(row, indent=4))) 
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            
    """
    Insert a row in a table
    """
    def createEntity (self, path, row):
        """
        Insert the row in the table.
        """
        try:
            url = '/entity/%s' % (path)
            resp = self.catalog.post(
                url,
                json=[row]
            )
            resp.raise_for_status()
            self.logger.debug('SUCCEEDED created in the table "%s" the entry "%s".' % (path, json.dumps(row, indent=4))) 
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))

    """
    Check if an entry exist in the table
    Return the RID if it exists, and None otherwise
    """
    def getRID (self, path, predicate):
        """
        Get the RID of the row.
        """
        try:
            RID = None
            url = '/entity/%s/%s' % (path, predicate)
            resp = self.catalog.get(url)
            resp.raise_for_status()
            rows = resp.json()
            if len(rows) == 1:
                RID = rows[0]['RID']
            self.logger.debug('RID for the url = "%s" is "%s".' % (url, RID)) 
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
        
        return RID


#!/usr/bin/python # # Copyright 2017 University of Southern California # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #    http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Client for generating pyramidal tiles. Network client for generating pyramidal tiles. ## Derived from the ermrest iobox service client Send email notification Start the process for generating pyramidal tiles Query for detecting new slides - the most recently first Extract the file from hatrac Create the directory for the tiles Convert the file to DZI Update the slide table with the failure result. Generate the thumbnail Update the slide table with the failure result. Extract the metadata Update the slide table with the success result. Update/Create the image entry with the metadata #2/Histological_Images:HE_Slide/ID=%d' % slideId Check if we have an update or create Extract the file from hatrac Generate the thumbnail Convert the input file to DZI Update the ermrest attributes Update the ermrest attributes with the row values. Insert a row in a table Insert the row in the table. Check if an entry exist in the table     Return the RID if it exists, and None otherwise Get the RID of the row.