STATUS_STOP    = 0 # Suicide; the test suite is immediately killed
STATUS_PAUSED  = 1 # After the current test is finished, the suite is frozen
STATUS_RUNNING = 2 # The suite is running
STATUS_RESUME  = 3 # Continue the paused tests
STATUS_INTERACT = 4 # Execution is paused, waiting for the user interaction
STATUS_INVALID = 8 # Invalid status
STATUS_RESTART = 5 # Restarting central engine
STATUS_PENDING  = 10 # Not yet run, waiting to start
STATUS_WORKING  = 1  # Is running now
STATUS_PASS     = 2  # Test is finished successful
STATUS_FAIL     = 3  # Test failed
STATUS_SKIPPED  = 4  # When file doesn't exist, or test has flag `runnable = False`
STATUS_ABORTED  = 5  # When test is stopped while running
STATUS_NOT_EXEC = 6  # Not executed, is sent from TC when tests are paused, and then stopped instead of being resumed
STATUS_TIMEOUT  = 7  # When timer expired
STATUS_INVALID  = 8  # When timer expired, the next run
STATUS_WAITING  = 9  # Is waiting for another test
ROLES = [

	'RUN_TESTS',		# Can run tests ? (server + applet)
	'EDIT_TC',			# Can edit test files ? (applet)

	'CREATE_PROJECT',	# Can create new projects ? (applet)
	'CHANGE_PROJECT',	# Can change defined projects ? (applet)
	'DELETE_PROJECT',	# Can delete projects ? (applet)

	'VIEW_REPORTS',		# Can view ? (server + applet)
	'CHANGE_FWM_CFG',	# Can change his main config ? (applet)
	'CHANGE_GLOBALS',	# Can change global parameters ? (applet)
	'CHANGE_DB_CFG',	# Can change database config ? (applet)
	'CHANGE_EML_CFG',	# Can change e-mail config ? (applet)
	'CHANGE_PLUGINS',	# Can load/ unload plugins ? (applet)

	'CHANGE_TESTBED',	# Can change the global testbed ? (server + applet)
	'CHANGE_SUT',		# Can change the global SUT ? (server + applet)
	'CHANGE_SERVICES',	# Can start/ stop services ? (server + applet)
	'CHANGE_USERS',		# Can create, change and delete users ? (server + applet)
	'LOCK_TESTBED',		# Can lock testbed
	'LOCK_SUT',			# Can lock sut
	'RESTART_SERVER'	# Can restart the central engine
]
EXEC_STATUS = {'stopped':STATUS_STOP, 'paused':STATUS_PAUSED, 'running':STATUS_RUNNING, 'resume':STATUS_RESUME,
	'invalid':STATUS_INVALID, 'interact': STATUS_INTERACT, 'restart': STATUS_RESTART}
FWMCONFIG_TAGS = (
	{'name':'ep_names',		'tag':'EpNames',			'default':''},
	{'name':'tests_path',	'tag':'TestCaseSourcePath',	'default':''},
	{'name':'logs_path',	'tag':'LogsPath',			'default':''},
	{'name':'libs_path',	'tag':'LibsPath',			'default':''},
	{'name':'db_config',	'tag':'DbConfigFile',		'default':''},
	{'name':'use_shared_db',	'tag':'UseSharedDb',	'default':False},
	{'name':'eml_config',	'tag':'EmailConfigFile',	'default':''},
	{'name':'glob_params',	'tag':'GlobalParams',		'default':''},
	{'name':'tcfg_path',	'tag':'TestConfigPath',		'default':''},
	{'name':'archive_logs_path', 'tag':'ArchiveLogsPath', 'default': False},
	{'name':'archive_logs_path_active', 'tag':'ArchiveLogsPathActive', 'default': ''},
	{'name':'sut_path',		'tag':'SutPath', 'default': ''},
	{'name':'sys_sut_path',	'tag':'SysSutPath', 'default': ''},
	{'name':'projects_path',		'tag':'UsersPath',			'default':''},
	{'name':'predefined_path',	'tag':'PredefinedSuitesPath',	'default':''},
)
PROJECTCONFIG_TAGS = (
	{'name':'exit_on_test_fail','tag':'stoponfail',			'default':False, 'type':'bool'},
	{'name':'db_auto_save',		'tag':'dbautosave',			'default':''},
	{'name':'tc_delay',			'tag':'tcdelay',			'default':0, 'type':'number'},
	{'name':'libraries',		'tag':'libraries',			'default':''},
	{'name':'dl_libs',			'tag':'DownloadLibraries',	'default':''},
	{'name':'script_pre',		'tag':'ScriptPre',			'default':''},
	{'name':'script_post',		'tag':'ScriptPost',			'default':''},
	{'name':'script_mandatory',	'tag':'PrePostMandatory',	'default':False, 'type':'bool'},
	{'name':'clear_case_view',	'tag':'ClearCaseView'	,	'default':''},
)
SUITES_TAGS = (
	{'name':'sut',			'tag':'SutName',			'default':''},
	{'name':'name',			'tag':'tsName',				'default':''},
	{'name':'pd',			'tag':'PanicDetect',		'default':''},
	{'name':'libraries',	'tag':'libraries',			'default':''},
)
TESTS_TAGS = (
	{'name':'file',			'tag':'tcName',				'default':''},
	{'name':'_dep_id',		'tag':'ID',					'default':''},
	{'name':'_cfg_files',	'tag':'ConfigFiles/Config',	'default':''},
	{'name':'_depend',		'tag':'Dependency',			'default':''},
	{'name':'clearcase',	'tag':'ClearCase',			'default':''},
	{'name':'iterationSave','tag':'iterationSave',		'default':''},
)
DATABASE_EXPORT_VARS = {
	'twister_user': '',
	'twister_rf_fname': '',
	'twister_pf_fname': '',
	'twister_ce_type': '',
	'twister_ce_os': '',
	'twister_ce_hostname':'',
	'twister_ce_ip': '',
	'twister_ce_timezone': '',
	'twister_server_location': '',
	'twister_ce_python_revision': '',
	'twister_ep_name': '',
	'twister_ep_os':'',
	'twister_ep_ip':'',
	'twister_ep_hostname':'',
	'twister_ep_python_revision': '',
	'twister_suite_id': '',
	'twister_suite_name': '',
	'twister_tc_reason': '',
	'twister_tc_iteration': '',
	'twister_tc_full_path': '',
	'twister_tc_name': '',
	'twister_tc_id': '',
	'twister_tc_log': '',
	'twister_tc_log_running': '',
	'twister_tc_log_debug': '',
	'twister_tc_log_test': '',
	'twister_tc_status': '',
	'twister_tc_crash_detected': '',
	'twister_tc_time_elapsed': '',
	'twister_tc_date_started': '',
	'twister_tc_date_finished': '',
	'twister_tc_revision': ''
	}
EMAIL_EXPORT_VARS = {
	'date': '',
	'twister_user': '',
	'twister_ce_hostname': '',
	'twister_ce_ip': '',
	'twister_ce_os': '',
	'twister_ce_python_revision': '',
	'twister_ce_type': '',
	'twister_ep_hostname': '',
	'twister_ep_ip': '',
	'twister_ep_os': '',
	'twister_ep_python_revision': '',
	'twister_pf_fname': '',
	'twister_rf_fname': '',
	'twister_server_location': ''
	} STATUS_NOT_EXEC = 6  # Not executed, is sent from TC when tests are paused, and then stopped instead of being resumed STATUS_STOP    = 0 # Suicide; the test suite is immediately killed STATUS_INVALID = 8 # Invalid status STATUS_INTERACT = 4 # Execution is paused, waiting for the user interaction STATUS_PAUSED  = 1 # After the current test is finished, the suite is frozen STATUS_RUNNING = 2 # The suite is running STATUS_RESUME  = 3 # Continue the paused tests EXEC_STATUS = {'stopped':STATUS_STOP, 'paused':STATUS_PAUSED, 'running':STATUS_RUNNING, 'resume':STATUS_RESUME,
	'invalid':STATUS_INVALID, 'interact': STATUS_INTERACT, 'restart': STATUS_RESTART} ROLES = [

	'RUN_TESTS',		# Can run tests ? (server + applet)
	'EDIT_TC',			# Can edit test files ? (applet)

	'CREATE_PROJECT',	# Can create new projects ? (applet)
	'CHANGE_PROJECT',	# Can change defined projects ? (applet)
	'DELETE_PROJECT',	# Can delete projects ? (applet)

	'VIEW_REPORTS',		# Can view ? (server + applet)
	'CHANGE_FWM_CFG',	# Can change his main config ? (applet)
	'CHANGE_GLOBALS',	# Can change global parameters ? (applet)
	'CHANGE_DB_CFG',	# Can change database config ? (applet)
	'CHANGE_EML_CFG',	# Can change e-mail config ? (applet)
	'CHANGE_PLUGINS',	# Can load/ unload plugins ? (applet)

	'CHANGE_TESTBED',	# Can change the global testbed ? (server + applet)
	'CHANGE_SUT',		# Can change the global SUT ? (server + applet)
	'CHANGE_SERVICES',	# Can start/ stop services ? (server + applet)
	'CHANGE_USERS',		# Can create, change and delete users ? (server + applet)
	'LOCK_TESTBED',		# Can lock testbed
	'LOCK_SUT',			# Can lock sut
	'RESTART_SERVER'	# Can restart the central engine
] class PluginParser(object):
    """
    Requirements: LXML.
    This parser will read user's Plugins.xml.
    """

    def __init__(self, user):

        self.user = user
        user_home = userHome(user)

        if not os.path.isdir('{}/twister'.format(user_home)):
            raise Exception('PluginParser: Cannot find Twister for user `{}`, '\
                'in path `{}/twister`!'.format(user, user_home))

        config_data = '{}/twister/config/plugins.xml'.format(user_home)
        if not os.path.isfile(config_data):
            raise Exception('PluginParser: Cannot find Plugins for user `{}`, '\
                'in path `{}/twister/config`!'.format(user, user_home))

        # Read directly from CE
        xml_data = localFs.read_system_file(config_data)
        self.config = OrderedDict()

        try:
            self.xmlTree = etree.fromstring(xml_data)
        except Exception:
            raise Exception('PluginParser: Cannot access plugins XML data!')

        for plugin in self.xmlTree.xpath('Plugin'):

            if not (plugin.xpath('name/text()') and plugin.xpath('pyfile') and plugin.xpath('jarfile')):
                logWarning('User {}: PluginParser: Invalid config for plugin: `{}`!'.format(self.user, plugin))
                continue

            prop_keys = plugin.xpath('property/propname')
            prop_vals = plugin.xpath('property/propvalue')
            res = dict(zip([k.text for k in prop_keys], [v.text for v in prop_vals])) # Pack Name + Value

            name = plugin.xpath('name')[0].text

            self.config[name] = res

            self.config[name]['jarfile'] = plugin.xpath('jarfile')[0].text.strip() \
                                             if plugin.xpath('jarfile/text()') else ''
            self.config[name]['pyfile'] = plugin.xpath('pyfile')[0].text.\
            strip() if plugin.xpath('pyfile/text()') else ''

            self.config[name]['status'] = plugin.xpath('status')[0].text.\
            strip() if plugin.xpath('status/text()') else ''


    def getPlugins(self):
        """ Return all plugins info """
        logFull('xmlparser:getPlugins')

        Base = BasePlugin.BasePlugin
        py_modules = [k +'::'+ os.path.splitext(self.config[k]['pyfile'])[0]
                      for k in self.config if self.config[k]['status'] == 'enabled']
        plugins = {}

        for module in py_modules:
            name = module.split('::')[0]
            if not name:
                continue
            mod = module.split('::')[1]
            if not mod:
                continue
            if not os.path.isfile('{}/plugins/{}.py'.format(TWISTER_PATH, mod)):
                continue
            plug = None
            try:
                # Import the plugin module
                mm = __import__('plugins.' + mod, fromlist=['Plugin'])
                # Reload all data, just in case
                mm = reload(mm)
                plug = mm.Plugin
            except Exception, e:
                logWarning('User {}: PluginParser ERROR: Unhandled exception' \
                ' in plugin file `{}`! Exception: {}!'.\
                format(self.user, mod, e))
                continue

            if not plug:
                logWarning('User {}: PluginParser ERROR: Plugin `{}` cannot be Null!'.format(self.user, plug))
                continue
            # Check plugin parent. Must be Base Plugin.
            if not issubclass(plug, Base):
                logWarning('User {}: PluginParser ERROR: Plugin `{}` must be' \
                    ' inherited from Base Plugin!'.format(self.user, plug))
                continue

            # Append plugin classes to plugins list
            d = self.config[name]
            d['plugin'] = plug
            plugins[name] = d

        return plugins class ClearCaseParser(object):
    """
    Requirements: LXML.
    This parser will read user's Clearcase.xml.
    """

    def __init__(self, user):

        self.user = user
        user_home = userHome(user)
        self.user_home = user_home
        self.config = {}

        if not os.path.isdir('{}/twister'.format(user_home)):
            raise Exception('ClearCaseParser: Cannot find Twister for user `{}`, '\
                'in path `{}/twister`!'.format(user, user_home))

        config_data = '{}/twister/config/clearcaseconfig.xml'.format(user_home)
        if not os.path.isfile(config_data):
            raise Exception('ClearCaseParser: Cannot find Clearcase XML for user `{}`, '\
                'in path `{}/twister/config`!'.format(user, user_home))

        # Read directly from CE
        xml_data = localFs.read_system_file(config_data)

        try:
            self.xmlTree = etree.fromstring(xml_data)
        except Exception:
            raise Exception('ClearCaseParser: Cannot access Clearcase XML data!')


    def getConfigs(self, filter_tag=None):
        """ Return all ClearCase info. """
        filter_name = False

        # Parse all known FWMCONFIG tags
        for tag_dict in FWMCONFIG_TAGS:
            tag = tag_dict['tag']
            name = tag_dict['name']
            # Filter ?
            if filter_tag and filter_tag != tag:
                continue
            # Found filter ?
            filter_name = name
            xobj = self.xmlTree.xpath('/Root/' + tag)
            # If the tag is active, get the View and the Path
            if len(xobj) >= 1:
                xobj = xobj[0]
                if xobj.get('active') == 'true':
                    path = xobj.get('path')
                    view = xobj.get('view')
                    # actv = xobj.get('activity')
                    self.config[name] = {'path': path, 'view': view, 'actv': ''}

        if filter_tag and filter_name:
            return self.config.get(filter_name, {})
        else:
            return self.config class TSCParser(object):
    """
    Requirements: LXML.
    This parser reads all client configuration files and returns information like:
    - Test Suite config File
    - Logs Path
    - Reports Path
    - EPs list, active EPs
    - E-mail and Globals config
    """

    def __init__(self, user, base_config='', files_config=''):

        self.user = user
        self.user_home = userHome(user)

        if os.path.isfile(base_config):
            base_config = localFs.read_user_file(user, base_config)
        elif base_config and (type(base_config) == type('') or type(base_config) == type(u'')) \
            and (base_config[0] == '<' and base_config[-1] == '>'):
            pass
        else:
            raise Exception('Parser ERROR: Invalid config data : `{}`!'.format(base_config))

        try:
            self.xmlDict = etree.fromstring(base_config)
        except Exception as e:
            raise Exception('Parser ERROR: Cannot access XML config! `{}`'.format(e))


        self.configTS = None
        self.configHash = None
        self.project_globals = {}
        self.files_config = ''

        self.updateConfigTS(files_config)
        self.updateProjectGlobals()


    def updateConfigTS(self, files_config=''):
        """
        Updates Test Suite Cofig file hash and recreates internal XML structure,
        only if the XML file is changed.
        The file number and suite number have to be unique.
        """

        if files_config and (type(files_config) == type('') or type(files_config) == type(u'')) \
                and (files_config[0] == '<' and files_config[-1] == '>'):

            # This is pure XML data
            config_ts = files_config
            # Hash check the XML file, to see if is changed
            newConfigHash = hashlib.md5(files_config).hexdigest()

        else:

            if not files_config or not os.path.isfile(files_config):
                # Get path to Test-Suites XML from Master config
                files_config = self.files_config

            if files_config.startswith('~/'):
                files_config = userHome(self.user) + files_config[1:]
            if not os.path.isfile(files_config):
                logError('User {}: Parser: Test-Suites XML file `{}` does '\
                'not exist! Please check framework config XML file!'.format(self.user, files_config))
                self.configTS = None
                return -1
            else:
                config_ts = localFs.read_user_file(self.user, files_config)

            # Hash check the XML file, to see if is changed
            newConfigHash = hashlib.md5(config_ts).hexdigest()

        if self.configHash != newConfigHash:
            logDebug('User {}: Parser: Test-Suites XML file changed, '\
            'rebuilding internal structure...\n'.format(self.user))
            # Use the new hash
            self.configHash = newConfigHash
            # Create XML Soup from the new XML file
            try:
                self.configTS = etree.fromstring(config_ts)
            except Exception:
                logError('User {}: Parser ERROR: Cannot access Test-Suites XML data!'.format(self.user))
                self.configTS = None
                return -1

        self.files_config = files_config


    def updateProjectGlobals(self):
        """
        Returns the values of many global tags, from FWM and Test-Suites XML.
        """
        logFull('xmlparser:updateProjectGlobals')
        if self.configTS is None:
            logError('User {}: Parser: Cannot get project globals, because'\
            ' Test-Suites XML is invalid!'.format(self.user))
            return False

        # Reset globals
        self.project_globals = OrderedDict()

        # Parse all known FWMCONFIG tags
        for tag_dict in FWMCONFIG_TAGS:
            # Create default entry
            self.project_globals[tag_dict['name']] = tag_dict['default']
            # Update value from XML
            if self.xmlDict.xpath(tag_dict['tag'] + '/text()'):
                path = self.xmlDict.xpath(tag_dict['tag'])[0].text
                if path[0] == '~':
                    path = self.user_home + path[1:]
                self.project_globals[tag_dict['name']] = path

        # Parse all known PROJECT tags
        for tag_dict in PROJECTCONFIG_TAGS:
            # Create default entry
            self.project_globals[tag_dict['name']] = tag_dict['default']
            # Update value from XML
            if self.configTS.xpath(tag_dict['tag'] + '/text()'):
                # If the variable should be a Boolean
                if tag_dict.get('type') == 'bool':
                    if self.configTS.xpath(tag_dict['tag'] + '/text()')[0].lower() == 'true':
                        value = True
                    else:
                        value = False
                # If the variable should be a Number
                elif tag_dict.get('type') == 'number':
                    value = self.configTS.xpath('round({})'.format(tag_dict['tag']))
                else:
                    value = self.configTS.xpath(tag_dict['tag'])[0].text
                self.project_globals[tag_dict['name']] = value

        return True


    def getActiveEps(self):
        """
        Returns a list with all EPs that appear in Test-Suites XML.
        """
        logFull('xmlparser:getActiveEps')
        if self.configTS is None:
            logError('User {}: Parser ERROR: Cannot get active EPs, because' \
            ' Test-Suites XML is invalid!'.format(self.user))
            return []

        activeEPs = []

        for epname in self.configTS.xpath('//EpId/text()'):
            ep = str(epname).strip()
            # Ignore the empty EP names
            if not ep:
                continue
            # Don't add EP twice
            if ep in activeEPs:
                continue
            activeEPs.append(ep)

        return activeEPs


    def list_settings(self, xmlFile, xFilter=''):
        """
        High level function for listing all settings from a Twister XML config file.
        """
        logFull('xmlparser:list_settings')
        if not os.path.isfile(xmlFile):
            logError('User {}: Parse settings error! File path `{}` does not exist!'.format(self.user, xmlFile))
            return False
        xmlSoup = parseXML(self.user, xmlFile)
        if xmlSoup is None:
            return []
        if xFilter:
            return [x.tag for x in xmlSoup.xpath('//*') if xFilter in x.tag]
        else:
            return [x.tag for x in xmlSoup.xpath('//*')]


    def get_settings_value(self, xmlFile, key):
        """
        High level function for getting a value from a Twister XML config file.
        """
        logFull('xmlparser:get_settings_value')
        if not os.path.isfile(xmlFile):
            logError('User {}: Parse settings error! File path `{}` does not exist!'.format(self.user, xmlFile))
            return False
        if not key:
            return False
        else:
            key = str(key)
        xmlSoup = parseXML(self.user, xmlFile)
        if xmlSoup is None:
            return False
        if xmlSoup.xpath(key):
            txt = xmlSoup.xpath(key)[0].text
            return txt or ''
        else:
            return False


    def set_settings_value(self, xmlFile, key, value):
        """
        High level function for setting a value in a Twister XML config file.
        """
        logFull('xmlparser:set_settings_value')
        if not os.path.isfile(xmlFile):
            logError('User {}: Parse settings error! File path `{}` does not exist!'.format(self.user, xmlFile))
            return False
        if not key:
            return False
        else:
            key = str(key)
        if not value:
            value = ''
        else:
            value = str(value)

        xmlSoup = parseXML(self.user, xmlFile)
        if xmlSoup is None:
            return False
        xml_key = xmlSoup.xpath(key)

        # If the key is found, update it
        if xml_key:
            xml_key[0].text = value

        # Else, create it
        else:
            # Try and split the key into parent and node
            if '/' in key:
                parent_path, node_name = '/'.join(key.split('/')[:-1]), key.split('/')[-1]
            else:
                parent_path, node_name = '/', key
            parent = xmlSoup.xpath(parent_path)
            # Invalid parent path ?
            if not parent:
                return False

            # Create the new node
            node = etree.Element(node_name)
            node.text = value
            node.tail = '\n'
            parent[0].insert(-1, node)

        return dumpXML(self.user, xmlFile, xmlSoup)


    def del_settings_key(self, xmlFile, key, index=0):
        """
        High level function for deleting a value from a Twister XML config file.
        If the `index` is specified and the `key` returns more values, only the
        index-th value is deleted; unless the `index` is -1, in this case, all
        values are deleted.
        """
        logFull('xmlparser:del_settings_key')
        if not os.path.isfile(xmlFile):
            logError('User {}: Parse settings error! File path `{}` does not exist!'.format(self.user, xmlFile))
            return False
        # The key must be string
        if not (isinstance(key, str) or isinstance(key, unicode)):
            return False
        # The index must be integer
        if not isinstance(index, int):
            return False
        # The key must not be Null
        if not key:
            return False
        else:
            key = str(key)

        xmlSoup = parseXML(self.user, xmlFile)
        if xmlSoup is None:
            return False
        xml_key = xmlSoup.xpath(key)

        if xml_key is None:
            return False

        # For index -1, delete all matches
        if index == -1:
            for xml_v in xml_key:
                xml_parent = xml_v.getparent()
                xml_parent.remove(xml_v)
        else:
            # Use the index-th occurence, or, if the index is wrong, exit
            try:
                xml_key = xml_key[index]
            except Exception:
                return False

            xml_parent = xml_key.getparent()
            xml_parent.remove(xml_key)

        return dumpXML(self.user, xmlFile, xmlSoup)


    def set_persistent_suite(self, xmlFile, suite, info={}, order=-1):
        """
        This function writes in TestSuites.XML file.
        """
        logFull('xmlparser:set_persistent_suite')
        if not os.path.isfile(xmlFile):
            logError('User {}: Parse settings error! File path `{}` does not exist!'.format(self.user, xmlFile))
            return False
        if not suite:
            return False
        else:
            suite = str(suite)
        try:
            order = int(order)
        except Exception:
            return False

        # Root element from Project XML
        xmlSoup = parseXML(self.user, xmlFile)
        if xmlSoup is None:
            return False

        suites_index = [xmlSoup.index(s) for s in xmlSoup.xpath('/Root/TestSuite')]

        if order == 0:
            # Add before the first suite
            insert_pos = 2
        elif abs(order) > len(suites_index):
            # If the negative pos is bigger than the index, add before the first suite
            insert_pos = 2
        elif order > len(suites_index):
            # Add after the last suite
            if not suites_index:
                insert_pos = 2
            else:
                insert_pos = suites_index[-1] + 1
        else:
            # If another position, add there
            if order > 0:
                order -= 1
            if not suites_index:
                insert_pos = 2
            else:
                insert_pos = suites_index[order] + 1

        # Suite XML object
        suite_xml = etree.Element('TestSuite')
        tsName = etree.SubElement(suite_xml, 'tsName')
        tsName.text = suite
        epName = etree.SubElement(suite_xml, 'EpId')
        epName.text = info.get('ep', ' ')
        try:
            del info['ep']
        except Exception:
            pass
        sutName = etree.SubElement(suite_xml, 'SutName')
        sutName.text = info.get('sut', ' ')
        try:
            del info['sut']
        except Exception:
            pass

        for k, v in info.iteritems():
            tag = etree.SubElement(suite_xml, 'UserDefined')
            prop = etree.SubElement(tag, 'propName')
            prop.text = str(k)
            val = etree.SubElement(tag, 'propValue')
            val.text = str(v)

        # Insert the new suite and save
        xmlSoup.insert(insert_pos, suite_xml)

        return dumpXML(self.user, xmlFile, xmlSoup)


    def set_persistent_file(self, xmlFile, suite, fname, info={}, order=-1):
        """
        This function writes in TestSuites.XML file.
        """
        logFull('xmlparser:set_persistent_file')
        if not os.path.isfile(xmlFile):
            logError('User {}: Parse settings error! File path `{}` does not exist!'.format(self.user, xmlFile))
            return False
        if not suite:
            return False
        else:
            suite = str(suite)
        if not fname:
            return False
        else:
            fname = str(fname)
        try:
            order = int(order)
        except Exception:
            return False

        # Root element from Project XML
        xmlSoup = parseXML(self.user, xmlFile)
        if xmlSoup is None:
            return False

        suite_xml = xmlSoup.xpath('/Root/TestSuite[tsName="{0}"]'.format(suite))
        if not suite_xml:
            return False
        else:
            suite_xml = suite_xml[0]

        files_index = [suite_xml.index(s) for s in \
            suite_xml.xpath('/Root/TestSuite[tsName="{0}"]/TestCase'.format(suite))]

        if order == 0:
            # Add before the first file
            insert_pos = 2
        elif abs(order) > len(files_index):
            # If the negative pos is bigger than the index, add before the first file
            insert_pos = 2
        elif order > len(files_index):
            # Add after the last file
            if not files_index:
                insert_pos = 2
            else:
                insert_pos = files_index[-1] + 1
        else:
            # If another position, add there
            if order > 0:
                order -= 1
            if not files_index:
                insert_pos = 2
            else: insert_pos = files_index[order] + 1

        # File XML object
        file_xml = etree.Element('TestCase')
        tcName = etree.SubElement(file_xml, 'tcName')
        tcName.text = fname

        for k, v in info.iteritems():
            tag = etree.SubElement(file_xml, 'Property')
            prop = etree.SubElement(tag, 'propName')
            prop.text = str(k)
            val = etree.SubElement(tag, 'propValue')
            val.text = str(v)

        # Insert the new file and save
        suite_xml.insert(insert_pos, file_xml)

        return dumpXML(self.user, xmlFile, xmlSoup)


    def _fixLogType(self, logType):
        """
        Helper function to fix log names.
        """
        logFull('xmlparser:_fixLogType')
        if logType.lower() == 'logrunning':
            logType = 'logRunning'
        elif logType.lower() == 'logdebug':
            logType = 'logDebug'
        elif logType.lower() == 'logsummary':
            logType = 'logSummary'
        elif logType.lower() == 'logtest':
            logType = 'logTest'
        elif logType.lower() == 'logcli':
            logType = 'logCli'
        return logType


    def getLogTypes(self):
        """
        All types of logs exposed from Python to the test cases.
        """
        logFull('xmlparser:getLogTypes')
        return [self._fixLogType(log.tag) for log in self.xmlDict.xpath('LogFiles/*')]


    def getLogFileForType(self, logType):
        """
        Returns the path for one type of log.
        CE will use this path to write the log received from EP.
        """
        logFull('xmlparser:getLogFileForType')
        logs_path = self.project_globals['logs_path']

        if not logs_path:
            logError('User {}: Parser: Logs path is not defined! Please ' \
            'check framework config XML file!'.format(self.user))
            return {}

        logType = self._fixLogType(logType)
        logFile = self.xmlDict.xpath('//{0}/text()'.format(logType))

        if logFile:
            return logs_path + os.sep + logFile[0]
        else:
            return ''


    def getEmailConfig(self, eml_file=''):
        """
        Returns the e-mail configuration.
        After Central Engine stops, an e-mail must be sent to the people interested.
        """
        logFull('xmlparser:getEmailConfig')
        if not eml_file:
            eml_file = self.project_globals['eml_config']

        if not os.path.isfile(eml_file):
            logError('User {}: Parser: E-mail Config file `{}` does not exist!'.format(self.user, eml_file))
            return {}

        econfig = parseXML(self.user, eml_file)
        if econfig is None:
            return {}

        res = {}
        res['Enabled'] = ''
        res['SMTPPath'] = ''
        res['SMTPUser'] = ''
        res['SMTPPwd'] = ''
        res['From'] = ''
        res['To'] = ''
        res['Subject'] = ''
        res['Message'] = ''

        if econfig.xpath('Enabled/text()'):
            res['Enabled'] = econfig.xpath('Enabled')[0].text
        if econfig.xpath('SMTPPath/text()'):
            res['SMTPPath'] = econfig.xpath('SMTPPath')[0].text
        if econfig.xpath('SMTPUser/text()'):
            res['SMTPUser'] = econfig.xpath('SMTPUser')[0].text
        if econfig.xpath('SMTPPwd/text()'):
            res['SMTPPwd'] = econfig.xpath('SMTPPwd')[0].text

        if econfig.xpath('From/text()'):
            res['From'] = econfig.xpath('From')[0].text
        if econfig.xpath('To/text()'):
            res['To'] = econfig.xpath('To')[0].text
        if econfig.xpath('Subject/text()'):
            res['Subject'] = econfig.xpath('Subject')[0].text
        if econfig.xpath('Message/text()'):
            res['Message'] = econfig.xpath('Message')[0].text
        return res


    def get_binding(self, fpath):
        """
        Read a binding between a CFG and a SUT.
        The result is XML.
        """
        logFull('xmlparser:get_binding')
        cfg_file = '{}/twister/config/bindings.xml'.format(userHome(self.user))

        if not os.path.isfile(cfg_file):
            err = '*ERROR* Bindings Config file `{}`, for user `{}` does not exist!'.format(cfg_file, self.user)
            logError(err)
            return err

        bind_xml = parseXML(self.user, cfg_file)
        if bind_xml is None:
            err = '*ERROR* Config file `{}`, for user `{}` cannot be parsed!'.format(cfg_file, self.user)
            return err
        # Find the old binding
        found = bind_xml.xpath('/root/binding/name[text()="{}"]/..'.format(fpath))

        if found:
            xml_string = etree.tostring(found[0])
            return xml_string.replace('binding>', 'root>')
        else:
            logWarning('*ERROR* Cannot find binding name `{}` for user {}!'.format(fpath, self.user))
            return False


    def set_binding(self, fpath, content):
        """
        Write a binding between a CFG and a SUT.
        Return True/ False.
        """
        logFull('xmlparser:set_binding')
        cfg_file = '{}/twister/config/bindings.xml'.format(userHome(self.user))

        if not os.path.isfile(cfg_file):
            err = '*ERROR* Bindings Config file `{}`, for user `{}` does not exist!'.format(cfg_file, self.user)
            logError(err)
            return err

        bind_xml = parseXML(self.user, cfg_file)
        if bind_xml is None:
            err = '*ERROR* Config file `{}`, for user `{}` cannot be parsed!'.format(cfg_file, self.user)
            return err
        # Find the old binding
        found = bind_xml.xpath('/root/binding/name[text()="{}"]/..'.format(fpath))

        # If found, use it
        if found:
            found = found[0]
            found.clear()
        # Or create it
        else:
            found = etree.SubElement(bind_xml, 'binding')

        name = etree.SubElement(found, 'name')
        name.text = fpath

        try:
            replace_xml = etree.fromstring(content, parser)
        except Exception:
            err = '*ERROR* Invalid XML content, user {}! Cannot parse!'.format(self.user)
            logWarning(err)
            return err

        for elem in replace_xml:
            found.append(elem)

        # Beautify XML ?
        return etree.tostring(bind_xml, pretty_print=True)


    def del_binding(self, fpath):
        """
        Delete a binding between a CFG and a SUT.
        Return True/ False.
        """
        logFull('xmlparser:del_binding')
        cfg_file = '{}/twister/config/bindings.xml'.format(userHome(self.user))

        if not os.path.isfile(cfg_file):
            err = '*ERROR* Bindings Config file `{}`, for user `{}` does not exist!'.format(cfg_file, self.user)
            logError(err)
            return err

        bind_xml = parseXML(self.user, cfg_file)
        if bind_xml is None:
            err = '*ERROR* Config file `{}`, for user `{}` cannot be parsed!'.format(cfg_file, self.user)
            return err
        # Find the binding
        found = bind_xml.xpath('/root/binding/name[text()="{}"]/..'.format(fpath))

        # If found, delete it
        if found:
            bind_xml.remove(found[0])
            logDebug('Removed binding `{}`, for user `{}`.'.format(fpath, self.user))
        else:
            err = '*WARN* Invalid binding `{}`, user `{}`! Cannot unbind!'.format(fpath, self.user)
            # logDebug(err)
            return False

        return dumpXML(self.user, cfg_file, bind_xml)


    def getBindingsConfig(self):
        """
        Parse the bindings file that connects Roots from a config file, with SUTs.
        """
        logFull('xmlparser:getBindingsConfig')
        cfg_file = '{}/twister/config/bindings.xml'.format(userHome(self.user))
        bindings = {}

        if not os.path.isfile(cfg_file):
            err = '*ERROR* Bindings Config file `{}`, for user `{}` does not exist!'.format(cfg_file, self.user)
            logError(err)
            return err

        bind_xml = parseXML(self.user, cfg_file)
        if bind_xml is None:
            err = '*ERROR* Config file `{}`, for user `{}` cannot be parsed!'.format(cfg_file, self.user)
            return err

        for binding in bind_xml.xpath('/root/binding'):
            name = binding.find('name')
            # Valid names ?
            if name is None:
                continue
            name = name.text.strip()
            if not name:
                continue
            bindings[name] = {}
            # All binds cfg -> sut
            for bind in binding.findall('bind'):
                cfg = bind.get('config')
                sut = bind.get('sut')
                bindings[name][cfg] = sut

        logDebug('Found `{}` bindings for user `{}`.'.format(len(bindings), self.user))
        return bindings


    def _suites_info(self, xml_object, children, epName):
        """
        Create recursive list of folders and files from Tests path.
        """
        if (xml_object is None) or (not epName):
            return {}

        # For each testsuite from current xml object
        for obj_xpath in xml_object.xpath('TestSuite[EpId="{}"] | TestCase'.format(epName)):
            # If XML object is suite
            if obj_xpath.tag == 'TestSuite':
                d = self.getSuiteInfo(obj_xpath)
                # Save the suite ID in XML, to be used by the files
                idTag = etree.SubElement(obj_xpath, 'id')
                idTag.text = d['id']
                d['children'] = self._suites_info(obj_xpath, OrderedDict(), epName)
            # If XML object is file, it has not children
            else:
                d = self.getFileInfo(obj_xpath)

            oid = d['id']
            del  d['id'] # Delete the ID tag
            children[oid] = d

        return children


    def getAllSuitesInfo(self, epName):
        """
        Shortcut function.
        """
        logFull('xmlparser:getAllSuitesInfo')
        return self._suites_info(self.configTS, SuitesManager(), epName)


    def getSuiteInfo(self, suite_soup):
        """
        Returns a dict with information about 1 Suite from Test-Suites XML.
        The "suite" must be a XML Soup class.
        """
        logFull('xmlparser:getSuiteInfo')
        # A suite can be a part of only 1 EP !
        res = OrderedDict()
        # The first parameter is the Suite name
        if suite_soup.getparent().xpath('id'):
            res['suite'] = suite_soup.getparent().xpath('id')[0].text
        else:
            res['suite'] = ''

        # Add properties from PROJECT
        prop_keys = self.configTS.xpath('/Root/UserDefined/propName')
        prop_vals = self.configTS.xpath('/Root/UserDefined/propValue')
        res.update(dict(zip([k.text for k in prop_keys], [v.text for v in prop_vals])))

        # Add property/ value tags from Suite
        prop_keys = suite_soup.xpath('UserDefined/propName')
        prop_vals = suite_soup.xpath('UserDefined/propValue')
        res.update(dict(zip([k.text for k in prop_keys], [v.text for v in prop_vals])))

        res['type'] = 'suite'
        # Get Suite ID from testsuites.xml
        res['id'] = suite_soup.xpath('ID')[0].text
        # The first parameter is the EP name
        res['ep'] = suite_soup.xpath('EpId')[0].text

        # Parse all known Suites Tags
        for tag_dict in SUITES_TAGS:
            # Create default entry
            res[tag_dict['name']] = tag_dict['default']
            # Update value from XML
            if suite_soup.xpath(tag_dict['tag'] + '/text()'):
                value = suite_soup.xpath(tag_dict['tag'])[0].text
                if not value.strip():
                    continue
                res[tag_dict['name']] = value

        return res


    def getFileInfo(self, file_soup):
        """
        Returns a dict with information about 1 File from Test-Suites XML.
        The "file" must be a XML class.
        """
        logFull('xmlparser:getFileInfo')

        res = OrderedDict()
        res['type'] = 'file'
        # Get File ID from testsuites.xml
        res['id'] = file_soup.xpath('ID')[0].text
        # The second parameter is the Suite name
        res['suite'] = file_soup.getparent().xpath('id')[0].text

        # Parse all known File Tags
        for tag_dict in TESTS_TAGS:
            # Create default entry
            res[tag_dict['name']] = tag_dict['default']
            # Exception for config files
            if tag_dict['name'] == '_cfg_files':
                cfg_files = []
                for cfg_soup in file_soup.xpath(tag_dict['tag']):
                    if cfg_soup.get('enabled').lower() == 'true':
                        cfg = {
                            'name': cfg_soup.get('name'),
                            'iter_default': cfg_soup.get('iterator_default'),
                            'iter_sof': cfg_soup.get('iterator_sof')
                        }
                        cfg_files.append(cfg)
                if cfg_files:
                    res[tag_dict['name']] = cfg_files
            # Update value from XML
            elif file_soup.xpath(tag_dict['tag'] + '/text()'):
                value = file_soup.xpath(tag_dict['tag'])[0].text
                if not value.strip():
                    continue
                res[tag_dict['name']] = value

        # Inject this empty variable
        res['twister_tc_revision'] = '-1'

        # Add property/ value tags
        prop_keys = file_soup.xpath('Property/propName')
        prop_vals = file_soup.xpath('Property/propValue')
        params = ''

        # The order of the properties is important!
        for i in range(len(prop_keys)):
            p_key = prop_keys[i].text
            p_val = prop_vals[i].text

            # Param tags are special
            if p_key == 'param':
                params += p_val + ','
                p_val = params
            res[p_key] = p_val

        return res class CeXmlParser(object):
    """
    Parse and generate project.xml files.
    """

    def __init__(self, project):
        self.project = project


    def _edit_suite(self, ep_id, sut, suite):
        '''
        @param:
            ep_id: the id of the ep
            sut: the name of the sut
            suite: the suite to be edited
        @summary:
            Update suite props with sut name, ep_id
            Delete test cases having running = false.
            Delete the empty suite/subsuite
        '''

        for epname in suite.findall('.//EpId'):
            epname.text = ep_id

        for sut_name in suite.findall('.//SutName'):
            sut_name.text = sut

        return True


    def _remove_empty_suites_tc(self, root):
        """
        Removes all the empty suites and tcs that have running = False
        """
        suites = root.findall('TestSuite')
        for suite in suites:
            # delete tc if running = false
            for prop in suite.findall(".//Property"):
                if prop.find('propName').text == 'Running' and prop.find('propValue').text == 'false':
                    grand_parent = (prop.getparent()).getparent()
                    grand_parent.remove(prop.getparent())

            # delete empty sub-suites from this suite
            subsuites = suite.findall('.//TestSuite')
            if subsuites:
                subsuites.reverse()
                for s in subsuites:
                    if s.find('TestSuite') is None and s.find('TestCase') is None:
                        suite_parent = s.getparent()
                        suite_parent.remove(s)
    
            # delete the suite if it remains empty
            if suite.find('TestSuite') is None and suite.find('TestCase') is None:
                parent = suite.getparent()
                parent.remove(suite)
        return True


    def _resolve_dependencies(self, xml, dependency_dict):
        """
        @param:
            xml: the project/last_edited xml in etree format
            dependency_dict: a dict that stores the link between new and old ids. Old ids are keys.
        @summary:
            Replaces the ids in all Dependency tags with the new and updated ones.
        """
        dependencies = xml.findall('.//Dependency')
        for dependency in dependencies:
            if dependency.text:
                dep_list = dependency.text.split(';')
                new_dep_list = ''
                for dep in dep_list:
                    condition = dep.split(':')[-1]
                    dep_id = dep.split(':')[0]
                    if dep_id in dependency_dict:
                        new_dep_id = [id+':'+condition for id in dependency_dict[dep_id]]
                        new_dep_list += ';'.join(new_dep_id)+';'
                dependency.text = new_dep_list


    def _inherit_suite_dependency(self, xml):
        """
        All the tests under a suite inherits the suite's dependencies.
        """
        logDebug('Inherit dependencies from suites.')

        suites = xml.findall('.//TestSuite')
        for suite in suites:
            suitedependency = suite.find('Dependency')
            if suitedependency.text:
                tcs = suite.findall('.//TestCase')
                for tc in tcs:
                    tcdependency = tc.find('Dependency')
                    if tcdependency.text:
                        suitedeplist = suitedependency.text.split(';')
                        for suitedep in suitedeplist:
                            if not suitedep in tcdependency.text:
                                tcdependency.text += suitedep + ';'
                    else:
                        tcdependency.text = suitedependency.text


    def _change_ids(self, xml, repeated_dict):
        '''
        @param:
            xml: the project/last_edited in etree format
            repeated_dict: the dict that storest the link between old and new IDs. Old ids are keys.
        @summary:
            Change IDs for suite/tc. Add them to repeated_dict dictionary to keep track of changes(to be able 
            to resolve dependencies later)
        '''
        suite_no = 100  # Base ID for suites
        file_no = 1000  # Base ID for tests
 
        # Fix long ugly IDs with short integer IDs
        for suite_id in xml.xpath('//TestSuite/ID'):
            suite_no += 1
            if suite_id.text in repeated_dict:
                repeated_dict[suite_id.text].append(str(suite_no))
            else:
                repeated_dict[suite_id.text] = [str(suite_no)]
            suite_id.text = str(suite_no)
  
        # Fix long ugly IDs with short integer IDs
        for file_id in xml.xpath('//TestCase/ID'):
            file_no += 1
            if file_id.text in repeated_dict:
                repeated_dict[file_id.text].append(str(file_no))
            else:
                repeated_dict[file_id.text] = [str(file_no)]
            file_id.text = str(file_no)


    def _get_iterator_values(self, config_file, iterator_default,\
    interval_values, user, part_interval_values=[], default_values_list=[]):
        """
        Calculates all the possible values for iterators for a test
        """
        data = self.project.configs.read_config_file(user, config_file)
        if data.startswith('*ERROR*'):
            logWarning(data)
            return

        # Try to parse the project file
        try:
            xml_config = etree.fromstring(data)
        except:
            msg = "Config file `{}` is invalid!".format(config_file)
            logWarning(msg)
            return

        config_file_st = etree.tostring(xml_config)
        config_file_fst = etree.fromstring(config_file_st)
        # find all entries having tag = iterator
        config_types = config_file_fst.xpath('//type')
        config_types = [x for x in config_types if x.text == 'iterator']

        # Iterators from config file
        for item in config_types:
            prop_iterator = item.getparent()

            # set the parent component(s) of the iterator
            comp_parent = prop_iterator.getparent()
            comp_parent_name = ''
            if comp_parent.find('fname') is not None:
                comp_parent_name = comp_parent.find('fname').text

            # if this is a sub-component, we need to go up to get all parents
            while True:
                comp_parent = comp_parent.getparent()
                if comp_parent is None:
                    # we reached the root; need to break out from this loop
                    break
                comp_fname = comp_parent.find('fname')
                if comp_fname is not None:
                    comp_parent_name = comp_parent.find('fname').text + '/' +\
                    comp_parent_name

            config_name = config_file + '#' + comp_parent_name +\
            '#' + prop_iterator.find('name').text
            values = prop_iterator.find('value').text
            if not values:
                continue

            values_list = values.replace(' ', '').split(',')

            # get the default value
            index_dot = values_list[0].find('..')
            if index_dot > -1:
                try:
                    default_value = ast.literal_eval(values_list[0][:index_dot])
                    default_value = int(default_value)
                except:
                    default_value = values_list[0][:index_dot]
            else:
                try:
                    default_value = ast.literal_eval(values_list[0])
                    default_value = int(default_value)
                except:
                    default_value = values_list[0]

            if iterator_default == 'true':
                part_interval_values.append(['{}={}'.\
                format(config_name, default_value)])
                default_values_list.append('{}={}'.\
                format(config_name, default_value))
            else:
                iter_interval_values = list()
                key_default_value = '{}={}'.format(config_name, default_value)

                for interv in values_list:
                    re_intervals = re.search('(\w*\d*\.?\d+)\.+(\w*\d*\.?\d+)',\
                    interv)
                    try:
                        x_val = ast.literal_eval(re_intervals.group(1))
                        y_val = ast.literal_eval(re_intervals.group(2))
                        range_res = range(int(x_val), int(y_val) + 1)
                        # avoid adding default value again ex: 2, 1...4
                        if default_value in range_res and \
                        key_default_value in iter_interval_values:
                            del range_res[range_res.index(default_value)]
                        for i in range_res:
                            iter_interval_values.append('{}={}'.\
                            format(config_name, i))
                    except:
                        try:
                            x_val = re_intervals.group(1)
                            y_val = re_intervals.group(2)
                            # try to convert to int if possible
                            try:
                                value = int(ast.literal_eval(x_val))
                                iter_interval_values.append('{}={}'.\
                                format(config_name, value))
                            except:
                                iter_interval_values.append('{}={}'.\
                                format(config_name, x_val))
                            try:
                                value = int(ast.literal_eval(y_val))
                                iter_interval_values.append('{}={}'.\
                                format(config_name, value))
                            except:
                                iter_interval_values.append('{}={}'.\
                                format(config_name, y_val))
                        except:
                            try:
                                interv = ast.literal_eval(interv)
                            except:
                                pass
                            # avoid adding default value again ex: 2, 1, 2, 3
                            if default_value != interv or\
                            key_default_value not in iter_interval_values:
                                iter_interval_values.append('{}={}'.\
                                format(config_name, interv))

                part_interval_values.append(iter_interval_values)

            if part_interval_values:
                if config_name in interval_values.keys():
                    interval_values[config_name].extend(part_interval_values)
                else:
                    interval_values[config_name] = part_interval_values


    def _get_cartesian_list(self, user, configs):
        """
        @param:
            user: the authenticated twister user
            configs: the configuration files in xml format
        @return:
            cartesian list of the iterators as list of pairs
        @summary:
            Computes the cartesian list of the iterators 
        """
        interval_values = OrderedDict()
        part_interval_values = []
        default_values_list = []
        for config in configs:
            config_file = config.get('name')
            enabled = config.get('enabled')
            if enabled == 'false':
                continue
            default = config.get('iterator_default')
            self._get_iterator_values(config_file, default, interval_values,\
            user, part_interval_values, default_values_list)
        cartesian_list = []
        if len(interval_values.values()) > 0:
            cartesian_list = cartesian(interval_values.values()[0])

        return cartesian_list


    def _add_property(self, parent, value, inherited_value=''):
        """
        @param:
            parent: element tree node, a TestSuite/TestCase node
            value: the value to be set
            inherited_value: the value inherited from the parent suite. 
                It is used to propagate iterator values on subsuites
        @return:
            cartesian list of the iterators as list of pairs
        @summary:
            Adds a property node to an existing TestSuite/TestCase element tree node
        """
        if value:
            new_prop = etree.SubElement(parent, 'Property')
            new_prop_name = etree.SubElement(new_prop, 'propName')
            new_prop_name.text = 'iterationNr'
            new_prop_value = etree.SubElement(new_prop, 'propValue')
            if inherited_value:
                new_prop_value.text = inherited_value + ', ' + value
            else:
                new_prop_value.text = value


    def _expand_global_configs(self, user, xml, config_list=[]):
        """
        @param:
            user: the twister authenticated user
            xml: the project/last_edited xml in etree format. When going into recursion, xml becomes a suite.
        @summary:
            Suites now support configs, seen as global configs for all test in a suite.
            It duplicates the suite as many times as the iterator says.
            The value from the top suite propagates to the subsuites.
            A property tag is added to store the iteration value.
            Between tc iterators and suite iterators, the last ones have priority.
        """
        for suite in xml.findall('TestSuite'):
            remove = True
            ts_name = suite.find('tsName').text
            cfg_prop = suite.find('ConfigFiles')
            config_info = cfg_prop.findall('Config')
            cartesian_list = self._get_cartesian_list(user, config_info)
            logDebug("CeParser: Will iterate test suite `{}`, {} times, from values: {}, user `{}`."\
                     .format(ts_name, len(cartesian_list), cartesian_list, user))

            inherited_val = ''
            parent_property = xml.findall('Property')
            if len(parent_property) > 0:
                parent_property = parent_property[0]
                inherited_val = parent_property.find('propValue').text
            index = xml.index(suite)
            # duplicate suites
            for item in reversed(cartesian_list):
                deep_copy = copy.deepcopy(suite)
                suite_st = etree.tostring(deep_copy)
                newSuite = etree.fromstring(suite_st)
                xml.insert(index+1, newSuite)
                if isinstance(item, tuple):
                    value = ', '.join(str(x) for x in item)
                else:
                    value = str(item)
                self._add_property(newSuite, value, inherited_val)
                self._expand_global_configs(user, newSuite, config_list+copy.deepcopy(config_info))
                self._expand_tc_configs(user, newSuite, config_list+copy.deepcopy(config_info))
            if len(cartesian_list) == 0:
                remove = False
                if inherited_val:
                    self._add_property(suite, inherited_val)
                self._expand_global_configs(user, suite, config_list+copy.deepcopy(config_info))
                self._expand_tc_configs(user, suite, config_list+copy.deepcopy(config_info))
            # if the suite has iterators, remove the one that is being duplicated
            if remove:
                xml.remove(suite)


    def _expand_tc_configs(self, user, suite, config_list):
        """
        @param: 
            user: the authenticated twister user
            suite: the suite that contains the tc to duplicate
        @summary:
            Explodes the iterators and duplicates the test cases accordingly
            It is called after a suite is expanded, in _expand_global_configs
        """
        for tc in suite.findall('TestCase'):
            remove = True
            tc_name = tc.find('tcName').text
            cfg_prop = tc.find('ConfigFiles')
            config_info = cfg_prop.findall('Config')
            cartesian_list = self._get_cartesian_list(user, config_info)
            cfg_names = []
            for elem in config_list:
                new_cfg_name = elem.get('name')
                cfg_names = [cfg.get('name') for cfg in cfg_prop.findall('Config')]
                if new_cfg_name not in cfg_names:
                    cfg_prop.insert(-1,copy.deepcopy(elem))

            logDebug("CeParser: Will iterate test case `{}`, {} times, from values: {}, user `{}`."\
                     .format(tc_name, len(cartesian_list), cartesian_list, user))

            inherited_val = ''
            parent_property = suite.findall('Property')
            if len(parent_property) > 0:
                parent_property = parent_property[0]
                inherited_val = parent_property.find('propValue').text
            index = suite.index(tc)
            # duplicate suites
            for item in reversed(cartesian_list):
                deep_copy = copy.deepcopy(tc)
                tc_st = etree.tostring(deep_copy)
                new_tc = etree.fromstring(tc_st)
                suite.insert(index+1, new_tc)
                if isinstance(item, tuple):
                    value = ', '.join(str(x) for x in item)
                else:
                    value = str(item)
                self._add_property(new_tc, value, inherited_val)
            if len(cartesian_list) == 0:
                remove = False
                if inherited_val:
                    self._add_property(tc, inherited_val)
            # if the tc has iterators, remove the one that is being duplicated
            if remove:
                suite.remove(tc)

    def _expand_repeat_tag_on_tc(self, suite):
        """
        Expand the test cases by the repeat tag.
        """
        for tc in suite.findall('TestCase'):
            tcName = tc.find('tcName').text
            repeat = None
            try:
                repeat = tc.find('Repeat')
                nb_repeat = int(repeat.text)
                logDebug("CeParser: Will repeat suite `{}`, {} times.".format(tcName, nb_repeat))
            except:
                nb_repeat = 1

            # before copying the suite for multiplication, remove the Repeat
            # tag because it MUST NOT be present in generated xml file
            if repeat is not None:
                tc.remove(repeat)

            index = suite.index(tc)
            for i in range(nb_repeat-1):
                deep_copy = copy.deepcopy(tc)
                tc_st = etree.tostring(deep_copy)
                tc_copy = etree.fromstring(tc_st)
                suite.insert(index+1, tc_copy)


    def _expand_repeat_tag(self, xml):
        """
        @param:
            user: the twister authenticated user
            xml: the project/last_edited xml in etree format. When going into recursion, xml becomes a suite.
        @summary:
            Suites can have a repeat tag. It duplicates the suite as many times as the repeat tag says.
        """
        for suite in xml.findall('TestSuite'):
            suite_name = suite.find('tsName').text
            repeat = None
            try:
                repeat = suite.find('Repeat')
                nb_repeat = int(repeat.text)
                logDebug("CeParser: Will repeat suite `{}`, {} times.".format(suite_name, nb_repeat))
            except:
                nb_repeat = 1

            # before copying the suite for multiplication, remove the Repeat
            # tag because it MUST NOT be present in generated xml file
            if repeat is not None:
                suite.remove(repeat)

            index = xml.index(suite)
            for i in range(nb_repeat-1):
                deep_copy = copy.deepcopy(suite)
                suite_st = etree.tostring(deep_copy)
                suite_copy = etree.fromstring(suite_st)
                xml.insert(index+1, suite_copy)
                self._expand_repeat_tag_on_tc(suite_copy)
                self._expand_repeat_tag(suite_copy)
            self._expand_repeat_tag_on_tc(suite)
            self._expand_repeat_tag(suite)


    def _expand_by_ep(self, user, xml):
        """
        @param:
            user: the twister authenticated user
            xml: the project/last_edited xml in etree format. When going into recursion, xml becomes a suite.
        @summary:
            The sut tag can contain multiple suts and a sut can contain multiple eps. Main suites must be
            duplicated to run on each ep from each sut.
        """
        suites = xml.findall('TestSuite')
        for suite in suites:
            # get all suts chosen by user
            all_suts = suite.find('SutName').text
            suite_name = suite.find('tsName').text
            if not all_suts:
                err = 'User `{}`: Invalid SUT for suite `{}`! Cannot generate project!'.format(user, suite_name)
                logWarning(err)
                return '*ERROR* ' + err
            suts_list = [q.replace('(', '.').replace(')', '') for q in all_suts.split(';') if q]
            # added get_sut to load the suts in resources dict. get_sut_info\
            # takes resources from the resources dict
            index = xml.index(suite)
            for sut in suts_list:
                self.project.sut.get_sut(sut, props={'__user': user})
                sut = '/' + sut
                sut_eps = self.project.sut.get_info_sut(sut + ':_epnames_' + user, {'__user': user})
                
                if sut_eps and '*ERROR*' in sut_eps:
                    logError(sut_eps)
                    return sut_eps

                if sut_eps and (isinstance(sut_eps, str) or isinstance(sut_eps, unicode)):
                    sut_eps_list = [ep for ep in sut_eps.split(';') if ep]
                    for ep_id in sut_eps_list:
                        deep_copy = copy.deepcopy(suite)
                        suite_st = etree.tostring(deep_copy)
                        suite_copy = etree.fromstring(suite_st)
                        if suite_copy.find('TestCase') is not None or \
                            suite_copy.find('TestSuite') is not None:
                            index += 1
                            xml.insert(index, suite_copy)
                            self._edit_suite(ep_id, sut, suite_copy)
                else:
                    # Find Anonimous EP in the active EPs
                    anonim_ep = self.project._find_anonim_ep(user)
                    if isinstance(anonim_ep, bool):
                        return anonim_ep
                    deep_copy = copy.deepcopy(suite)
                    suite_st = etree.tostring(deep_copy)
                    suite_copy = etree.fromstring(suite_st)
                    if suite_copy.find('TestCase') is not None or \
                        suite_copy.find('TestSuite') is not None:
                        index += 1
                        xml.insert(index, suite_copy)
                        self._edit_suite(anonim_ep, sut, suite_copy)
            xml.remove(suite)


    def generate_xml(self, user, filename):
        '''
        Receives project file.
        Creates testsuites.xml file by multiplying tests depending
        on the suts number and eps.
        '''
        logDebug("CeParser: preparing to convert project file: `{}`,\
        user `{}`.".format(filename, user))

        data = self.project.read_project_file(user, filename)
        if data.startswith('*ERROR*'):
            logWarning(data)
            return data

        # try to parse the project file
        try:
            xml = etree.fromstring(data)
        except Exception as e:
            msg = "The file: '{}' it's not an xml file. Try again!\n{}".\
            format(filename, e)
            logDebug(msg)
            return '*ERROR* ' + msg

        self._expand_global_configs(user, xml)
        
        self._expand_repeat_tag(xml)
        
        self._expand_by_ep(user, xml)

        repeated_dict = {}
        self._change_ids(xml, repeated_dict)

        self._resolve_dependencies(xml, repeated_dict)

        self._inherit_suite_dependency(xml)

        self._remove_empty_suites_tc(xml)

        for suite in xml.findall('.//TestSuite'):
            prop = suite.find('Property')
            if prop:
                suite.remove(prop)
        # Final XML string
        xml_string = etree.tostring(xml, pretty_print=True)

        # write the xml file
        xml_file = userHome(user) + '/twister/config/testsuites.xml'

        resp = self.project.localFs.write_user_file(user, xml_file, xml_string, 'w')
        if resp != True:
            logError(resp)
            return '*ERROR* ' + resp

        logDebug('CeParser: Successfully generated: `{}`, user `{}`.'.\
        format(xml_file, user))
        return True


    def get_number_of_iterations(self, user, configs):
        """
        Computes the number of iterations each time a user manipulates the
        configs for a test.
        """
        interval_values = OrderedDict()
        part_interval_values = []
        default_values_list = []
        for config in configs.split(';')[:-1]:
            cfg = config.split(',')
            config_file = cfg[0].strip()
            default = cfg[1].strip()
            self._get_iterator_values(config_file, default, interval_values,\
            user, part_interval_values, default_values_list)
        cartesian_list = list()
        if len(interval_values.values()) > 0:
            cartesian_list = cartesian(interval_values.values()[0])

        return len(cartesian_list) class ServiceManager(object):
    """ manager for user service """

    def __init__(self):

        logDebug('SM: Starting Service Manager...')

        self.twister_services = []
        cfg_path = '{0}/config/services.ini'.format(TWISTER_PATH)
        cfg = iniparser.ConfigObj(cfg_path)

        for service in cfg:
            if service == 'DEFAULT':
                continue
            cfg[service]['name'] = service
            self.twister_services.append(cfg[service])

        logDebug('SM: Found `{0}` services: `{1}`.'.format(len(self.twister_services), ', '.join(cfg.keys())))
        del cfg, cfg_path


    def __del__(self):

        logDebug('SM: Stopping Service Manager...')

        for service in self.twister_services:
            if self.service_status(service) == -1:
                self.service_stop(service)

        del self.twister_services


    def send_command(self, command, name='', *args, **kwargs):
        """ send command to service """
        logFull('CeServices:send_command')

        if command == SM_LIST or command == SM_COMMAND_MAP[SM_LIST]:
            return self.list_services()

        found = False

        service = None
        for service_item in self.twister_services:
            if name == service_item['name']:
                found = True
                service = service_item
                break

        if not found:
            logDebug('SM: Invalid service name: `{}`!'.format(name))
            return False

        elif command == SM_STATUS or command == SM_COMMAND_MAP[SM_STATUS]:
            return self.service_status(service)

        elif command == SM_DESCRIP or command == SM_COMMAND_MAP[SM_DESCRIP]:
            return service.get('descrip')

        if command == SM_START or command == SM_COMMAND_MAP[SM_START]:
            return self.service_start(service)

        elif command == SM_STOP or command == SM_COMMAND_MAP[SM_STOP]:
            return self.service_stop(service)

        elif command == SM_GET_CONFIG or command == SM_COMMAND_MAP[SM_GET_CONFIG]:
            return self.read_config(service)

        elif command == SM_SET_CONFIG or command == SM_COMMAND_MAP[SM_SET_CONFIG]:
            try:
                return self.save_config(service, args[0][0])
            except:
                return 'SM: Invalid number of parameters for save config!'

        elif command == SM_GET_LOG or command == SM_COMMAND_MAP[SM_GET_LOG]:
            try:
                return self.get_console_log(service, read=args[0][0], fstart=args[0][1])
            except:
                return 'SM: Invalid number of parameters for read log!'

        else:
            return 'SM: Unknown command number: `{0}`!'.format(command)


    def list_services(self):
        """ return list of services """
        logFull('CeServices:list_services')
        srv = []
        for service in self.twister_services:
            srv.append(service['name'])
        return ','.join(srv)


    def service_status(self, service):
        """ return status of service """
        logFull('CeServices:service_status')
        # Values are: -1, 0, or any error code
        # -1 means the app is still running

        tprocess = service.get('pid', 0)
        retc = 0

        if tprocess:
            tprocess.poll()
            retc = tprocess.returncode

        if retc is None:
            retc = -1

        return retc


    def service_start(self, service):
        """ start service """
        logFull('CeServices:service_start')

        tprocess = service.get('pid', 0)

        if tprocess:
            # Check if child process has terminated
            tprocess.poll()

            if tprocess.returncode is None:
                logDebug('SM: Service name `{}` is already running with PID `{}`.'.format(
                    service['name'], tprocess.pid))
                return True

        del tprocess

        script_path = '{}/services/{}/{}'.format(TWISTER_PATH, service['name'], service['script'])

        if service['config']:
            config_path = '{}/services/{}/{}'.format(TWISTER_PATH, service['name'], service['config'])
        else:
            config_path = ''

        if not os.path.isfile(script_path):
            error = 'SM: Cannot start service `{}`! No such script file `{}`!'.format(
                service['name'], script_path)
            logError(error)
            return error

        if service['config'] and (not config_path):
            error = 'SM: Cannot start service `{}`! No such config file `{}`!'.format(
                service['name'], config_path)
            logError(error)
            return error

        service['pid'] = 0 # Delete process here
        env = os.environ
        env.update({'TWISTER_PATH': TWISTER_PATH})

        log_path = '{}/services/{}/{}'.format(TWISTER_PATH, service['name'], service['logfile'])
        logs_dir = os.path.split(log_path)[0]

        if not os.path.isdir(logs_dir):
            try:
                os.mkdir(logs_dir)
            except:
                logError('SM: Cannot create logs folder `{}`!'.format(logs_dir))

        p_cmd = [sys.executable, '-u', script_path, config_path]

        with open(log_path, 'wb') as out:
            try:
                tprocess = subprocess.Popen(p_cmd, stdout=out, stderr=out, env=env)
            except Exception as exp_err:
                error = 'SM: Cannot start service `{}` with config file `{}`!\n'\
                    'Exception: `{}`!'.format(service['name'], config_path, exp_err)
                logError(error)
                return error

        service['pid'] = tprocess
        logDebug('Started service `{}`, using script `{}` and config `{}`, with PID `{}`.'.format(
            service['name'], script_path, config_path, tprocess.pid))
        return True


    def service_stop(self, service):
        """ stop service """
        logFull('CeServices:service_stop')

        rc = self.service_status(service)
        if not rc:
            logDebug('SM: Service name `{}` is not running.'.format(service['name']))
            return False

        tprocess = service.get('pid', 0)

        if isinstance(tprocess, int):
            logError('SM: Cannot stop service `{}`!'.format(service['name']))

        try:
            tprocess.terminate()
        except Exception as exp_err:
            logError('SM: Cannot stop service: `{}`, exception `{}`!'.format(service['name'], exp_err))
            return False

        try:
            time.sleep(0.1)
            os.killpg(tprocess.pid, signal.SIGTERM)
            time.sleep(0.1)
            tprocess.kill()
        except:
            pass

        logWarning('SM: Stopped service: `{}`.'.format(service['name']))
        return True


    def service_kill(self, service):
        """ forced stop user service """
        logFull('CeServices:service_kill')

        return self.service_stop(service)


    def read_config(self, service):
        """ Read configuration """
        logFull('CeServices:read_config')

        config_path = '{0}/services/{1}/{2}'.format(TWISTER_PATH, service['name'], service['config'])

        if not os.path.isfile(config_path):
            logError('SM: No such config file `{0}`!'.format(config_path))
            return False

        with open(config_path, 'rb') as out:
            data = out.read()

        return data or ''


    def save_config(self, service, data):
        """ Save configuration """
        logFull('CeServices:save_config')

        config_path = '{0}/services/{1}/{2}'.format(TWISTER_PATH, service['name'], service['config'])

        if not os.path.isfile(config_path):
            logError('SM: No such config file `{0}`!'.format(config_path))
            return False

        with open(config_path, 'wb') as out:
            out.write(data)

        return True


    def get_console_log(self, service, read, fstart):
        """
        Called in the Java GUI to show the logs.
        """
        logFull('CeServices:get_console_log')
        if fstart is None:
            return '*ERROR for {0}!* Parameter FSTART is NULL!'.format(service['name'])

        filename = '{0}/services/{1}/{2}'.format(TWISTER_PATH, service['name'], service['logfile'])

        if not os.path.exists(filename):
            return '*ERROR for {}!* No such log file `{}`!'.format(service['name'], filename)

        if not read or read == '0':
            return os.path.getsize(filename)

        fstart = long(fstart)
        f = open(filename)
        f.seek(fstart)
        data = f.read()
        f.close()

        return binascii.b2a_base64(data) class WebInterface(object):
    """ class for web interface """

    def __init__(self, project):

        self.project = project


    def user_agent(self):
        """
        User agent returns Browser or XML RPC client.
        This function is not exposed.
        """
        user_agent = cherrypy.request.headers['User-Agent'].lower()
        if 'xmlrpc' in user_agent or 'xml rpc' in user_agent:
            # XML RPC client
            return 'x'
        else:
            # Browser
            return 'b'


    @cherrypy.expose
    def index(self):
        """ Initial page """
        logFull('CeWebUi:index')
        if self.user_agent() == 'x':
            return 0

        try:
            srv_ver = open(TWISTER_PATH + '/server/version.txt').read().strip()
        except:
            srv_ver = '-'
        srv_type = self.project.server_init['ce_server_type']
        ip_port = cherrypy.request.headers['Host']
        machine = platform.uname()[1]
        system = ' '.join(platform.linux_distribution())
        users = self.project.list_users()

        output = Template(filename=TWISTER_PATH + '/server/template/rest_main.htm')
        return output.render(srv_type=srv_type, srv_ver=srv_ver,\
        ip_port=ip_port, machine=machine, system=system, users=users)


    @cherrypy.expose
    def users(self, user=''):
        """ list all users """
        logFull('CeWebUi:users')
        if self.user_agent() == 'x':
            return 0
        if not user:
            raise cherrypy.HTTPRedirect('/web/#tab_users')

        host = cherrypy.request.headers['Host']
        rev_dict = dict((v, k) for k, v in EXEC_STATUS.iteritems())
        int_status = self.project.get_user_info(user, 'status') or STATUS_INVALID
        status = rev_dict[int_status]
        try:
            eps_file = self.project.parsers[user].project_globals['ep_names']
        except:
            eps_file = ''

        eps = self.project.get_user_info(user, 'eps')
        ep_statuses = [rev_dict[eps[ep].get('status', STATUS_INVALID)] \
        for ep in eps]
        logs = self.project.get_user_info(user, 'log_types')

        output = Template(filename=TWISTER_PATH + '/server/template/rest_user.htm')
        return output.render(host=host, user=user, status=status,\
        exec_status=rev_dict, eps_file=eps_file, eps=eps,\
        ep_statuses=ep_statuses, logs=logs)

# # #

    @cherrypy.expose
    def json_stats(self):
        """ get stats """
        logFull('CeWebUi:json_stats')
        if self.user_agent() == 'x':
            return 0

        cherrypy.response.headers['Content-Type'] = 'application/json; charset=utf-8'
        cherrypy.response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
        cherrypy.response.headers['Pragma'] = 'no-cache'
        cherrypy.response.headers['Expires'] = 0
        data = {'mem': calcMemory(), 'cpu': calcCpu()}
        return json.dumps(data)


    @cherrypy.expose
    def json_get_project(self):
        """ get the project """
        logFull('CeWebUi:json_get_project')
        if self.user_agent() == 'x':
            return 0

        cherrypy.response.headers['Content-Type'] = 'application/json; charset=utf-8'
        cherrypy.response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
        cherrypy.response.headers['Pragma'] = 'no-cache'
        cherrypy.response.headers['Expires'] = 0
        return open(TWISTER_PATH + '/config/project_users.json', 'r').read()


    @cherrypy.expose
    def json_save_project(self, user, epname):
        """ save project """
        logFull('CeWebUi:json_save_project user `{}`.'.format(user))
        if self.user_agent() == 'x':
            return 0

        c_l = cherrypy.request.headers['Content-Length']
        raw_data = cherrypy.request.body.read(int(c_l))
        json_data = json.loads(raw_data)
        del c_l, raw_data

        # Delete everything from XML Root
        self.project.del_settings_key(user, 'project', '//TestSuite')
        changes = 'Reset project file...\n'

        for suite_data in json_data:
            self.project.set_persistent_suite(user, suite_data['data'], {'ep': decode(epname)})
            changes += 'Created suite: {0}.\n'.format(suite_data['data'])
            for file_data in suite_data.get('children', []):
                changes += 'Created file: {0}.\n'.format(file_data['data'])
                self.project.set_persistent_file(user, suite_data['data'], file_data['data'], {})

        changes += '>.<\n'
        logDebug(changes)
        return 'true'


    @cherrypy.expose
    def json_eps(self, user, epname):
        """ list of EPs """
        logFull('CeWebUi:json_eps user `{}`.'.format(user))
        if self.user_agent() == 'x':
            return 0

        epname = decode(epname)
        cherrypy.response.headers['Content-Type'] = 'application/json; charset=utf-8'
        cherrypy.response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
        cherrypy.response.headers['Pragma'] = 'no-cache'
        cherrypy.response.headers['Expires'] = 0

        suites_mngr = self.project.get_ep_info(user, epname).get('suites', False)

        data = []
        default_suite = {
            'data': '',
            'metadata': '',
            'children': [],
            }

        for suite_id in suites_mngr.get_suites():
            node = suites_mngr.find_id(suite_id)

            current_suite = dict(default_suite)
            current_suite['data'] = node['name']
            current_suite['metadata'] = suite_id
            current_suite['attr'] = dict({'id': suite_id, 'rel': 'suite'}, **node)
            del current_suite['attr']['children']
            current_suite['children'] = [
                {'data': v['file'], 'attr': dict({'id': k}, **v)}
                for k, v in node['children'].iteritems()
                if v.get('type', 'file') == 'file'
                ]
            data.append(current_suite)

        return json.dumps(data, indent=2)


    @cherrypy.expose
    def json_folders(self, user):
        """ return list of folders in json format """
        logFull('CeWebUi:json_folders user `{}`.'.format(user))
        if self.user_agent() == 'x':
            return 0

        paths = {'data':'Tests Path', 'attr': {'rel': 'folder'}, 'children':[]}
        dirpath = self.project.get_user_info(user, 'tests_path')
        dirList(dirpath, dirpath, paths)

        cherrypy.response.headers['Content-Type'] = 'application/json; charset=utf-8'
        return json.dumps([paths], indent=2)


    @cherrypy.expose
    def json_logs(self, user='', log=''):
        """ get logs in json format """
        logFull('CeWebUi:json_logs')
        if self.user_agent() == 'x':
            return 0

        if user and log:
            logs = self.project.get_user_info(user, 'log_types')
            logs_path = self.project.get_user_info(user, 'logs_path')

            if log.startswith('logCli_'):
                epname = '_'.join(log.split('_')[1:])
                log_cli = os.path.split(logs.get('logCli', 'CLI.log'))[1]
                log = logs_path + os.sep + decode(epname) +'_'+ log_cli
            else:
                log = logs.get(log)

        cherrypy.response.headers['Content-Type'] = 'application/json; charset=utf-8'
        cherrypy.response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
        cherrypy.response.headers['Pragma'] = 'no-cache'
        cherrypy.response.headers['Expires'] = 0
        return json.dumps(prepare_log(log or LOG_FILE))

# # #

    @cherrypy.expose
    def reset_user(self, user):
        """ clean user related information """
        logFull('CeWebUi:reset_user user `{}`.'.format(user))
        self.project.reset_project(user)
        self.project.reset_logs(user)
        raise cherrypy.HTTPRedirect('http://{host}/web/users/{user}'.\
        format(host=cherrypy.request.headers['Host'], user=user))


    @cherrypy.expose
    def set_user_status(self, user, status):
        """ Update user status """
        logFull('CeWebUi:set_user_status user `{}`.'.format(user))
        output = Template(filename=TWISTER_PATH + '/server/template/rest_error.htm')
        try:
            status = int(status)
        except:
            return output.render(title='Error!', body='<b>Status value `{0}` is invalid!</b>'.format(status))
        if status not in EXEC_STATUS.values():
            return output.render(title='Error!', body='<b>Status value `{0}`' \
                ' is not in the list of valid statuses: {1}!</b>'\
                .format(status, EXEC_STATUS.values()))
        self.project.set_exec_status_all(user, status, 'User status changed from REST interface.')
        raise cherrypy.HTTPRedirect('http://{host}/web/users/{user}#tab_home'.\
        format(host=cherrypy.request.headers['Host'], user=user))


    @cherrypy.expose
    def set_ep_status(self, user, epname, status):
        """ Update EP status """
        logFull('CeWebUi:set_ep_status user `{}`.'.format(user))
        output = Template(filename=TWISTER_PATH + '/server/template/rest_error.htm')
        try:
            status = int(status)
        except:
            return output.render(title='Error!', body='<b>Status value `{0}` is invalid!</b>'.format(status))
        if status not in EXEC_STATUS.values():
            return output.render(title='Error!', body='<b>Status value `{0}`' \
                   ' is not in the list of valid statuses: {1}!</b>'\
                .format(status, EXEC_STATUS.values()))
        self.project.set_exec_status(user, epname, status, 'EP status changed from REST interface.')
        raise cherrypy.HTTPRedirect('http://{host}/web/users/{user}#tab_proc'.\
        format(host=cherrypy.request.headers['Host'],\
        user=user, epname=epname)) class ReportingServer(object):
    """
    Reporting server class.
    """

    db_parser = {}
    db_servers = {}
    glob_fields = {}
    glob_reports = {}
    glob_redirects = {}
    glob_links = {}
    timers = {}


    def __init__(self, project):
        """
        Initialization function.
        """
        self.project = project


    def load_config(self, usr, force=False):
        """
        Read DB Config File for 1 user.
        """
        if not os.path.isdir(userHome(usr) + '/twister/config'):
            logError('Report Server: Cannot find Twister for user `{}` !'.format(usr))
            return False

        # Get the path to DB.XML
        db_file = self.project.get_user_info(usr, 'db_config')
        if not db_file:
            logError('Report Server: Null DB.XML file for user `{}`! Nothing to do!'.format(usr))
            return False

        # Current timer
        c_time = time.time()

        # Create database parser IF necessary, or FORCED, or old connection...
        if force or (usr not in self.db_parser) or (c_time - self.timers.get(usr, 0) > 5.0):

           # logDebug('Rebuilding fields, reports and redirects for user `{}`...'.format(usr))

            self.timers[usr] = c_time
            self.db_parser[usr] = True
            self.glob_fields[usr] = OrderedDict()
            self.glob_reports[usr] = OrderedDict()
            self.glob_redirects[usr] = OrderedDict()
            self.glob_links[usr] = [{'name': 'Home', 'link': 'Home', 'type': 'link'}]

            # DB.xml + Shared DB parser
            users_groups = self.project._parse_users_and_groups()
            shared_db_path = users_groups['shared_db_cfg']
            db_cfg_role = 'CHANGE_DB_CFG' in users_groups['users'][usr]['roles']

            # Use shared DB or not ?
            use_shared_db = self.project.get_user_info(usr, 'use_shared_db')
            if use_shared_db and use_shared_db.lower() in ['true', 'yes']:
                use_shared_db = True
            else:
                use_shared_db = False

            dbp = DBParser(usr, db_file, shared_db_path, use_shared_db)
            self.db_servers = dbp.db_config['servers']
            report_queries = dbp.get_reports(db_cfg_role)
            del dbp

            for host_db in report_queries:

                self.glob_fields[usr].update(report_queries[host_db]['fields'])
                self.glob_redirects[usr].update(report_queries[host_db]['redirects'])

                for report_name, report_data in report_queries[host_db]['reports'].iteritems():
                    srv_name = report_data['srv_name']
                    # Add the new server name in reports ?
                    if srv_name not in self.glob_reports[usr]:
                        self.glob_reports[usr][srv_name] = OrderedDict()
                    # Add the report inside the server
                    self.glob_reports[usr][srv_name][report_name] = report_data

                # There are more types of reports:
                # Normal links, like Home, Help and other normal reports
                # Redirect links, that don't contain reports
                # Folders, that don't go anywhere, are just labels for reports
                for rname, rval in report_queries[host_db]['reports'].iteritems():
                    # Shared report ?
                    srv_name = rval['srv_name']
                    # Link name used in reports
                    link = ('S&' if srv_name=='Shared' else 'U&') + rname
                    # Server, user, password
                    db_server, db_name, db_user, _, _ = host_db
                    srv_db = '{} server: {}/ {}/ {}'.format(srv_name, db_server, db_name, db_user)
                    report = {
                        'name': rname,
                        'link': link,
                        'type': 'link',
                        'folder': rval.get('folder', ''),
                        'srvr': srv_db
                    }
                    self.glob_links[usr].append(report)

                for rname, rval in report_queries[host_db]['redirects'].iteritems():
                    link = ('S&' if rval['srv_name']=='Shared' else 'U&') + rname
                    self.glob_links[usr].append({
                        'name': rname,
                        'link': link,
                        'type': 'redir',
                        'folder': ''
                    })

            # Append the Help link at the end
            self.glob_links[usr].append({'name': 'Help', 'link': 'Help', 'type': 'link'})

        return True


    @cherrypy.expose
    def index(self, usr=''):
        """
        The index page.
        """
        if not usr:
            users = self.project.list_users()
            output = Template(filename=TWISTER_PATH + '/server/template/rep_base.htm')
            return output.render(title='Users', usr='#' + '#'.join(users), links=[])

        if not os.path.isdir(userHome(usr) + '/twister/config'):
            return '<br><b>Error! Username `{}` doesn\'t have a Twister config folder!</b>'.format(usr)

        # FORCE re-load all Database XML on INDEX/ HOME links !
        self.load_config(usr, True)
        output = Template(filename=TWISTER_PATH + '/server/template/rep_base.htm')
        return output.render(title='Home', usr=usr, links=self.glob_links[usr])

    # Report link 2
    @cherrypy.expose
    def home(self, usr=''):
        """ Get reporting link 2 """
        return self.index(usr=usr)

    # Report link 3
    @cherrypy.expose
    def report(self, usr=''):
        """ Get reporting link 3 """
        return self.index(usr=usr)

    # Report link 4
    @cherrypy.expose
    def reporting(self, usr=''):
        """ Get reporting link 4 """
        return self.index(usr=usr)


    @cherrypy.expose
    def help(self, usr=''):
        """
        Help page.
        """
        if not usr:
            return '<br><b>Error! This link should be accessed by passing a username, eg: /help/some_user<b/>'

        if not os.path.isdir(userHome(usr) + '/twister/config'):
            return '<br><b>Error! Username `{}` doesn\'t have a Twister config folder!</b>'.format(usr)

        self.load_config(usr) # Re-load all Database XML
        output = Template(filename=TWISTER_PATH + '/server/template/rep_help.htm')
        return output.render(title='Help', usr=usr, links=self.glob_links[usr])


    @cherrypy.expose
    def rep(self, report=None, usr=None, **kw):
        """
        Reporting link.
        """
        if not usr:
            return '<br><b>Error! This link should be accessed by passing a username, eg: /rep/some_user<b/>'

        if not os.path.isdir(userHome(usr) + '/twister/config'):
            return '<br><b>Error! Username `{}` doesn\'t have a Twister config folder!</b>'.format(usr)

        self.load_config(usr) # Re-load all Database XML

        cherrypy.response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
        cherrypy.response.headers['Pragma'] = 'no-cache'
        cherrypy.response.headers['Expires'] = 0

        if not report:
            raise cherrypy.HTTPRedirect('/error')

        # The report name is like "U&..." or "S&..."
        rlink = report
        shared_db, report = rlink[0], rlink[2:]

        if shared_db == 'S':
            shared_db = True
            srv_name = 'Shared'
        else:
            shared_db = False
            srv_name = 'User'

        if report in self.glob_redirects[usr]:
            redirect_dict = self.glob_redirects[usr][report]['path']
            raise cherrypy.HTTPRedirect(redirect_dict)

        if srv_name not in self.glob_reports[usr]:
            output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
            return output.render(title='Missing server', usr=usr, rlink=rlink,
                links=self.glob_links[usr],
                msg='Server `<b>{}</b>` is not defined!<br/><br/>'
                    'Go <a href="/report/home/{}">Home</a> ...'.format(srv_name, usr))

        if report not in self.glob_reports[usr][srv_name]:
            output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
            return output.render(title='Missing report', usr=usr, rlink=rlink,
                links=self.glob_links[usr],
                msg='Report `<b>{}</b>` is not defined!<br/><br/>'
                    'Go <a href="/report/home/{}">Home</a> ...'.format(report, usr))

        logDebug('Prepare {} report `{}`, for user `{}`...'.format(srv_name, report, usr))

        # All info about the report, from DB XML
        report_dict = self.glob_reports[usr][srv_name][report]

        query = report_dict['sqlquery']
        db_server, db_name, db_user, db_passwd, _ = self.db_servers[srv_name]

        conn = self.project.dbmgr.connect_db(usr, db_server, db_name, db_user, db_passwd,
            shared_db=shared_db)
        if not conn:
            output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
            return output.render(title=report, usr=usr, rlink=rlink, links=self.glob_links[usr],
                msg='Cannot connect to MySql server `{} / {}` !'.format(db_server, db_name))

        curs = conn.cursor()

        # All variables that must be replaced in Query
        vars_to_replace = re.findall('(@.+?@)', query)

        # ------------------------------------------------------------------------------------------
        # If the user didn't select fields YET :
        # ------------------------------------------------------------------------------------------

        if vars_to_replace and not cherrypy.request.params:
            # Options are defined as: Type, Label, Data
            u_options = OrderedDict()

            for opt in vars_to_replace:
                u_field = self.glob_fields[usr].get(opt.replace('@', ''))
                this_option = {}

                if not u_field:
                    output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
                    return output.render(title=report, usr=usr, rlink=rlink,
                        links=self.glob_links[usr],
                        msg='Cannot build query!<br><br>Field `<b>{}</b>` '\
                            'is not defined in the fields section!'.format(opt.replace('@', '')))

                this_option['type'] = u_field.get('type')
                this_option['label'] = u_field.get('label')

                # Field type : User Select
                if this_option['type'] == 'UserSelect':

                    u_query = u_field.get('sqlquery')

                    if not u_query:
                        output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
                        return output.render(title=report, usr=usr, rlink=rlink,
                            links=self.glob_links[usr],
                            msg='Cannot build query!<br><br>Field `<b>{}</b>` doesn\'t '\
                            'have a query!'.format(opt.replace('@', '')))

                    # Execute User Query
                    try:
                        curs.execute(u_query)
                    except MySQLdb.Error as err:
                        output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
                        return output.render(title=report, usr=usr, rlink=rlink,
                            links=self.glob_links[usr],
                            msg='Error in query `{}`!<br><br><b>MySQL Error {}</b>: {}!'.format(
                                u_query, err.args[0], err.args[1]))

                    try:
                        u_vals = curs.fetchall()
                    except Exception as err:
                        output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
                        return output.render(title=report, usr=usr, rlink=rlink,
                            links=self.glob_links[usr],
                            msg='Error in query `{}`!<br><br><b>Exception</b>: {}!'.format(u_query, err))

                    # No data available
                    if not u_vals:
                        this_option['data'] = []
                    # Data has one column
                    elif len(u_vals[0]) == 1:
                        field_data = [(val[0], val[0]) for val in u_vals]
                        this_option['data'] = field_data
                    # Data has 2 or more columns
                    else:
                        field_data = [(str(val[0]), str(val[0]) + ': ' + '| '.join(val[1:])) for val in u_vals]
                        this_option['data'] = field_data

                # Field type : User Text
                elif this_option['type'] == 'UserText':
                    this_option['data'] = ''

                else:
                    output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
                    return output.render(title=report, usr=usr, rlink=rlink,
                        links=self.glob_links[usr],
                        msg='Field `<b>{}</b>` is of unknown type: <b>{}</b>!'.format(
                            opt.replace('@', ''), this_option['type']))

                u_options[opt] = this_option

            output = Template(filename=TWISTER_PATH + '/server/template/rep_base.htm')
            return output.render(title=report, usr=usr, rlink=rlink,
                    links=self.glob_links[usr], options=u_options)


        # ------------------------------------------------------------------------------------------
        # If the user has selected the fields :
        # ------------------------------------------------------------------------------------------

        ajax_links = []

        # ... For normal Queries ...
        for field in vars_to_replace:
            # The value chosen by the user
            u_select = cherrypy.request.params.get(field)
            if not u_select:
                u_select = ''
            ajax_links.append(field +'='+ u_select)
            # Replace @variables@ with user chosen value
            query = query.replace(field, str(u_select))

        ajax_links = sorted(set(ajax_links))
        ajax_link = '/report/json/' + rlink + '/' + usr + '?' + '&'.join(ajax_links)
        user_choices = ('", '.join(ajax_links))
        user_choices = user_choices.replace('@', '').replace('=', '="') + '"'
        del ajax_links

        try:
            curs.execute(query)
        except MySQLdb.Error as err:
            output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
            return output.render(title=report, usr=usr, rlink=rlink, links=self.glob_links[usr],
                msg='Error in query `{}`!<br><br>' \
                    '<b>MySQL Error {}</b>: {}!'.format(query, err.args[0], err.args[1]))

        descr = [desc[0] for desc in curs.description]

        # ... For Query Compare side by side, the table is double ...
        query_compr = report_dict['sqlcompr']

        if query_compr:
            # All variables that must be replaced in Query
            vars_to_replace = re.findall('(@.+?@)', query_compr)

            for field in vars_to_replace:
                # The value chosen by the user
                u_select = cherrypy.request.params.get(field)
                # Replace @variables@ with user chosen value
                query_compr = query_compr.replace(field, str(u_select))

            try:
                curs.execute(query_compr)
            except MySQLdb.Error as err:
                output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
                return output.render(title=report, usr=usr, rlink=rlink, links=self.glob_links[usr],
                    msg='Error in query `{}`!<br><br>' \
                    '<b>MySQL Error {}</b>: {}!'.format(query_compr, err.args[0], err.args[1]))

            headers_tot = [desc[0] for desc in curs.description]

            # Update headers: must contain both headers.
            descr = descr + ['vs.'] + headers_tot

            # Write DEBUG
            #DEBUG.write(report +' -> '+ user_choices +' -> '+ query_compr + '\n\n') ; DEBUG.flush()

        output = Template(filename=TWISTER_PATH + '/server/template/rep_base.htm')
        return output.render(usr=usr, title=report, rlink=rlink, links=self.glob_links[usr],
                             ajax_link=ajax_link, user_choices=user_choices,
                             report=descr, chart=report_dict['type'])


    @cherrypy.expose
    def json(self, report, usr, **args):
        """
        The report data, in json format.
        """
        if not usr:
            output = {'aaData':[], 'error':'Error! This link should be '\
                      'accessed by passing a username, eg: ' \
                      '/json/some_report/some_user'}
            return json.dumps(output, indent=2)

        if not os.path.isdir(userHome(usr) + '/twister/config'):
            output = {'aaData':[], 'error':'Error! Username `{}` doesn\'t have '\
                'a Twister config folder!'.format(usr)}
            return json.dumps(output, indent=2)

        self.load_config(usr) # Re-load all Database XML

        cherrypy.response.headers['Content-Type'] = 'application/json; charset=utf-8'
        cherrypy.response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
        cherrypy.response.headers['Pragma'] = 'no-cache'
        cherrypy.response.headers['Expires'] = 0

        # The report name is like "U&..." or "S&..."
        shared_db, report = report[0], report[2:]

        if shared_db == 'S':
            shared_db = True
            srv_name = 'Shared'
        else:
            shared_db = False
            srv_name = 'User'

        if srv_name not in self.glob_reports[usr]:
            output = {'aaData':[], 'error':'Server `{}` is not in the list of defined servers!'.format(srv_name)}
            return json.dumps(output, indent=2)

        if report not in self.glob_reports[usr][srv_name]:
            output = {'aaData':[], 'error':'Report `{}` is not in the list of defined reports!'.format(report)}
            return json.dumps(output, indent=2)

        # All info about the report, from DB XML.
        report_dict = self.glob_reports[usr][srv_name][report]

        query = report_dict['sqlquery']
        db_server, db_name, db_user, db_passwd, _ = self.db_servers[srv_name]

        conn = self.project.dbmgr.connect_db(usr, db_server, db_name, db_user, db_passwd,
                shared_db=shared_db)
        if not conn:
            output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
            return output.render(links=self.glob_links[usr], title=report, usr=usr,
                msg='Cannot connect to MySql server `{} / {}` !'.format(db_server, db_name))

        curs = conn.cursor()

        # All variables that must be replaced in Query
        vars_to_replace = re.findall('(@.+?@)', query)

        for field in vars_to_replace:
            # The value chosen by the user
            u_select = cherrypy.request.params.get(field)
            # Replace @variables@ with user chosen value
            query = query.replace(field, str(u_select))

        try:
            curs.execute(query)
        except MySQLdb.Error as err:
            output = {'aaData':[], 'error':'Error in query `{}`! ' \
                'MySQL Error {}: {}!'.format(query, err.args[0], err.args[1])}
            return json.dumps(output, indent=2)

        headers = [desc[0] for desc in curs.description]
        rows = curs.fetchall()
        del query

        query_total = report_dict['sqltotal']
        query_compr = report_dict['sqlcompr']


        # ... Calculate SQL Query Total ...
        if query_total:
            # All variables that must be replaced in Query
            vars_to_replace = re.findall('(@.+?@)', query_total)

            for field in vars_to_replace:
                # The value chosen by the user
                u_select = cherrypy.request.params.get(field)
                # Replace @variables@ with user chosen value
                query_total = query_total.replace(field, str(u_select))

            try:
                curs.execute(query_total)
            except MySQLdb.Error as err:
                output = {'aaData':[], 'error':'Error in query total `{}`! ' \
                    'MySQL Error {}: {}!'.format(query_total, err.args[0], err.args[1])}
                return json.dumps(output, indent=2)

            headers_tot = [desc[0] for desc in curs.description]
            rows_tot = curs.fetchall()

            if len(headers) != len(headers_tot):
                output = {'aaData':[], 'error':'The first query has {} columns and the second has {} columns!'
                    .format(len(headers), len(headers_tot))}
                return json.dumps(output, indent=2)

            # Will calculate the new rows like this:
            # The first column of the first query will not be changed
            # The second row of the first query / the second row of the second query * 100
            calc_rows = []

            rows = {r[0]:r[1] for r in rows}
            rows_tot = {r[0]:r[1] for r in rows_tot}

            for rnb in rows_tot.keys():
                if rnb in rows.keys():
                    # Calculate percent...
                    percent = '%.2f' % (float(rows[rnb]) / rows_tot[rnb] * 100.0)
                    # Using the header from Total, because it might be Null in the first query
                    calc_rows.append([rnb, float(percent)])
                else:
                    calc_rows.append([rnb, 0.0])

        # ... SQL Query Compare side by side ...
        elif query_compr:
            # All variables that must be replaced in Query
            vars_to_replace = re.findall('(@.+?@)', query_compr)

            for field in vars_to_replace:
                # The value chosen by the user
                u_select = cherrypy.request.params.get(field)
                # Replace @variables@ with user chosen value
                query_compr = query_compr.replace(field, str(u_select))

            try:
                curs.execute(query_compr)
            except MySQLdb.Error as err:
                output = {'aaData':[], 'error':'Error in query compare `{}`! '\
                    'MySQL Error {}: {}!'.format(query_total, err.args[0], err.args[1])}
                return json.dumps(output, indent=2)

            headers_tot = [desc[0] for desc in curs.description]
            rows_tot = curs.fetchall()

            if len(headers) != len(headers_tot): # Must be the same number of columns
                output = {'aaData':[], 'error':'The first query has {} columns and the second has {} columns!'
                    .format(len(headers), len(headers_tot))}
                return json.dumps(output, indent=2)

            headers_len = len(headers)
            rows_max_size = max(len(rows), len(rows_tot))
            calc_rows = []

            for i in range(rows_max_size):
                row1 = rows[i:i+1]
                row2 = rows_tot[i:i+1]
                if not row1:
                    row1 = [' ' for i in range(headers_len)]
                else:
                    row1 = row1[0]
                if not row2:
                    row2 = [' ' for i in range(headers_len)]
                else:
                    row2 = row2[0]
                calc_rows.append(tuple(row1) +(' <---> ',)+ tuple(row2))

            # Update headers: must contain both headers.
            headers = headers + ['vs.'] + headers_tot

        # ... Normal Query ...
        else:
            calc_rows = rows
            del rows

        if (not calc_rows) or (not calc_rows[0:1]):
            output = {'aaData':[], 'error':'The select is empty!'}
            return json.dumps(output, indent=2)

        if isinstance(calc_rows[0][0], datetime.datetime):
            is_date = True
        else:
            is_date = False

        dthandler = lambda obj: obj.strftime('%Y-%m-%d %H:%M:%S') if isinstance(obj, datetime.datetime) else None
        return json.dumps({'headers':headers, 'type':report_dict['type'], 'isDate':is_date, 'aaData':calc_rows},
            indent=2, default=dthandler)


    @cherrypy.expose
    def error(self, **args):
        """
        The error page.
        """
        output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
        return output.render(title='Error 404', links=[], msg='Sorry, this page does not exist!')


    @cherrypy.expose
    def default(self, **args):
        """
        The error page.
        """
        output = Template(filename=TWISTER_PATH + '/server/template/rep_error.htm')
        return output.render(title='Error 404', links=[], msg='Sorry, this page does not exist!') class Suts(_cptools.XMLRPCController, CommonAllocator):
    """
    Basic operations for SUTs.
    """

    def __init__(self, project):

        self.project = project
        self.type = 'sut'
        self.resources = CONSTANT_DICTIONARY
        self.reservedResources = {}
        self.lockedResources = {}
        self.id_list = {}
        self.acc_lock = thread.allocate_lock() # Task change lock
        self.ren_lock = thread.allocate_lock() # Rename lock
        self.imp_lock = thread.allocate_lock() # Import lock
        self.save_lock = thread.allocate_lock() # Save lock
        self.load_lock = thread.allocate_lock() # Save lock


    def save_sut(self, props={}, resource_name=None):
        """
        Function used to write the changes on HDD.
        The save is separate for Devices and SUTs, so the version is not incremented
        for both, before saving.
        """
        user = self.user_info(props)[0]
        logDebug('CeSuts:_save {} {} {} '.format(props, resource_name, user))
        log = []

        # Write changes, using the Access Lock.
        with self.save_lock:
            if resource_name[0] == '/':
                resource_name = resource_name[1:]

            if resource_name.split('.')[-1] == 'user':
                suts_gb_path = self.project.get_user_info(user, 'sut_path')
            else:
                suts_gb_path = self.project.get_user_info(user, 'sys_sut_path')

            if not suts_gb_path:
                suts_gb_path = '{}/config/sut/'.format(TWISTER_PATH)

            filename = os.path.join(suts_gb_path, '.'.join(resource_name.split('.')[:-1] + ['json']))

            if resource_name.split('.')[-1] == 'system':
                try:
                    resp = self.project.localFs.write_system_file(filename, \
                        json.dumps(self.resources['children'][resource_name], indent=4), 'w')
                except Exception as exp_err:
                    log.append(exp_err)
                    logError('User {}: Saving ERROR system:: `{}`.'.format(user, exp_err))

            if resource_name.split('.')[-1] == 'user':
                # user SUT file; we have to check if the cleacase plugin
                # is activated; if so, use it to write the SUT file; else
                # use the UserService to read it
                cc_cfg = self.project.get_clearcase_config(user, 'sut_path')
                if cc_cfg:
                    view = cc_cfg['view']
                    actv = cc_cfg['actv']
                    path = cc_cfg['path']
                    user_view_actv = '{}:{}:{}'.format(user, view, actv)
                    f_name = ''.join(resource_name.split('.')[:-1])
                    resp = self.project.clearFs.write_user_file(user_view_actv, path +'/'+ f_name + '.json', \
                        json.dumps(self.resources['children'][resource_name], indent=4))
                else:
                    # Get the user connection
                    resp = self.project.localFs.write_user_file(user, filename, \
                        json.dumps(self.resources['children'][resource_name], indent=4), 'w')
                if resp is not True:
                    log.append(resp)
                    logError('User {}: Saving ERROR user:: `{}`.'.format(user, resp))

                # targeted resource is saved now; do not continue with
                # the rest of resources
        if log:
            return '*ERROR* ' + str(log)

        # update id_list
        self.parse_sut(self.resources['children'][resource_name], resource_name)

        return True


    @cherrypy.expose
    def save_sut_as(self, sut_name, path):
        """
        Function used to write the sut on the user specified path on the disk.
        sut_name: is the new name of the sut to be saved
        path: is the path where the sut is saved. It doesn't contain the sut name at the end.
        """
        user = self.user_info({})[0]
        logFull('CeSuts:_save_as {} {} {} '.format(sut_name, path, user))
        log = []

        # Write changes, using the Access Lock.
        with self.save_lock:
            if sut_name[0] == '/':
                sut_name = sut_name[1:]
            if not os.path.exists(path):
                msg = 'Path does not exist: {}'.format(path)
                logError(msg)
                return '*ERROR* '+msg
            # remove the '_' added when the sut was created
            real_name = sut_name[1:]
            filename = os.path.join(path, '.'.join(real_name.split('.')[:-1] + ['json']))
            try:
                self.project.localFs.write_user_file(user, filename, \
                json.dumps(self.resources['children'][sut_name], indent=4), 'w')
            except Exception as exp_err:
                log.append(exp_err)
                logError('User {}: Saving ERROR system:: `{}`.'.format(user, exp_err))
                # targeted resource is saved now; do not continue with
                # the rest of resources
        if log:
            return '*ERROR* ' + str(log)

        # remove the new sut from resources
        self.resources['children'].pop(sut_name)
        logDebug('Sut resource: `{}` removed from the resources'.format(sut_name))
        return True


    def format_content(self, content, kids_list):
        """
        Find all the ids from a sut.
        Helps creating id_list.
        """
        if not content:
            return kids_list

        if not content.get('children'):
            kids_list.add(content['id'])
            return kids_list

        for node in content['children']:
            kids_list.add(content['children'][node]['id'])
            self.format_content(content['children'][node], kids_list)


    def parse_sut(self, sut_content=None, sut_name=None):
        """
        Adds elements to id_list.
        """
        kids_list = set()
        kids_list.add(sut_content['id'])

        self.format_content(sut_content, kids_list)

        kids_list = list(kids_list)
        self.id_list[sut_name] = kids_list

        return True


    def find_sut_id(self, sut_id):
        """
        Search for an ID in id_list.
        """
        for key, value in self.id_list.items():
            if sut_id in value:
                return key
        return False


    def _format_dict_sut(self, result, query):
        """
        Helper function.
        """
        try:
            result = self.format_resource(result, query)
        except Exception:
            logFull('The sut `{}` is already formated !'.format(query))

        if isinstance(result['path'], list):
            result['path'] = '/'.join(result['path'])
        return result


    @cherrypy.expose
    def index_suts(self, props={}):
        """
        Open all SUT files and creats dict having
        entries like: {sut_name : [ids]}
        """
        user_info = self.user_info(props)
        user = user_info[0]
        usr_home = userHome(user)

        sut_content = False

        try:
            # System SUT path
            suts_gb_path = self.project.get_user_info(user, 'sys_sut_path')
            if not suts_gb_path:
                suts_gb_path = '{}/config/sut/'.format(TWISTER_PATH)

            sut_all_pats = [p for p in os.listdir(suts_gb_path)\
                if os.path.isfile(os.path.join(suts_gb_path, p))\
                     and p.split('.')[-1] == 'json']
            for sut_path in sut_all_pats:
                sut_name = '.'.join(['.'.join(sut_path.split('.')[:-1]  + ['system'])])

                with open(os.path.join(suts_gb_path, sut_path), 'r') as f_p:
                    sut_content = json.load(f_p)
                    self.parse_sut(sut_content, sut_name)
        except Exception as exp_err:
            logError('_load ERROR:: {} for user {}'.format(exp_err, user))

        # User SUT path
        suts_gb_path = self.project.get_user_info(user, 'sut_path')
        if not suts_gb_path:
            suts_gb_path = '{}/twister/config/sut/'.format(usr_home)

        # user SUT file; we have to check if the cleacase plugin
        # is activated; if so, use it to read the SUT file; else
        # use the UserService to read it
        # open all the json files and parse them - need to index all the ids
        # that sut contains
        cc_cfg = self.project.get_clearcase_config(user, 'sut_path')
        if cc_cfg:
            view = cc_cfg['view']
            actv = cc_cfg['actv']
            path = cc_cfg['path']
            user_view_actv = '{}:{}:{}'.format(user, view, actv)

            sut_all_pats = [p for p in os.listdir(suts_gb_path)\
                if os.path.isfile(os.path.join(suts_gb_path, p))\
                     and p.split('.')[-1] == 'json']

            for sut_path in sut_all_pats:
                sut_name = '.'.join(['.'.join(sut_path.split('.')[:-1]  + ['user'])])
                resp = self.project.clearFs.read_user_file(user_view_actv, path +'/'+ sut_name)
                try:
                    sut_content = json.loads(resp)
                except Exception:
                    msg = "User {}: Cannot load ClearCase SUT `{}`!".format(user, sut_name)
                    logWarning(msg)
                    return '*ERROR* ' + msg
                self.parse_sut(sut_content, sut_name)

        else:
            sut_all_pats = [p for p in os.listdir(suts_gb_path)\
                if os.path.isfile(os.path.join(suts_gb_path, p))\
                     and p.split('.')[-1] == 'json']
            for sut_path in sut_all_pats:
                sut_name = '.'.join(['.'.join(sut_path.split('.')[:-1]  + ['user'])])

                if suts_gb_path[-1] != '/' and sut_path[-1] != '/':
                    complete_sut_path = suts_gb_path + '/' + sut_path
                else:
                    complete_sut_path = suts_gb_path + sut_path

                resp = self.project.localFs.read_user_file(user, complete_sut_path)
                try:
                    sut_content = json.loads(resp)
                except Exception:
                    msg = "User {}: Cannot load SUT `{}`!".format(user, sut_name)
                    logWarning(msg)
                    return '*ERROR* ' + msg
                self.parse_sut(sut_content, sut_name)

        return True


    @cherrypy.expose
    def get_sut(self, query, props={}):
        """
        Get the contant of one SUT file using it's name.
        Must provide a SUT name.<type> (type = user/system)

        If query is an id -> positive answer only if sut is in self.resources
        """
        user_info = self.user_info(props)
        username = user_info[0]
        follow_links = False

        # Follow links to TestBeds ?
        if 'follow_links' in props:
            follow_links = props['follow_links']
            del props['follow_links']

        logDebug('CeSuts: get_sut {} {} {}'.format(query, username, follow_links))
        usr_home = userHome(username)
        initial_query = None

        if not query:
            msg = "The name of the SUT is empty!"
            logDebug(msg)
            return False

        if ':' in query:
            meta = query.split(':')[1]
            query = query.split(':')[0]
        else:
            meta = ''

        sut_type = query.split('.')[-1]

        # This is an ID
        # Will return result only if the SUT is in self.resources
        if '/' not in query and sut_type == query:
            initial_query = query
            res_dict = self.get_resource(query)

            if isinstance(res_dict, dict):
                if len(res_dict['path']) > 1:
                    sut_content = self._format_dict_sut(res_dict, query)

                    # If this SUT / component is linked with a TB
                    if sut_content['meta'].get('_id') and follow_links:

                        # Ok, this might be a Device path, instead of SUT path!
                        tb_id = sut_content['meta']['_id']
                        self.project.testbeds.load_tb(verbose=False)
                        sut_content = self.project.testbeds.\
                        get_tb(tb_id, props)

                        # If the Device ID is invalid, bye bye!
                        if not isinstance(sut_content, dict):
                            return False

                    if meta:
                        return sut_content['meta'].get(meta, '')
                    else:
                        return sut_content

                query = res_dict['path'][0]
                sut_type = query.split('.')[-1]

            else:
                # Maybe this sut is already indexed
                result = self.find_sut_id(query)
                # If not index all suts and search
                if not result:
                    self.index_suts(props)
                    result = self.find_sut_id(query)
                    # Probably is a component not saved yet, get it from self.reservedResources
                    if not result:
                        if meta:
                            result = self.get_info_sut(query + ":" + meta, props)
                        else:
                            result = self.get_info_sut(query, props)

                        if isinstance(result, dict):
                            result = self._format_dict_sut(result, query)
                            return result
                        else:
                            msg = 'Invalid SUT ID `{}`, for user `{}`!'.format(query, username)
                            logWarning(msg)
                            return msg

                query = result
                sut_type = query.split('.')[-1]

        # if the query is for a component return the entire SUT
        if query[1:].count('/') >= 1:
            initial_query = query
            parts = [q for q in query.split('/') if q]
            query = parts[0]

        try:
            index = sut_type.index('/')
            sut_type = sut_type[:index]
        except Exception:
            logFull('SutType does not contain any /')

        if sut_type == 'system':
            # System SUT path
            sut_path = self.project.get_user_info(username, 'sys_sut_path')
            if not sut_path:
                sut_path = '{}/config/sut/'.format(TWISTER_PATH)
        elif sut_type == 'user':
            # User SUT path
            sut_path = self.project.get_user_info(username, 'sut_path')
            if not sut_path:
                sut_path = '{}/twister/config/sut/'.format(usr_home)
        else:
            msg = 'Invalid SUT type `{}`, for user `{}`!'.format(sut_type, username)
            logWarning(msg)
            return msg

        # avoid having more than one "/" or not having at all
        if query[0] == '/' and sut_path[-1] == '/':
            query = query[1:]
        elif query[0] != '/' and sut_path[-1] != '/':
            query = '/' + query

        f_name = '.'.join(query.split('.')[:-1]) + '.json'
        sut_file = sut_path + f_name
        sut_content = False

        if not os.path.isdir(sut_path):
            # Cannot get read access to the SUT directory
            msg = '*ERROR* Cannot get access to SUT path `{}`, user `{}`!'.format(sut_path, username)
            logWarning(msg)
            return msg

        if sut_type == 'system':
            # System SUT file
            try:
                with open(sut_file, 'r') as f_p:
                    sut_content = json.load(f_p)
            except Exception as exp_err:
                return '*ERROR* User {}: Cannot read SUT file `{}`! Exception {}'.format(
                    username, sut_file, exp_err)

        else:
            # User SUT file, check if the ClearCase plugin is activated
            # If so, use it to read the SUT file; else use the UserService to read it
            cc_cfg = self.project.get_clearcase_config(username, 'sut_path')
            if cc_cfg:
                view = cc_cfg['view']
                actv = cc_cfg['actv']
                path = cc_cfg['path']
                user_view_actv = '{}:{}:{}'.format(username, view, actv)
                sut_path = (path +'/'+ f_name).replace('//', '/')
                resp = self.project.clearFs.read_user_file(user_view_actv, sut_path)
                # Invalid sut file?
                if resp.startswith('*ERROR*'):
                    logWarning(resp)
                    return resp
                try:
                    sut_content = json.loads(resp)
                except Exception:
                    logWarning('User {}: Cannot load ClearCase SUT `{}`!'.format(username, sut_path))
                    sut_content = False
            else:
                sut_path = (sut_path + '/' + f_name).replace('//', '/')
                resp = self.project.localFs.read_user_file(username, sut_path)
                # Invalid sut file?
                if resp.startswith('*ERROR*'):
                    logWarning(resp)
                    return resp
                try:
                    sut_content = json.loads(resp)
                except Exception:
                    logWarning('User {}: Cannot load SUT `{}`!'.format(username, sut_path))
                    sut_content = False

        if isinstance(sut_content, str) and sut_content.startswith('*ERROR*'):
            return sut_content

        if (sut_content is False) or (not isinstance(sut_content, dict)):
            msg = 'User {}: Invalid SUT `{}`!'.format(username, f_name)
            logWarning(msg)
            return '*ERROR* ' + msg

        # At this point, the SUT is a valid dictionary
        if query[0] == '/':
            query = query[1:]

        if sut_content.get('path'):
            sut_content['path'] = sut_content['path'][0]
        else:
            sut_content['path'] = query

        self.resources['children'][query] = copy.deepcopy(sut_content)
        # make older resources files that don't have 'path' compatible
        self.resources['children'][query]['path'] = [query]
        modified = self.fix_path(self.resources['children'][query], [query])

        if modified:
            # now we have to save the version with path
            issaved = self.save_sut(props, query)
            if isinstance(issaved, str):
                msg = 'Cannot save SUT `{}`, for user `{}`!'.format(query, username)
                logWarning(msg)
                return msg

        if initial_query:
            result = self.get_info_sut(initial_query, props)

            if isinstance(result, dict):
                sut_content = self._format_dict_sut(result, query)

            elif follow_links:
                parts = [p for p in initial_query.split('/') if p]

                if len(parts) == 1:
                    msg = '*ERROR* Internal server error!'
                    logWarning(msg)
                    return msg

                curr_parts = [parts.pop(0)]

                while 1:
                    try:
                        curr_parts.append(parts.pop(0))
                    except Exception:
                        break
                    curr_path = '/' + '/'.join(curr_parts)

                    result = self.get_resource(curr_path)

                    # SUT not loaded
                    if not isinstance(result, dict):
                        msg = '*ERROR* Invalid SUT path `{}`!'.format(curr_path)
                        logWarning(msg)
                        return False

                    if result['meta'].get('_id'):
                        # Ok, this might be a Device path, instead of SUT path!
                        tb_id = result['meta']['_id']
                        self.project.testbeds.load_tb(verbose=False)
                        result = self.project.testbeds.get_tb(tb_id, props)

                        # If the Device ID is invalid, bye bye!
                        if not isinstance(result, dict):
                            return False

                        tb_path = '/' + result['path'] + '/' + '/'.join(parts)
                        result = self.project.testbeds.get_tb(tb_path, props)

                        # If the Device ID is invalid, bye bye!
                        if not isinstance(result, dict):
                            return False

                        if meta:
                            return result['meta'].get(meta, '')
                        else:
                            return result

            # Internal error?
            else:
                msg = '*ERROR* Invalid SUT path `{}`!'.format(initial_query)
                logWarning(msg)
                return msg

        try:
            sut_content = self.format_resource(sut_content, query)
        except Exception:
            logFull('User {}: The sut is already formated {}.'.format(username, query))

        # If this SUT / component is linked with a TB
        if sut_content['meta'].get('_id'):

            # Ok, this might be a Device path, instead of SUT path!
            tb_id = sut_content['meta']['_id']
            self.project.testbeds.load_tb(verbose=False)
            sut_content = self.project.testbeds.get_tb(tb_id, props)

            # If the Device ID is invalid, bye bye!
            if not isinstance(sut_content, dict):
                msg = '*ERROR* Invalid TB link on SUT `{}`!'.format(initial_query)
                logWarning(msg)
                return msg

        if meta:
            return sut_content['meta'].get(meta, '')
        else:
            return sut_content


    @cherrypy.expose
    def get_info_sut(self, res_query, props={}):
        """
        Get the current version of the meta SUT modified and unsaved or
        the version from the disk.
        """
        user_info = self.user_info(props)
        logDebug('CeSuts:get_meta_sut {} {}'.format(res_query, props))

        if ':' in res_query:
            res_query, meta = res_query.split(':')
        else:
            meta = ''

        result = None

        # If the SUT is reserved, get the latest unsaved changes
        if user_info[0] in self.reservedResources:
            for i in range(len(self.reservedResources[user_info[0]].values())):
                current_res_reserved = CONSTANT_DICTIONARY
                current_path_root = self.reservedResources[user_info[0]].values()[i]['path'][0]
                current_res_reserved['children'][current_path_root] = self.reservedResources[user_info[0]].values()[i]

                result = self.get_resource(res_query, current_res_reserved)
                if isinstance(result, dict):
                    break

        if not isinstance(result, dict):
            result = self.get_resource(res_query)
            # SUT not loaded
            if not isinstance(result, dict):
                msg = 'Cannot find `{}`. Call `get_sut` for a SUT, component, or meta.'.format(res_query)
                logWarning(msg)
                return False

        if isinstance(result, dict):
            if meta:
                return result['meta'].get(meta, '')
            else:
                return result

        return False


    @cherrypy.expose
    def reserve_sut(self, res_query, props={}):
        """
        load the SUT wanted and then reserve it
        """
        self.get_sut(res_query, props)
        return self.reserve_resource(res_query, props)


    @cherrypy.expose
    def create_component_sut(self, name, parent=None, props={}):
        """
        Create new component for an existing SUT
        """
        logFull('CeSuts:create_component_sut: parent = {}  props = {}  name = {}'.format(parent, props, name))

        if parent == '/' or parent == '1':
            msg = "The parent value is not an existing SUT. Mayebe you want to add a new SUT. Parent: {}".format(parent)
            logError(msg)
            return '*ERROR* ' + msg

        user_info = self.user_info(props)

        _is_res_reserved = self.is_resource_reserved(parent, props)
        if _is_res_reserved and _is_res_reserved != user_info[0]:
            msg = 'User {}: Cannot create new component: The SUT is reserved for {} !'\
            .format(user_info[0], _is_res_reserved)
            logError(msg)
            return '*ERROR* ' + msg

        _is_res_locked = self.is_resource_locked(parent)
        if _is_res_locked and _is_res_locked != user_info[0]:
            msg = 'User {}: Reserve SUT: The SUT is locked for {} !'\
            .format(user_info[0], _is_res_locked)
            logError(msg)
            return '*ERROR* ' + msg

        props = self.valid_props(props)
        with self.acc_lock:
            #the resource should be reserved previously
            parent_p = self.get_reserved_resource(parent, props)

            if not parent_p:
                logError('Cannot access reserved SUT, path or ID `{}` !'.format(parent))
                return False

            if name in parent_p['children']:
                msg = "A component with this name '{}'' already exists for this Sut: '{}'".format(name, parent)
                logDebug(msg)
                return '*ERROR* ' + msg

            #the resources is deep in the tree, we have to get its direct parent
            if len(parent_p['path']) >= 2:
                full_path = parent_p['path']
                base_path = "/".join(parent_p['path'][1:])
                parent_p = self.get_path(base_path, parent_p)
                parent_p['path'] = full_path

            if '/' in name:
                logDebug('Stripping slash characters from `{}` `{}`...'.format(name, parent))
                name = name.replace('/', '')

            # the resource doesn't exist - create it
            res_id = self.generate_index()
            parent_p['children'][name] = {'id': res_id, 'meta': props, 'children': {}, 'path':parent_p['path'] + [name]}
            epnames_tag = '_epnames_{}'.format(user_info[0])

            # If the epnames tag exists in resources
            if epnames_tag in parent_p['children'][name]['meta']:
                # And the tag is empty
                if not parent_p['children'][name]['meta'][epnames_tag]:
                    logDebug('User {}: Deleting `{}` tag from new resource.'.format(user_info[0], epnames_tag))
                    del parent_p['children'][name]['meta'][epnames_tag]

        return res_id


    @cherrypy.expose
    def create_new_sut(self, name, parent=None, props={}, save=True):
        """
        Create a SUT.
        """
        user_info = self.user_info(props)
        logFull('CeSuts: create_new_sut: parent = {} -- props = {} -- name = {}'.format(parent, props, name))
        props = self.valid_props(props)

        if parent != '/' and parent != "1":
            msg = "User {}: The parent value is not root. Mayebe you want to add a component \
            to an existing SUT. Parent: {}".format(user_info, parent)
            logError(msg)
            return '*ERROR* ' + msg

        with self.acc_lock:
            #root can not be reserved so we just take it
            if name.split('.')[-1] != 'user' and \
            name.split('.')[-1] != 'system':
                name = '.'.join([name, 'user'])

            if '/' in name:
                logDebug('Stripping slash characters from `{}`...'.format(name))
                name = name.replace('/', '')

            if name in self.resources['children']:
                msg = "User {}: A SUT with this name '{}'' already exists'".format(user_info[0], name)
                logDebug(msg)
                return '*ERROR* ' + msg

            # the resource doesn't exist - create it
            res_id = self.generate_index()
            self.resources['children'][name] = {'id': res_id, 'meta': props, 'children': {}, 'path': [name]}

            #save this new SUT
            if save:
                issaved = self.save_sut(props, name)
                if  isinstance(issaved, str):
                    msg = "We could not save this SUT {}".format(name)
                    logDebug(msg)
                    return issaved

            return res_id


    @cherrypy.expose
    def update_meta_sut(self, name, parent=None, props={}):
        """
        Modify a SUT using a name, a parent Path or ID and some properties.
        This method changes meta for a certain SUT.
        """
        user_info = self.user_info(props)
        logDebug('CeSuts:update_meta_sut: parent = {} -- props = {} -- username = {} -- name = {}'\
            .format(parent, props, user_info[0], name))

        # if props does not have the correct format we stop this operation
        if not props or not self.valid_props(props):
            msg = "Wrong format for props = {}".format(props)
            logDebug(msg)
            return '*ERROR* ' + msg

        # can not verify if reserved '/' because I load only the sut that I need
        if parent == "/" or parent == "1":
            if name.split('.')[-1] != 'user' and \
            name.split('.')[-1] != 'system':
                name = '.'.join([name, 'user'])
            # we can not reserve the root so we just take the sut we need
            verify_reserved = name
        else:
            # take the sut that has the component we need
            verify_reserved = parent

        # what if this sut is already reserved by other user? We STOP
        _is_res_reserved = self.is_resource_reserved(verify_reserved, props)
        if _is_res_reserved and _is_res_reserved != user_info[0]:
            msg = 'User {}: Cannot update meta: The resource is reserved for {} !'\
            .format(user_info[0], _is_res_reserved)
            logError(msg)
            return '*ERROR* ' + msg

        # what if this sut is already locked by other user? We STOP
        _is_res_locked = self.is_resource_locked(verify_reserved)
        if _is_res_locked and _is_res_locked != user_info[0]:
            msg = 'User {}: Reserve resource: The resource is locked for {} !'\
            .format(user_info[0], _is_res_locked)
            logError(msg)
            return '*ERROR* ' + msg

        with self.acc_lock:

            parent_p = self.get_reserved_resource(verify_reserved, props)

            if not parent_p:
                logError('Cannot access reserved SUT, path or ID `{}` !'.format(verify_reserved))
                return False

            if '/' in name:
                logDebug('Stripping slash characters from `{}`...'.format(name))
                name = name.replace('/', '')

            # the resources is deep in the tree, we have to get its direct parent
            if len(parent_p['path']) >= 2:
                full_path = parent_p['path']
                base_path = "/".join(parent_p['path'][1:])
                parent_p = self.get_path(base_path, parent_p)
                parent_p['path'] = full_path

            # get the child, update its meta
            if name in parent_p['children']:
                child_p = parent_p['children'][name]
            # update the parent itself
            elif name in parent_p['path']:
                child_p = parent_p

            # We have to update the props
            props = self.valid_props(props)
            epnames_tag = '_epnames_{}'.format(user_info[0])
            child_p['meta'].update(props)

            # if _id key is present in meta and it has no value, we have
            # to remove it from meta dictionary
            if '_id' in child_p['meta'].keys() and not child_p['meta'].get('_id', False):
                child_p['meta'].pop('_id')

            # If the epnames tag exists in resources
            if epnames_tag in child_p['meta']:
                # And the tag is empty
                if not child_p['meta'][epnames_tag]:
                    logDebug('User {}: Deleting `{}` tag from resources.'.format(user_info[0], epnames_tag))
                    del child_p['meta'][epnames_tag]
        return "true"


    @cherrypy.expose
    def set_sut(self, name, parent=None, props={}):
        """
        Higher level wrapper over functions Create new SUT, create component and update meta.
        """
        if parent == '/' or parent == '1':
            if name[0] != '/':
                name = '/' + name
            ndata = self.get_sut(name, props)
            if not isinstance(ndata, dict):
                return self.create_new_sut(name, parent, props)
            else:
                return self.update_meta_sut(name, parent, props)

        # The parent is NOT root
        self.get_sut(parent, props)
        pdata = self.get_resource(parent)
        user_info = self.user_info(props)

        if not isinstance(pdata, dict):
            logWarning('User `{}`: No such parent `{}`!'.format(user_info[0], parent))
            return False

        # If exists, update meta
        if name in pdata['children']:
            return self.update_meta_sut(name, parent, props)
        # This is a new component
        else:
            return self.create_component_sut(name, parent, props)


    @cherrypy.expose
    def rename_sut(self, res_query, new_name, props={}):
        """
        Rename a SUT if it is not reserved or locked by someone.
        If its asked to rename the root of a sut and
        new_name doesn't specify the type (user/system)
        we add ".user". We do that by creating a new Sut having
        name: new_name and delete the old Sut.
        """
        user_info = self.user_info(props)
        logFull('CeSuts:rename_sut {} {}'.format(res_query, props))

        if '/' in new_name or ':' in new_name:
            msg = 'New resource name ({}) cannot contain `/` or `:`!'.format(new_name)
            logError(msg)
            return '*ERROR* ' + msg

        if ':' in res_query:
            res_query = res_query.split(':')[0]

        usr_sut_list = self.list_all_suts(user_info[0])

        # check if res_query exists
        if res_query[0] == '/':
            res_query = res_query[1:]

        found_old_sut = [item for item in usr_sut_list if item['name'] == res_query]
        if not found_old_sut:
            msg = 'User {}: SUT file {} doesn\'t exist !'.format(user_info[0], res_query)
            logError(msg)
            return '*ERROR* ' + msg

        # add 'user' or 'system' at the end of the new_name if it does not have it
        if new_name.split('.')[-1] != 'user' and new_name.split('.')[-1] != 'system':
            if res_query.split('.')[-1] == 'user' or res_query.split('.')[-1] == 'system':
                new_name = '.'.join([new_name, res_query.split('.')[-1]])
            else:
                new_name = '.'.join([new_name, 'user'])

        # check that the 'new_name' doesn't exists
        found_new_sut = [item for item in usr_sut_list if item['name'] == new_name]
        if found_new_sut:
            msg = 'User {}: New SUT file {} already exist !'.format(user_info[0], new_name)
            logError(msg)
            return '*ERROR* ' + msg

        # make sure the SUT file names start with /
        if res_query[0] != '/':
            res_query = '/' + res_query
        if new_name[0] != '/':
            new_name = '/' + new_name

        if new_name == res_query:
            msg = "Not renaming: Both the current name and the new name are the same."
            logError(msg)
            return "*ERROR*" + msg

        # Check if resource is reserved; if so, it cannot be renamed
        _is_res_reserved = self.is_resource_reserved(res_query, props)
        if _is_res_reserved:
            msg = 'User {}: Cannot delete: The resource is reserved for {} !'\
            .format(user_info[0], _is_res_reserved)
            logError(msg)
            return '*ERROR* ' + msg

        _is_res_locked = self.is_resource_locked(res_query)
        if _is_res_locked and _is_res_locked != user_info[0]:
            msg = 'User {}: Reserve resource: The resource is locked for {} !'\
            .format(user_info[0], _is_res_locked)
            logError(msg)
            return '*ERROR* ' + msg

        # Create a new SUT having this new name
        new_sut_id = self.create_new_sut(new_name, "/", props)

        if  isinstance(new_sut_id, str):
            if '*ERROR*' in new_sut_id:
                msg = 'User {}: New SUT file {} cannot be created!'.format(user_info[0], new_name)
                logError(msg)
                return '*ERROR* ' + msg

        reserve_res = self.reserve_resource(new_name, props)
        if  isinstance(reserve_res, str):
            if '*ERROR*' in reserve_res:
                msg = 'User {}: New SUT file {} cannot be reserved!'.format(user_info[0], new_name)
                logError(msg)
                return '*ERROR* ' + msg

        # method to clean the new SUT if needed
        def clean_new_sut(new_sut, user):
            """
            Delete the new SUT if the rename Failed.
            """
            self.reservedResources[user].pop(new_sut)
            if not self.reservedResources[user]:
                self.reservedResources.pop(user)
            self.delete_sut(new_sut, props)

        # Try to reserve source SUT file; if error, clean up the new SUT
        reserve_res = self.reserve_resource(res_query, props)
        if  isinstance(reserve_res, str):
            if '*ERROR*' in reserve_res:
                msg = 'User {}: Source SUT file {} cannot be reserved!'.format(user_info[0], res_query)
                logError(msg)
                clean_new_sut(new_sut_id, user_info[0])
                return '*ERROR* ' + msg

        # get the new path
        new_sut = self.reservedResources[user_info[0]][new_sut_id]
        # get the old sut
        old_sut = self.get_reserved_resource(res_query, props)

        # get all the structure of the old sut
        new_sut['meta'] = old_sut['meta']
        new_sut['children'] = old_sut['children']
        # if the sut has children we have to update the path
        self.change_path(new_sut, new_sut['path'])

        # release the old sut; and delete if needed
        self.reservedResources[user_info[0]].pop(old_sut['id'])
        if not self.reservedResources[user_info[0]]:
            self.reservedResources.pop(user_info[0])

        deleted = self.delete_sut(res_query, props)
        if  isinstance(deleted, str) and deleted.startswith('*ERROR*'):
            #could not delete the old sut so we keep it and delete the new one
            clean_new_sut(new_sut_id, user_info[0])
            msg = "User {} :We couldn't delete the old sut so we cannot complete the rename operation."\
            .format(user_info[0])
            logError(msg)
            return '*ERROR* ' + msg
        else:
            # save and release the new sut
            self.save_release_reserved_sut(new_name, props)

        return "True"


    @cherrypy.expose
    def rename_meta_sut(self, res_query, new_name, props={}):
        """
        Rename meta for SUT.
        SUT must be reserved.
        """
        user_info = self.user_info(props)
        logFull('CeSuts:get_meta_sut {} {}'.format(res_query, props))

        if ':' in res_query:
            meta = res_query.split(':')[1]
            res_query = res_query.split(':')[0]
        else:
            logError("CeSuts:get_meta_sut: User {} called this method without meta!".format(user_info[0]))
            return "false"

        #what if this sut is already reserved by other user? We STOP
        _is_res_reserved = self.is_resource_reserved(res_query, props)
        if _is_res_reserved and _is_res_reserved != user_info[0]:
            msg = 'User {}: Cannot update meta: The resource is reserved for {} !'\
            .format(user_info[0], _is_res_reserved)
            logError(msg)
            return '*ERROR* ' + msg

        #what if this sut is already locked by other user? We STOP
        _is_res_locked = self.is_resource_locked(res_query)
        if _is_res_locked and _is_res_locked != user_info[0]:
            msg = 'User {}: Reserve resource: The resource is locked for {} !'.format(user_info[0], _is_res_locked)
            logError(msg)
            return '*ERROR* ' + msg

        with self.acc_lock:
            parent_p = self.get_reserved_resource(res_query, props)

            if not parent_p:
                logError('Cannot access reserved SUT, path or ID `{}` !'.format(res_query))
                return False
            try:
                if isinstance(parent_p['path'], list):
                    # modify meta to the parent
                    if len(parent_p['path']) == 1:
                        child = parent_p
                    # modify to a component
                    else:
                        base_path = "/".join(parent_p['path'][1:])
                        child = self.get_path(base_path, parent_p)
                child['meta'][new_name] = child['meta'].pop(meta)
            except Exception:
                msg = "This meta that you entered thoes not exist {}".format(meta)
                logDebug(msg)
                return "false"

        return self.save_reserved_sut(res_query, props)


    @cherrypy.expose
    def delete_component_sut(self, res_query, props={}):
        """
        Permanently delete a component of a SUT or meta.
        It can be deleted only if SUT is reserved.
        """
        user_info = self.user_info(props)
        user = user_info[0]
        logFull('CeSuts:delete_component_sut {}'.format(res_query))

        if ':' in res_query:
            meta = res_query.split(':')[1]
            res_query = res_query.split(':')[0]
        else:
            meta = ''

        # Check if resource is reserved; if so, it cannot be deleted
        _is_res_reserved = self.is_resource_reserved(res_query, props)
        if _is_res_reserved and _is_res_reserved != user:
            msg = 'User {}: Cannot delete: The resource is reserved for {} !'.format(
                user, _is_res_reserved)
            logWarning(msg)
            return '*ERROR* ' + msg

        _is_res_locked = self.is_resource_locked(res_query)
        if _is_res_locked and _is_res_locked != user:
            msg = 'User {}: Cannot delete: The resource is locked for {} !'.format(
                user, _is_res_locked)
            logWarning(msg)
            return '*ERROR* ' + msg

        # The resource should be reserved
        parent_p = self.get_reserved_resource(res_query, props)

        if not parent_p:
            logWarning('User {}: Cannot access reserved SUT, path or ID `{}` !'.format(
                user, res_query))
            return False

        # Delete meta
        if meta:
            # we have to delete only the meta property
            correct_path = copy.deepcopy(parent_p['path'])

            # modify meta for parent
            if len(parent_p['path']) == 1:
                child = parent_p
            # modify meta for component
            else:
                base_path = "/".join(parent_p['path'][1:])
                child = self.get_path(base_path, parent_p)
            try:
                child['meta'].pop(meta)
            except Exception:
                msg = 'User {}: Property `{}` does not exist!'.format(user, meta)
                logWarning(msg)
                return 'false'
            child['path'] = correct_path

        # Delete component
        else:
            full_path = ''
            # the resources is deep in the tree, have to get its direct parent
            if len(parent_p['path']) > 2:
                full_path = copy.deepcopy(parent_p['path'])
                base_path = "/".join(parent_p['path'][1:-1])
                parent_p = self.get_path(base_path, parent_p)

            if not full_path:
                full_path = parent_p['path']
            parent_p['children'].pop(full_path[-1])
            parent_p['path'] = full_path[:-1]

        return 'true'


    @cherrypy.expose
    def delete_sut(self, res_query, props={}):
        """
        Permanently delete a SUT.
        Sut can be deteleted only if it is not reserved by anyone.
        """
        user_info = self.user_info(props)
        logFull('CeSuts:delete_sut {}'.format(res_query))

        # Check if resource is reserved; if so, it cannot be deleted
        _is_res_reserved = self.is_resource_reserved(res_query, props)
        if _is_res_reserved:
            msg = 'User {}: Cannot delete: The resource is reserved for {} !'\
            .format(user_info[0], _is_res_reserved)
            logError(msg)
            return '*ERROR* ' + msg

        _is_res_locked = self.is_resource_locked(res_query)
        if _is_res_locked:
            msg = 'User {}: Reserve resource: The resource is locked for {} !'\
            .format(user_info[0], _is_res_locked)
            logError(msg)
            return '*ERROR* ' + msg

        usr_home = userHome(user_info[0])

        # temporary fix; the SUT must be removed from self.resources
        def delete_sut_memory(sut_to_remove):
            parent_p = self.get_resource('/')

            if parent_p is not None and parent_p['children'].get(sut_to_remove) is not None:
                parent_p['children'].pop(sut_to_remove)
        # end temporary fix

        # SUT file can be user or system file
        if res_query.split('.')[-1] == 'system':
            suts_gb_path = self.project.get_user_info(user_info[0], 'sys_sut_path')
            if not suts_gb_path:
                suts_gb_path = '{}/config/sut/'.format(TWISTER_PATH)
            try:
                os.remove(suts_gb_path + res_query.split('.')[0] + '.json')
                delete_sut_memory(res_query.split('/')[-1])
                return "True"
            except Exception as exp_err:
                msg = 'User {}: Cannot delete SUT file: `{}` !'.format(user_info[0], res_query.split('.')[0] + '.json')
                logError(msg + " ERROR: " + exp_err)
                return '*ERROR* ' + msg
            return "True"
        else:
            usr_sut_path = self.project.get_user_info(user_info[0], 'sut_path')
            if not usr_sut_path:
                usr_sut_path = '{}/twister/config/sut/'.format(usr_home)
            delete_sut_memory(res_query.split('/')[-1])

            # delete from id_list if possible
            try:
                del self.id_list[res_query]
            except Exception:
                logDebug('User {}: id_list does not contain the sut: {}'.format(user_info[0], res_query))

            # get user SUT file; we have to check if the cleacase plugin
            # is activated; if so, use it to read the SUT files from view;
            # else use the UserService to read it
            cc_cfg = self.project.get_clearcase_config(user_info[0], 'sut_path')
            if cc_cfg:
                view = cc_cfg['view']
                actv = cc_cfg['actv']
                path = cc_cfg['path']
                user_view_actv = '{}:{}:{}'.format(user_info[0], view, actv)
                if path[-1] != '/' and res_query[-1] != '/':
                    complete_sut_path = path + '/' + res_query.split('.')[0] + '.json'
                else:
                    complete_sut_path = path + res_query.split('.')[0] + '.json'
                return self.project.clearFs.delete_user_file(user_view_actv, complete_sut_path)
            else:
                if usr_sut_path[-1] != '/' and res_query[-1] != '/':
                    complete_sut_path = usr_sut_path + '/' + res_query.split('.')[0] + '.json'
                else:
                    complete_sut_path = usr_sut_path + res_query.split('.')[0] + '.json'
                return self.project.localFs.delete_user_file(user_info[0], complete_sut_path)


    @cherrypy.expose
    def list_all_suts(self, user):
        """
        Fast list suts.
        """
        suts = []
        result = []
        usr_home = userHome(user)

        # System SUT path
        sys_sut_path = self.project.get_user_info(user, 'sys_sut_path')
        if not sys_sut_path:
            sys_sut_path = '{}/config/sut/'.format(TWISTER_PATH)

        # User SUT path
        usr_sut_path = self.project.get_user_info(user, 'sut_path')
        if not usr_sut_path:
            usr_sut_path = '{}/twister/config/sut/'.format(usr_home)

        # first, get all system SUT files
        if os.path.isdir(sys_sut_path):
            s_suts = ['{}.system'.format(os.path.splitext(d)[0]) \
            for d in os.listdir(sys_sut_path) \
            if os.path.splitext(d)[1] == '.json']
            suts.extend(s_suts)

        # get user SUT file; we have to check if the cleacase plugin
        # is activated; if so, use it to read the SUT files from view;
        # else use the UserService to read it
        cc_cfg = self.project.get_clearcase_config(user, 'sut_path')
        if cc_cfg:
            view = cc_cfg['view']
            actv = cc_cfg['actv']
            path = cc_cfg['path']
            user_view_actv = '{}:{}:{}'.format(user, view, actv)
            resp = self.project.clearFs.list_user_files(user_view_actv, path, False, False)
            if isinstance(resp, str):
                logWarning(resp)
                return '*ERROR* ' + resp
            for f_file in resp['children']:
                f_name, file_ext = os.path.splitext(f_file['path'])
                if file_ext and file_ext == '.json':
                    suts.append(f_name + '.user')
        else:
            if os.path.isdir(usr_sut_path):
                resp = self.project.localFs.list_user_files(user, usr_sut_path, False, False)
                if isinstance(resp, str):
                    logWarning(resp)
                for f_file in resp['children']:
                    f_name, file_ext = os.path.splitext(f_file['path'])
                    if file_ext and file_ext == '.json':
                        suts.append(f_name + '.user')

        def quick_find_path(dictionary, spath):
            """
            Find path.
            """
            for usr, locks in dictionary.iteritems():
                for id_sut, data in locks.iteritems():
                    path = data.get('path', [''])
                    if isinstance(path, str) or isinstance(path, unicode):
                        path = [path]
                    if path == [spath]:
                        return usr
            return False

        for s_sut in sorted(suts):
            ruser = quick_find_path(self.reservedResources, s_sut)
            luser = quick_find_path(self.lockedResources, s_sut)

            if (not ruser) and (not luser):
                result.append({'name': s_sut, 'status': 'free'})
            elif ruser:
                result.append({'name': s_sut, 'status': 'reserved', 'user': ruser})
            elif luser:
                result.append({'name': s_sut, 'status': 'locked', 'user': luser})
            # Both reserved and locked ?
            else:
                result.append({'name': s_sut, 'status': 'reserved', 'user': ruser})
        
        suts_on_disk = [s['name'] for s in result]

        # if the status of the CE is stopped for the current user
        # update the CE memory; check for consistency
        status = self.project.users[user]['status']
        if status == 0:
            suts_in_memory = self.resources['children'].keys()
            for sut in suts_in_memory:
                if sut not in suts_on_disk:
                    id = self.resources['children'][sut].get('id')
                    if user in self.reservedResources and id in self.reservedResources[user]:
                        logDebug('Removed sut `{}` from reserved resources.'.format(sut))
                        self.reservedResources[user].pop(id)
                    if user in self.lockedResources and id in self.lockedResources[user]:
                        logDebug('Removed sut `{}` from locked resources.'.format(sut))
                        self.lockedResources[user].pop(id)
                    logDebug('Removed sut `{}` from memory.'.format(sut))
                    self.resources['children'].pop(sut)

        logDebug('User {}: Fast listing SUTs... Found {}.'.format(user, suts))

        return result


    @cherrypy.expose
    def import_sut_xml(self, xml_file, sut_type='user', props={}):
        """
        Import one sut XML file.
        """
        user_info = self.user_info(props)
        user = user_info[0]

        logDebug('User {}: import XML file `{}`...'.format(user, xml_file))

        try:
            params_xml = etree.parse(xml_file)
        except Exception:
            msg = "The file you selected: '{}' it's not an xml file. Try again!".format(xml_file)
            logDebug(msg)
            return '*ERROR* ' + msg

        # parse the xml file and build the json format
        xml_ret = xml_to_res(params_xml, {})

        # build the filename to be saved; xml_file has absolute path; we need
        # to extract the last string after /, remove extension and add .json
        sut_file = xml_file.split('/')[-1].split('.')[0]
        sut_file = sut_file + '.json'

        sut_path = None
        if sut_type == 'system':
            # System SUT path
            sut_path = self.project.get_user_info(user, 'sys_sut_path')
            if not sut_path:
                sut_path = '{}/config/sut/'.format(TWISTER_PATH)
        else:
            # User SUT path
            sut_path = self.project.get_user_info(user, 'sut_path')
            if not sut_path:
                usr_home = userHome(user)
                sut_path = '{}/twister/config/sut/'.format(usr_home)
        sut_file = sut_path + '/' + sut_file

        resp = True
        if sut_type == 'system':
            resp = self.project.localFs.write_system_file(sut_file, json.dumps(xml_ret, indent=4), 'w')
        else:
            resp = self.project.localFs.write_user_file(user, sut_file, json.dumps(xml_ret, indent=4), 'w')

        return resp


    @cherrypy.expose
    def export_sut_xml(self, xml_file, query, props={}):
        """
        Export as XML file.
        """
        user_info = self.user_info(props)
        user = user_info[0]

        logDebug('User {}: export XML file `{}`, query = {}...'.format(user, xml_file, query))

        sut_path = None
        sut_type = query.split('.')[-1]
        if sut_type == 'system':
            # System SUT path
            sut_path = self.project.get_user_info(user, 'sys_sut_path')
            if not sut_path:
                sut_path = '{}/config/sut/'.format(TWISTER_PATH)
        else:
            # User SUT path
            sut_path = self.project.get_user_info(user, 'sut_path')
            if not sut_path:
                usr_home = userHome(user)
                sut_path = '{}/twister/config/sut/'.format(usr_home)

        sut_filename = sut_path + '/' + query.split('/')[1].split('.')[0] + '.json'
        logInfo('User {}: Export SUT file `{}` to `{}`.'.format(user, sut_filename, xml_file))

        # read the content of the user SUT file and load it in json
        if sut_type == 'system':
            resp = self.project.localFs.read_system_file(sut_filename, 'r')
        else:
            resp = self.project.localFs.read_user_file(user, sut_filename, 'r')

        try:
            json_resp = json.loads(resp)
        except Exception:
            msg = "User {}: Cannot load SUT file `{}`!".format(user, sut_filename)
            logWarning(msg)
            return '*ERROR* ' + msg

        # generate the xml structure
        xml = etree.Element('root')
        res_to_xml(json_resp, xml)

        # write the xml file
        xml_header = '<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n\n'
        resp = self.project.localFs.write_user_file(user, xml_file, xml_header, 'w')
        if resp != True:
            logError(resp)
            return resp

        resp = self.project.localFs.write_user_file(user, xml_file, etree.tostring(xml, pretty_print=True), 'w')
        if resp != True:
            logError(resp)
            return resp

        return True


    @cherrypy.expose
    def lock_sut(self, res_query, props={}):
        """
        Lock SUT. Add to lockedResources
        """
        return self.lock_resource(res_query, props)


    @cherrypy.expose
    def unlock_sut(self, res_query, props={}):
        """
        Unlock SUT. Delete from lockedResources.
        """
        return self.unlock_resource(res_query, props)


    @cherrypy.expose
    def is_sut_locked(self, res_query):
        """
        Verify if SUT is locked.
        """
        result = self.is_resource_locked(res_query)
        if not result:
            return "false"
        return result


    @cherrypy.expose
    def is_sut_reserved(self, res_query, props={}):
        """
        Verify if SUT is reserved.
        """
        result = self.is_resource_reserved(res_query, props)
        if not result:
            return "false"
        return result


    @cherrypy.expose
    def save_reserved_sut_as(self, name, res_query, props={}):
        """
        Save a reserved SUT as name+path.
        """
        user_info = self.user_info(props)
        user = user_info[0]
        logFull('CeSuts:save_reserved_sut_as {} {} {}'.format(name, res_query, user))
        path = '/'.join(name.split('/')[:-1])
        name = name.split('/')[-1]
        # add a '_' at the begining to be able to save as a new sut with the same name as the old one
        # the sut is deleted from memory after the json.dump
        target_name = '/_'+name+ '.' +res_query.split('.')[-1]#'.user'

        # verify if the sut exists
        usr_sut_list = self.list_all_suts(user)
        if res_query[0] == '/':
            res_query = res_query[1:]

        found_old_sut = [item for item in usr_sut_list if item['name'] == res_query]
        if not found_old_sut:
            msg = 'User {}: SUT file {} doesn\'t exist !'.format(user, res_query)
            logError(msg)
            return '*ERROR* ' + msg

        res_query = '/' + res_query
        # #verify if a SUT having the same name exists
        found_new_sut = [item for item in usr_sut_list if item['name'] == target_name]
        if found_new_sut:
            msg = 'User {}: New SUT file {} already exist !'.format(user, target_name)
            logError(msg)
            return '*ERROR* ' + msg

        # Check if resource is reserved; if so, it cannot be deleted
        _is_res_reserved = self.is_resource_reserved(res_query, props)
        if _is_res_reserved and _is_res_reserved != user:
            msg = 'User {}: Cannot save as: The resource is reserved for {} !'.\
            format(user, _is_res_reserved)
            logError(msg)
            return '*ERROR* ' + msg

        _is_res_locked = self.is_resource_locked(res_query)
        if _is_res_locked and _is_res_locked != user:
            msg = 'User {}: Reserve resource: The resource is locked for {} !'.\
            format(user, _is_res_locked)
            logError(msg)
            return '*ERROR* ' + msg

        # Create a new SUT having this new name
        new_sut_id = self.create_new_sut(target_name, '/', props, False)

        if isinstance(new_sut_id, str) and '*ERROR*' in new_sut_id:
            msg = 'User {}: New SUT file {} cannot be created!'.format(user, target_name)
            logError(msg)
            return '*ERROR* ' + msg

        # Reserve the target SUT
        self.reserve_resource(target_name, props)
        # Get the new path
        new_sut = self.reservedResources[user_info[0]][new_sut_id]
        # Get the old sut
        old_sut = self.get_reserved_resource(res_query, props)

        # Get all the structure of the old sut
        new_sut['meta'].update(copy.deepcopy(old_sut['meta']))
        # Copy children
        new_sut['children'].update(copy.deepcopy(old_sut['children']))

        # Recursive fix children IDs
        self.change_ids(new_sut)
        # Recursive fix children paths
        self.change_path(new_sut, new_sut['path'])

        # Exit now
        return self.save_reserved_sut(target_name, '{}', path)


    @cherrypy.expose
    def save_reserved_sut(self, res_query, props={}, path=''):
        """
        User has made some changes only on self.reserved_resources.
        In this method we sync self.reserved_resources with self.resources
        and the store on the disk
        """
        user_info = self.user_info(props)
        logFull('CeSuts:save_reserved_sut {} {}'.format(res_query, user_info[0]))

        resources = self.resources

        # If no resources...
        if not resources.get('children'):
            msg = 'User {}: Save reserved resource: There are no resources defined !'.format(user_info[0])
            logError(msg)
            return '*ERROR* ' + msg

        user_info = self.user_info(props)

        if ':' in res_query:
            res_query = res_query.split(':')[0]

        if not self.reservedResources.get(user_info[0]):
            msg = "It seems that this user does not have changes to save! {}".format(user_info[0])
            logDebug(msg)
            return '*ERROR* ' + msg

        resource_node = self.get_resource(res_query)
        if not resource_node or isinstance(resource_node, str):
            msg = 'User {}: Cannot find SUT {}'.format(user_info[0], res_query)
            logFull(msg)
            return '*ERROR* ' + msg

        if len(resource_node['path']) > 1:
            resource_node = self.get_path(resource_node['path'][0], resources)

        reserved_node = self.reservedResources[user_info[0]][resource_node['id']]

        # Maybe the user renamed the TB
        if reserved_node['path'][0] != resource_node['path'][0]:
            self.resources['children'][reserved_node['path'][0]] = \
            self.resources['children'].pop(resource_node['path'][0])
        # ...or maybe the name of the resource is the same
        resources['children'].update([(reserved_node['path'][0], reserved_node), ])

        # Update path
        resources['children'][reserved_node['path'][0]]['path'] = [reserved_node['path'][0]]
        # Now save
        if path == '':
            issaved = self.save_sut(props, resource_node['path'][0])
        else:
            issaved = self.save_sut_as(resource_node['path'][0], path)

        if isinstance(issaved, str):
            logDebug("We could not save this Sut for user = {}.".format(user_info[0]))
            return issaved
        return True


    @cherrypy.expose
    def save_release_reserved_sut(self, res_query, props={}):
        """
        Save the changes. Sync self.resources with self.reserved_resources
        and save to the disk
        """
        user_info = self.user_info(props)
        logFull('CeSuts:save_release_reserved_sut {} {} {}'.format(res_query, props, user_info[0]))

        result = self.save_reserved_sut(res_query, props)

        if result and not isinstance(result, str):
            if ':' in res_query:
                res_query = res_query.split(':')[0]

            # Having the id, we can discard and release directly
            if user_info[0] in self.reservedResources.keys():

                if res_query in self.reservedResources[user_info[0]]:
                    node_id = res_query
                else:

                    resource_node = self.get_resource(res_query)
                    if not resource_node or isinstance(resource_node, str):
                        logFull('User {} can not find the resource {}'.format(user_info[0], res_query))
                        return None

                    if len(resource_node['path']) > 1:
                        resource_node = self.get_path(resource_node['path'][0], self.resources)

                    node_id = resource_node['id']

                reserved_node = self.reservedResources[user_info[0]][node_id]
                self.reservedResources[user_info[0]].pop(reserved_node['id'])
                if not self.reservedResources[user_info[0]]:
                    self.reservedResources.pop(user_info[0])
            return True
        else:
            return result


    @cherrypy.expose
    def discard_release_reserved_sut(self, res_query, props={}):
        """
        Discard changes and release SUT.
        Delete entry from reservedResources.
        """
        return self.discard_release_reserved_resource(res_query, props)


    @cherrypy.expose
    def consistency_check_suts(self, user):
        """
        Makes a diff between the content of the self.resources dictionary and what is on the disk.
        If a sut is in dictionary and is not on the disk anymore, then it is deleted form the self.resources.
        If a new sut is on the disk, it is loaded in memory.
        """
        def get_disk_sut_names(suts_on_disk):
            sut_names = []
            for sut_d in suts_on_disk:
                sut_names.append(sut_d['name'])
            return sut_names
        
        # listing all suts
        result = self.list_all_suts(user)

        suts_on_disk = get_disk_sut_names(result)

        # if the status of the CE is stopped for the current user
        # update the CE memory
        status = self.project.users[user]['status']
        if status == 0:
            suts_in_memory = self.resources['children'].keys()
            for sut in suts_in_memory:
                if sut not in suts_on_disk:
                    id = self.resources['children'][sut].get('id')
                    if user in self.reservedResources and id in self.reservedResources[user]:
                        logDebug('Removed sut `{}` from reserved resources.'.format(sut))
                        self.reservedResources[user].pop(id)
                    else:
                        logDebug('Sut `{}` cannot be removed from the reserved resources.'.format(sut))
                        continue
                    if user in self.lockedResources and id in self.lockedResources[user]:
                        logDebug('Removed sut `{}` from locked resources.'.format(sut))
                        self.lockedResources[user].pop(id)
                    else:
                        logDebug('Sut `{}` cannot be removed from the locked resources.'.format(sut))
                        continue
                    logDebug('Removed sut `{}` from memory.'.format(sut))
                    self.resources['children'].pop(sut)

        return result class TestBeds(_cptools.XMLRPCController, CommonAllocator):
    """
    Basic operations for TestBeds.
    """

    def __init__(self, project):

        self.project = project
        self.type = 'tb'
        self.resources = CONSTANT_DICTIONARY
        self.reservedResources = {}
        self.lockedResources = {}
        self.acc_lock = thread.allocate_lock() # Task change lock
        self.ren_lock = thread.allocate_lock() # Rename lock
        self.imp_lock = thread.allocate_lock() # Import lock
        self.save_lock = thread.allocate_lock() # Save lock
        self.load_lock = thread.allocate_lock() # Save lock
        self.res_file = '{}/config/resources.json'.format(TWISTER_PATH)
        self._loaded_users = {}
        self.load_tb(verbose=True)


    def load_tb(self, verbose=False):
        """
        Parse resources file.
        """
        logDebug('CeTestBeds:load_tb {}'.format(verbose))

        with self.load_lock:
            if not self.resources.get('children'):
                self.resources = CONSTANT_DICTIONARY

            # try to load test bed resources file
            try:
                f_p = open(self.res_file, 'r')
                self.resources = json.load(f_p)
                f_p.close()
                del f_p
                if verbose:
                    logDebug('TBs loaded successfully.')
            except Exception as exp_err:
                logError('Error loading TBs! {}'.format(exp_err))

            is_res_modified = False
            # make older resources files that don't have 'path' compatible
            for res in self.resources.get('children'):
                self.resources['children'][res]['path'] = [res]
                modified = self.fix_path(self.resources['children'][res], [res])
                if modified:
                    is_res_modified = True

            # save the resources updated (with path field) for later usage
            if is_res_modified:
                issaved = self.save_tb()
                if isinstance(issaved, str):
                    if issaved.startswith('*ERROR* '):
                        msg = "We could not save this TB for user = {}.".format(user_info[0])
                        logDebug(msg)
                        return "*ERROR* " + msg

        return self.resources


    def save_tb(self, props={}):
        """
        Function used to write the changes on HDD.
        """
        logFull('CeTestBeds:_save {}'.format(props))
        log = []

        with self.save_lock:
            ver = self.resources.get('version', 0)
            self.resources['version'] = ver + 1
            try:
                logDebug('Saving test bed file.')
                with open(self.res_file, 'w') as f_p:
                    json.dump(self.resources, f_p, indent=4)
            except Exception as exp_err:
                log.append(exp_err)
        if log:
            logError(log)
            return '*ERROR* ' + str(log)

        return True


    @cherrypy.expose
    def get_tb(self, query, props={}):
        """
        Get the current version of the tb modified and unsaved or
        the version from the disk.
        """
        logDebug('CeTestBeds:get_tb {} {}'.format(query, props))
        user_info = self.user_info(props)

        result = None
        # If the resource is reserved, get the latest unsaved changes
        if user_info[0] in self.reservedResources:
            for i in range(len(self.reservedResources[user_info[0]].values())):
                result = self.get_resource(query, self.reservedResources[user_info[0]].values()[i])
                if isinstance(result, dict):
                    break
        # Or get it from the disk
        if not isinstance(result, dict):
            result = self.get_resource(query)

        if isinstance(result, dict):
            if ':' in query:
                meta = query.split(':')[1]
                ret = result['meta'].get(meta, '')
                return ret
            else:
                return self.format_resource(result, query)

        return "*ERROR* no such resource: {}".format(query)


    @cherrypy.expose
    def delete_tb(self, res_query, props={}):
        """
        Permanently delete a resource.
        """
        logDebug('Delete TB `{}`, props {}.'.format(res_query, props))

        resources = self.resources
        user_info = self.user_info(props)

        # If no resources...
        if not resources.get('children'):
            msg = 'User {}: There are no resources defined !'.format(user_info[0])
            logError(msg)
            return "*ERROR* " + msg

        if ':' in res_query:
            meta = res_query.split(':')[1]
            res_query = res_query.split(':')[0]
        else:
            meta = ''

        _is_res_reserved = self.is_resource_reserved(res_query, props)
        if _is_res_reserved and _is_res_reserved != user_info[0]:
            msg = 'User {}: The resource is reserved for {} !'.format(user_info[0], _is_res_reserved)
            logError(msg)
            return '*ERROR* ' + msg

        _is_res_locked = self.is_resource_locked(res_query)
        if _is_res_locked and _is_res_locked != user_info[0]:
            msg = 'User {}: The resource is locked for {} !'.format(user_info[0], _is_res_locked)
            logError(msg)
            return '*ERROR* ' + msg

        if meta:
            # the path for the resource that has to be modified (delete it or meta)
            result = self.get_reserved_resource(res_query, props)
            if not result:
                logDebug("This resource is not reserved")
                return False

            # we have to delete only the meta property
            correct_path = copy.deepcopy(result['path'])

            #modify meta for parent
            if len(result['path']) == 1:
                child = result
            # modify to a component
            else:
                base_path = '/'.join(result['path'][1:])
                child = self.get_path(base_path, result)
            try:
                child['meta'].pop(meta)
            except:
                msg = "This meta that you entered thoes not exist {}".format(meta)
                logDebug(msg)
                return "false"
            child['path'] = correct_path

            return "true"
        else:
            #we don't have meta, we delete the resource
            resource_node = self.get_resource(res_query)
            if isinstance(resource_node, dict) and ''.join(resource_node['path']) in self.resources['children']:
                # user wants to delete the entire TB
                if ''.join(resource_node['path']) in self.resources['children']:
                    self.resources['children'].pop(resource_node['path'][0])
                    issaved = self.save_tb(props)
                    if not issaved:
                        msg = "We could not save this TB: {}.".format(res_query)
                        logDebug(msg)
                        return "*ERROR* " + msg
            #user wants to delete a component of the TB
            else:
                reserved_node = self.get_reserved_resource(res_query, props)
                if not reserved_node:
                    logError('Cannot access reserved resource, path or ID `{}` !'.format(res_query))
                    return False
                #get the direct parent of the resource
                base_path = '/'.join(reserved_node['path'][1:-1])
                extract_r = self.get_path(base_path, reserved_node)
                try:
                    extract_r['children'].pop(reserved_node['path'][-1])
                except:
                    logError('User {}: Can not find child. Maybe it just was renamed.'.format(user_info[0]))
                    return False
                extract_r['path'] = reserved_node['path'][:-1]

            return "true"


    @cherrypy.expose
    def rename_tb(self, query, new_name, props={}):
        """
        Rename a resource.
        """
        logDebug('Rename TB `{}`, new name `{}`, props {}.'.format(query, new_name, props))

        user_info = self.user_info(props)

        if ':' in query:
            meta = query.split(':')[1]
            query = query.split(':')[0]
        else:
            meta = ''

        _is_res_reserved = self.is_resource_reserved(query, props)
        if _is_res_reserved and _is_res_reserved != user_info[0]:
            msg = 'User {}: The resource is reserved for {} !'.format(user_info[0], _is_res_reserved)
            logWarning(msg)
            return '*ERROR* ' + msg

        _is_res_locked = self.is_resource_locked(query)
        if _is_res_locked and _is_res_locked != user_info[0]:
            msg = 'User {}: The resource is locked for {} !'.format(user_info[0], _is_res_locked)
            logWarning(msg)
            return '*ERROR* ' + msg

        if '/' in new_name or ':' in new_name:
            logWarning('New resource name cannot contain `/` or `:`!')
            return False

        # If no resources...
        if not self.resources.get('children'):
            msg = 'There are no resources defined !'
            logError(msg)
            return '*ERROR* ' + msg

        # Correct node path
        result = self.get_reserved_resource(query, props)

        if not result:
            logWarning('Cannot access reserved TB `{}` !')
            return False

        if result['path'][-1] == new_name:
            logWarning('Nothing to rename for TB `{}` !'.format(new_name))
            return True

        with self.ren_lock:
            # If must rename a Meta info
            if meta:
                try:
                    # Modify meta to the parent
                    if len(result['path']) == 1:
                        child = result
                    # Modify to a component
                    else:
                        base_path = '/'.join(result['path'][1:])
                        child = self.get_path(base_path, result)

                    child['meta'][new_name] = child['meta'].pop(meta)
                except:
                    msg = 'Rename meta `{}` error, for TB `{}`!'.format(meta, result['path'][-1])
                    logWarning(msg)
                    return 'false'
                return self.save_reserved_tb(query, props)

            # If must rename a normal node
            else:
                # The parent is directly from the root and we want to rename its immediate children
                if len(result['path']) == 2:
                    result['children'][new_name] = result['children'].pop(result['path'][-1])
                # The component we want to rename is deep in the tree
                elif len(result['path']) > 2:
                    base_path = '/'.join(result['path'][1:-1])
                    parent = self.get_path(base_path, result)
                    if not isinstance(parent, dict):
                        msg = msg = 'Rename error for TB `{}`, invalid parent \
                        on {}!'.format(result['path'][-1], result['id'])
                        logWarning(msg)
                        return 'false'
                    parent['children'][new_name] = parent['children'].pop(result['path'][-1])
                else:
                    result['path'] = [new_name]
                # Only have to change the current path and the path of the children
                result['path'] = [result['path'][0]]
                # Recursive update paths
                self.change_path(result, result['path'])

        return True


    @cherrypy.expose
    def create_component_tb(self, name, parent=None, props={}):
        """
        Create a component for an existing TB.
        Return new component's id.
        """
        user_info = self.user_info(props)

        props = self.valid_props(props)
        if parent == '/' or parent == '1':
            msg = "The parent value is not an existing TB. Maybe you want to add a new TB. Parent: {}".format(parent)
            logError(msg)
            return "*ERROR* " + msg

        _is_res_reserved = self.is_resource_reserved(parent, props)
        if _is_res_reserved and _is_res_reserved != user_info[0]:
            msg = 'User {}: The resource is reserved for {} !'.format(user_info[0], _is_res_reserved)
            logError(msg)
            return '*ERROR* ' + msg

        _is_res_locked = self.is_resource_locked(parent)
        if _is_res_locked and _is_res_locked != user_info[0]:
            msg = 'User {}: The resource is locked for {} !'.format(user_info[0], _is_res_locked)
            logError(msg)
            return '*ERROR* ' + msg

        with self.acc_lock:
            #the resource should be reserved previously
            parent_p = self.get_reserved_resource(parent, props)
            if not parent_p:
                msg = "User {}: Could not find this TB: '{}'".format(user_info[0], parent)
                logDebug(msg)
                return "*ERROR* " + msg

            #the resources is deep in the tree, we have to get its direct parent
            if len(parent_p['path']) >= 2:
                full_path = parent_p['path']
                base_path = '/'.join(parent_p['path'][1:])
                parent_p = self.get_path(base_path, parent_p)
                parent_p['path'] = full_path

            if '/' in name:
                logDebug('Stripping slash characters from `{}`...'.format(name))
                name = name.replace('/', '')

            if name in parent_p['children']:
                msg = "A component with this name '{}' already exists for this TB: '{}'".format(name, parent)
                logDebug(msg)
                return "*ERROR* " + msg

            # the resource doesn't exist - create it
            res_id = self.generate_index()
            parent_p['children'][name] = {'id': res_id, 'meta': props, 'children': {}, \
            'path': parent_p['path'] + [name]}

            return res_id


    @cherrypy.expose
    def create_new_tb(self, name, parent=None, props={}):
        """
        Create new test bed.
        Return the id of the new created tb.
        """

        user_info = self.user_info(props)
        resources = self.resources

        if parent != '/' and parent != '1':
            msg = "The parent value is not root. Maybe you want to add a component\
             to an existing SUT. Parent: {}".format(parent)
            logError(msg)
            return "*ERROR* " + msg

        props = self.valid_props(props)

        with self.acc_lock:
            # root can not be reserved so we just take it
            parent_p = self.get_resource('/', resources)

            if not parent_p or isinstance(parent_p, str):
                logFull("User: {} no result for query `{}`" .format(user_info[0], parent))
                return None

            if '/' in name:
                logDebug('Stripping slash characters from `{}`...'.format(name))
                name = name.replace('/', '')

            if name in self.resources['children']:
                msg = "User {}: A TB with name `{}` already exists!".format(user_info[0], name)
                logDebug(msg)
                return "*ERROR* " + msg

            # the resource doesn't exist - create it
            res_id = self.generate_index()
            parent_p['children'][name] = {'id': res_id, 'meta': props, 'children': {}, 'path': [name]}

            issaved = self.save_tb(props)
            if not issaved:
                msg = "User {}: Could not save TB `{}`".format(user_info[0], name)
                logDebug(msg)
                return "*ERROR* " + msg

            return res_id


    @cherrypy.expose
    def update_meta_tb(self, name, parent=None, props={}):
        """
        Modify a resource, using a name, a parent Path or ID and some properties.
        """

        logDebug('parent = {} -- props = {} -- name = {}'.format(parent, props, name))
        user_info = self.user_info(props)
        resources = self.resources

        props = self.valid_props(props)
        if not props or not self.valid_props(props):
            msg = "Wrong format for props = {}".format(props)
            logDebug(msg)
            return "*ERROR* " + msg

        if parent == '/' or parent == "1":
            #we can not reserve the root so we just take the TB we need
            if name[0] != '/':
                name = '/' + name
            verify_reserved = name
        else:
            #take the TB that has the component we need
            verify_reserved = parent

        _is_res_reserved = self.is_resource_reserved(verify_reserved, props)
        if _is_res_reserved and _is_res_reserved != user_info[0]:
            msg = 'User {}: The resource is reserved for {} !'.format(user_info[0], _is_res_reserved)
            logError(msg)
            return '*ERROR* ' + msg

        _is_res_locked = self.is_resource_locked(verify_reserved)
        if _is_res_locked and _is_res_locked != user_info[0]:
            msg = 'User {}: Reserve resource: The resource is locked for {} !'.format(user_info[0], _is_res_locked)
            logError(msg)
            return '*ERROR* ' + msg

        with self.acc_lock:

            l_props = dict(props)
            if '__user' in l_props:
                del l_props['__user']

            # If this is the root resource, update the properties
            if name == '/' and parent == '/':
                resources['meta'].update(l_props)
                # Write changes for Device or SUT
                issaved = self.save_tb(props)
                if not issaved:
                    msg = "User {}: We didnt save this entry = {} having props = {}".format(user_info[0], name, props)
                    logDebug(msg)
                    return "*ERROR* " + msg
                return "true"

            parent_p = self.get_reserved_resource(verify_reserved, props)

            if not parent_p:
                logError('User {}: Cannot access reserved resource `{}` !'.format(user_info[0], verify_reserved))
                return False

            #the resources is deep in the tree, we have to get its direct parent
            if len(parent_p['path']) >= 2:
                full_path = parent_p['path']
                base_path = '/'.join(parent_p['path'][1:])
                parent_p = self.get_path(base_path, parent_p)
                parent_p['path'] = full_path

            if '/' in name:
                logDebug('User {}: Stripping slash characters from `{}`...'.format(user_info[0], name))
                name = name.replace('/', '')

            # the resource exists
            if name in parent_p['children']:
                child_p = parent_p['children'][name]
            elif name in parent_p['path']:
                child_p = parent_p
            else:
                return "User {}: *ERROR* the resource {} can not be found!".format(user_info[0], name)

            # We have to update the props
            child_p['meta'].update(l_props)

            return "true"


    @cherrypy.expose
    def set_tb(self, name, parent=None, props={}):
        """
        Higher level wrapper over functions Create new TB, create component and update meta.
        """

        pdata = self.get_resource(parent)
        user_info = self.user_info(props)

        if not isinstance(pdata, dict):
            logWarning('User `{}`: No such parent `{}`!'.format(user_info[0], parent))
            return False

        if (parent == '/' or parent == '1') and name not in pdata['children']:
            return self.create_new_tb(name, parent, props)

        # If exists, update meta
        if name in pdata['children']:
            return self.update_meta_tb(name, parent, props)
        # This is a new component
        else:
            return self.create_component_tb(name, parent, props)


    @cherrypy.expose
    def reserve_tb(self, res_query, props={}):
        """
        Reserve TB. Add to reservedResources.
        """

        return self.reserve_resource(res_query, props)


    @cherrypy.expose
    def lock_tb(self, res_query, props={}):
        """
        Lock TB. Add to lockedResources
        """

        return self.lock_resource(res_query, props)


    @cherrypy.expose
    def unlock_tb(self, res_query, props={}):
        """
        Unlock TB. Delete from lockedResources.
        """

        return self.unlock_resource(res_query, props)


    @cherrypy.expose
    def is_tb_locked(self, res_query):
        """
        Verify if TB is locked.
        """

        result = self.is_resource_locked(res_query)
        if not result:
            return "false"
        return result


    @cherrypy.expose
    def is_tb_reserved(self, res_query, props={}):
        """
        Verify if TB is reserved.
        """

        result = self.is_resource_reserved(res_query, props)
        if not result:
            return "false"
        return result


    @cherrypy.expose
    def list_reserved_tb(self):
        """
        Return all the reserved TBs.
        """

        return self.reservedResources


    @cherrypy.expose
    def list_locked_tb(self):
        """
        Return all the locked TBs.
        """

        return self.lockedResources


    @cherrypy.expose
    def list_all_tbs(self):
        """
        Fast list testbeds.
        """

        #maybe some resources changed meanwhile
        self.load_tb(verbose=False)

        res = []
        for k_val, v_val in self.resources.get('children').iteritems():
            res.append([k_val, v_val['id']])
        result = []

        def quick_find_path(dictionary, spath):
            """
            Find path.
            """

            for usr, locks in dictionary.iteritems():
                for id_tb, data in locks.iteritems():
                    path = data.get('path', [''])
                    if isinstance(path, str) or isinstance(path, unicode):
                        path = [path]
                    if path == [spath]:
                        return usr
            return None

        for tb_name, tb_id in sorted(res):
            ruser = quick_find_path(self.reservedResources, tb_name)
            luser = quick_find_path(self.lockedResources, tb_name)

            if (not ruser) and (not luser):
                result.append({'id': tb_id, 'name': tb_name, 'status': 'free'})
            elif ruser:
                result.append({'id': tb_id, 'name': tb_name, 'status': 'reserved', 'user': ruser})
            elif luser:
                result.append({'id': tb_id, 'name': tb_name, 'status': 'locked', 'user': luser})
            # Both reserved and locked ?
            else:
                result.append({'id': tb_id, 'name': tb_name, 'status': 'reserved', 'user': ruser})

        logDebug('Fast listing Resources... Found {}.'.format(res))

        return result


    @cherrypy.expose
    def export_tb_xml(self, xml_file, props={}):
        """
        Export as XML file.
        """
        user_info = self.user_info(props)

        logDebug('User {}: exporting to XML file `{}`'.format(user_info[0], xml_file))

        try:
            f_p = open(xml_file, 'w')
        except:
            msg = 'User {}: export XML: XML file `{}` cannot be written !'.format(user_info[0], xml_file)
            logError(msg)
            return '*ERROR* ' + msg

        logDebug('User {}: exporting to XML file `{}`...'.format(user_info[0], xml_file))

        xml = etree.Element('root')
        res_to_xml(self.resources, xml, False)
        f_p.write('<?xml version="1.0" encoding="UTF-8" standalone="no"?>\n\n')
        f_p.write(etree.tostring(xml, pretty_print=True))
        f_p.close()

        return True


    @cherrypy.expose
    def import_tb_xml(self, xml_file, props={}):
        """
        Import one XML file.
        WARNING! This erases everything!
        """
        user_info = self.user_info(props)
        self.resources = CONSTANT_DICTIONARY

        logDebug('{} -- XML file `{}` !'.format(user_info[0], xml_file))
        if not os.path.isfile(xml_file):
            msg = 'User {} import XML: XML file `{}` does not exist!'.format(user_info[0], xml_file)
            logError(msg)
            return '*ERROR* ' + msg

        logDebug('User {}: importing XML file `{}`...'.format(user_info[0], xml_file))
        try:
            params_xml = etree.parse(xml_file)
        except:
            msg = "The file you selected: '{}' it's not an xml file. Try again!".format(xml_file)
            logDebug(msg)
            return '*ERROR* ' + msg

        with self.imp_lock:
            try:
                self.resources = xml_to_res(params_xml, {})
            except Exception as exp_err:
                msg = 'User {}: Import XML: Exception `{}`.'.format(user_info[0], exp_err)
                logError(msg)
                return '*ERROR* ' + msg
        # Write changes for Device or SUT
        issaved = self.save_tb(props)
        if isinstance(issaved, str):
            logDebug("We could not save this TB.")
            return False

        return True


    @cherrypy.expose
    def save_reserved_tb(self, res_query, props={}):
        """
        User has made some changes only on self.reserved_resources.
        In this method we sync self.reserved_resources with self.resources
        and the store on the disk
        """

        logDebug('CeTestBeds:save_reserved_tb {}'.format(res_query))

        user_info = self.user_info(props)
        resources = self.resources

        # If no resources...
        if not resources.get('children'):
            msg = 'User {}: Save reserved resource: There are no resources defined !'.format(user_info[0])
            logError(msg)
            return '*ERROR* ' + msg

        user_info = self.user_info(props)

        if ':' in res_query:
            res_query = res_query.split(':')[0]

        if not self.reservedResources.get(user_info[0]):
            msg = "User {}: It seems that this user does not have changes to save!".format(user_info[0])
            logError(msg)
            return "*ERROR* " + msg

        reserved_resources = copy.deepcopy(self.reservedResources.get(user_info[0]))


        resource_node = self.get_resource(res_query)
        if not resource_node or isinstance(resource_node, str):
            msg = "User {}: Can not find this resource = {}".format(user_info[0], res_query)
            logError(msg)
            return "*ERROR* " + msg

        if len(resource_node['path']) > 1:
            resource_node = self.get_path(resource_node['path'][0], resources)

        if not resource_node:
            msg = "User {}: We didnt find this resource {}".format(user_info[0], res_query)
            logDebug(msg)
            return "*ERROR* " + msg

        reserved_node = reserved_resources[resource_node['id']]

        # maybe the user renamed the TB
        if reserved_node != resource_node:
            self.resources['children'][reserved_node['path'][0]] = \
            self.resources['children'].pop(resource_node['path'][0])
        # or maybe the name of the resource is the same
        resources['children'].update([(reserved_node['path'][0], reserved_node), ])

        # update path
        resources['children'][reserved_node['path'][0]]['path'] = [reserved_node['path'][0]]

        #now we have to save
        issaved = self.save_tb(props)
        if isinstance(issaved, str):
            if issaved.startswith('*ERROR* '):
                msg = "We could not save this TB for user = {}.".format(user_info[0])
                logDebug(msg)
                return "*ERROR* " + msg
        return "true"


    @cherrypy.expose
    def save_release_reserved_tb(self, res_query, props={}):
        """
        Save the changes. Sync self.resources with self.reserved_resources
        and save to the disk
        """
        logDebug('CeTestBeds:save_release_reserved_tb {} {}'.format(res_query, props))

        # save changes
        result = self.save_reserved_tb(res_query, props)

        if result and not result.startswith("*ERROR*"):
            user_info = self.user_info(props)

            if ':' in res_query:
                res_query = res_query.split(':')[0]

            # get only the component
            resource_node = self.get_resource(res_query)
            if not resource_node or isinstance(resource_node, str):
                logFull("Can not find the resoruce {}".format(res_query))
                return None

            # get the entire TB
            if len(resource_node['path']) > 1:
                resource_node = self.get_path(resource_node['path'][0], self.resources)

            # delete this entry from reservedResources
            reserved_node = self.reservedResources[user_info[0]][resource_node['id']]
            self.reservedResources[user_info[0]].pop(reserved_node['id'])
            if not self.reservedResources[user_info[0]]:
                self.reservedResources.pop(user_info[0])
        else:
            return result

        return True


    @cherrypy.expose
    def discard_release_reserved_tb(self, res_query, props={}):
        """
        Discard changes and release test bed.
        Delete entry from reservedResources.
        """
        return self.discard_release_reserved_resource(res_query, props) class LocalFS(BaseFS, FsBorg):
    """
    All local file operations should be done via THIS class.
    This is a singleton.
    """

    def __init__(self):
        FsBorg.__init__(self)
        self.name = 'Local'
        if os.getuid():
            logError('{} FS: Central Engine must run as ROOT in order to start the User Service!'.format(self.name))
        logInfo('Created {} FS.'.format(self.name))


    def _usr_service(self, user, oper='read'):
        """
        Launch a user service.
        """
        if oper not in ['read', 'write']:
            logWarning('Invalid FS operation `{}`, for user `{}`! Will reset to "read".'.format(oper, user))
            oper = 'read'

        # Must block here, so more users cannot launch Logs at the same time and lose the PID
        with self._srv_lock:

            # Try to re-use the logger server, if available
            conn = self._services.get(user, {}).get('conn_' + oper, None)
            if conn:
                try:
                    conn.ping(data='Hello', timeout=30.0)
                    # logDebug('Reuse old {} User Service connection for `{}` OK.'.format(op, user))
                    return conn
                except Exception as exp_err:
                    logWarning('Cannot reuse {} User Service for `{}`: `{}`.'.format(oper, user, exp_err))
                    self._kill(user)
            else:
                logInfo('Launching a User Service for `{}`, the first time...'.format(user))

            port = None

            # If the server is not available, search for a free port in the safe range...
            while 1:
                port = random.randrange(63000, 65000)
                try:
                    socket.create_connection((None, port), 1)
                except Exception:
                    break

            p_cmd = 'su {} -c "{} -u {}/server/UserService.py {} {}"'.\
            format(user, sys.executable, TWISTER_PATH, port, self.name)
            proc = subprocess.Popen(p_cmd, cwd='{}/twister'.\
            format(userHome(user)), shell=True, close_fds=True,\
            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            proc.poll()
            time.sleep(2.0)

            config = {
                'allow_pickle': True,
                'allow_getattr': True,
                'allow_setattr': True,
                'allow_delattr': True
            }

            retry = 10
            delay = 0.5
            success = False

            while retry > 0:
                if success:
                    break

                try:
                    stream_r = rpyc.SocketStream.connect('127.0.0.1', port, timeout=5.0)
                    conn_read = rpyc.connect_stream(stream_r, config=config)
                    conn_read.root.hello()
                    logDebug('Connected to User Service for `{}`, operation `read`.'.format(user))
                    success = True
                except Exception as exp_err:
                    logWarning('Cannot connect to User Service for `{}` - \
                    Exception: `{}`! Wait {}s...'.format(user, exp_err, delay))

                if success:
                    try:
                        stream_w = rpyc.SocketStream.connect('127.0.0.1', port, timeout=5.0)
                        conn_write = rpyc.connect_stream(stream_w, config=config)
                        conn_write.root.hello()
                        logDebug('Connected to User Service for `{}`, operation `write`.'.format(user))
                        break
                    except Exception as exp_err:
                        logWarning('Cannot connect to User Service for `{}` \
                        - Exception: `{}`! Wait {}s...'.\
                        format(user, exp_err, delay))
                        success = False

                time.sleep(delay)
                retry -= 1
                delay += 0.75

            if not success:
                logError('Error on starting User Service for `{}`!'.format(user))
                return None

            # Save the process inside the block.  99% of the time, this block is executed instantly!
            self._services[user] = {'proc': proc, 'conn_read': conn_read, 'conn_write': conn_write, 'port': port}

        logDebug('User Service for `{}` launched on `127.0.0.1:{}` - PID `{}`.'.format(user, port, proc.pid))

        return self._services[user].get('conn_' + oper, None) class ClearCaseFs(BaseFS, CcBorg):
    """
    All ClearCase file operations should be done via THIS class.
    This is a singleton.
    """

    def __init__(self):
        CcBorg.__init__(self)
        self.name = 'ClearCase'
        if os.getuid():
            logError('{} FS: Central Engine must run as ROOT in order to start the User Service!'.format(self.name))
        logInfo('Created {} FS.'.format(self.name))


    def _usr_service(self, user_view_actv, op='read'):
        """
        Launch a user service.
        Open a ClearCase view first.
        """
        if user_view_actv.count(':') == 1 and user_view_actv[-1] != ':':
            user_view_actv += ':'
        try:
            user, view, actv = user_view_actv.split(':')
        except Exception:
            # We don't really know the user in here !
            msg = 'Invalid ClearCase user-view-activity parameter: `{}`!'.format(user_view_actv)
            logWarning(msg)
            return '*ERROR* ' + msg

        if op not in ['read', 'write']:
            logWarning('Invalid CC operation `{}`, for user `{}`! Will reset to "read".'.format(op, user))
            op = 'read'

        view = view.strip()
        actv = actv.strip()
        user_view = user + ':' + view

        if not view:
            # We don't know the view in here !
            msg = 'Empty view in `{}`!'.format(user_view_actv)
            logWarning(msg)
            return '*ERROR* ' + msg

        def pread():
            """ Read proc stdout """
            while 1:
                try:
                    line = proc.readline().strip()
                    if not line:
                        continue
                    plog.append(line)
                except:
                    break

        # Must block here, so more users cannot launch Logs at the same time and lose the PID
        with self._srv_lock:

            # Try to re-use the FS, if available
            conn = self._services.get(user_view, {}).get('conn_' + op, None)
            if conn:
                try:
                    conn.ping(data='Hello', timeout=30.0)
                    # logDebug('Reuse old {} ClearCase Service connection for `{}` OK.'.format(op, user_view))
                    proc = self._services.get(user_view, {}).get('proc', None)
                    old_actv = self._services.get(user_view, {}).get('actv', None)
                    if actv != old_actv:
                        logInfo('Changing activity to `{}`, for `{}`.'.format(actv, user_view))
                        # Set cc activity again !
                        proc.sendline('cleartool setactivity {}'.format(actv))
                        time.sleep(1.0)
                        pread()
                        self._services.get(user_view, {})['actv'] = actv
                    return conn
                except Exception as exp_err:
                    logWarning('Cannot connect to `{}` ClearCase Service \
                    for `{}`: `{}`.'.format(op, user_view, exp_err))
                    self._kill(user)
                    proc = self._services.get(user_view, {}).get('proc', None)
                    PID = proc.pid
                    proc.terminate()
                    logInfo('Terminated CC User Service `{}` for user `{}`.'.\
                    format(PID, user))
            else:
                logInfo('Launching a ClearCase Service for `{}`, the first \
                time...'.format(user_view))

            proc = pexpect.spawn(['bash'], timeout=2.5, maxread=2048)
            time.sleep(1.0)
            plog = []

            proc.sendline('su {}'.format(user))
            time.sleep(1.0)
            pread()
            # User's home folder
            proc.sendline('cd ~/twister')
            pread()
            # Set cc view only the first time !
            proc.sendline('cleartool setview {}'.format(view))
            time.sleep(2.0)
            pread()
            # Empty line after set view
            proc.sendline('')
            pread()

            if actv:
                # Set cc activity for the first time !
                proc.sendline('cleartool setactivity {}'.format(actv))
                time.sleep(1.0)
                pread()

            port = None

            # If the server is not available, search for a free port in the safe range...
            while 1:
                port = random.randrange(63000, 65000)
                try:
                    socket.create_connection((None, port), 1)
                except Exception:
                    break

            # Launching 1 UserService inside the SSH terminal, with ClearCase View open
            p_cmd = '{} -u {}/server/UserService.py {} {} & '.format(sys.executable, TWISTER_PATH, port, self.name)
            proc.sendline(p_cmd)
            time.sleep(2.0)
            pread()

            # Empty line after proc start
            proc.sendline('')
            pread()

            logDebug('ClearCase startup log \n:: -------\n{}\n:: -------'.format('\n'.join(plog)))

            config = {
                'allow_pickle': True,
                'allow_getattr': True,
                'allow_setattr': True,
                'allow_delattr': True
            }

            retry = 10
            delay = 0.5
            success = False

            while retry > 0:
                if success:
                    break

                try:
                    stream_r = rpyc.SocketStream.connect('127.0.0.1', port, timeout=5.0)
                    conn_read = rpyc.connect_stream(stream_r, config=config)
                    conn_read.root.hello()
                    logDebug('Connected to ClearCase Service for `{}`, operation `read`.'.format(user_view))
                    success = True
                except Exception as exp_err:
                    logWarning('Cannot connect to ClearCase Service for `{}`!'\
                        'Exception: `{}`! Retry...'.format(user_view, exp_err))

                if success:
                    try:
                        stream_w = rpyc.SocketStream.connect('127.0.0.1', port, timeout=5.0)
                        conn_write = rpyc.connect_stream(stream_w, config=config)
                        conn_write.root.hello()
                        logDebug('Connected to ClearCase Service for `{}`, operation `write`.'.format(user_view))
                        break
                    except Exception as exp_err:
                        logWarning('Cannot connect to ClearCase Service \
                        for `{}`! Exception: `{}`! Retry...'.\
                        format(user_view, exp_err))
                        success = False

                time.sleep(delay)
                retry -= 1
                delay += 0.75

            if not success:
                logError('Error on starting ClearCase Service for `{}`!'.format(user_view))
                return None

            # Save the process inside the block.
            self._services[user_view] = {'proc': proc, 'port': port,\
            'conn_read': conn_read, 'conn_write': conn_write, 'actv': actv}

        logDebug('ClearCase Service for `{}` launched on `127.0.0.1:{}`.'.format(user_view, port))

        return self._services[user_view].get('conn_' + op, None)


    def system_command(self, user_view, cmd):
        """ Execute a system command """
        logDebug('System command {} {}'.format(user_view, cmd))

        # if the view string ends with : , it means the activity is not set
        # and we have to strip the last :
        if user_view.endswith(':'):
            user_view = user_view.rstrip(':')

        # make sure the CC service is started
        self._usr_service(user_view)

        proc = self._services.get(user_view, {}).get('proc')
        if proc:
            # Empty buffer
            plog = []
            while 1:
                try:
                    line = proc.readline().strip()
                    if not line:
                        continue
                    plog.append(line)
                except:
                    break
            # Send command
            proc.sendline(cmd)
            time.sleep(1)
            plog = []
            # Catch buffer
            while 1:
                try:
                    line = proc.readline().strip()
                    if not line:
                        continue
                    plog.append(line)
                except:
                    break
            return '\n'.join(plog)
        else:
            return False class CeConfigs(object):
    """
    Central Engine config manager class.
    """

    project = None # Pointer to Project instance
    glb_lock = allocate_lock() # Global variables lock
    cfg_lock = allocate_lock() # Config access lock
    config_locks = {} # Config locks list


    def __init__(self, project):
        """
        Initialize config manager.
        """
        self.project = project


    def _parse_common(self, xml, gparams):
        """
        Recursive parse globals / configs.
        """
        for folder in xml.xpath('folder'):
            tmp = {gparam.find('name').text: gparam.find('value').text or '' \
            for gparam in folder.xpath('param')}
            tmp.update( self._parse_common(folder, tmp))
            gparams[folder.find('fname').text] = tmp
        return gparams


    def _parse_globals(self, user):
        """
        Returns a dictionary containing all global parameters.
        """
        path_glob = self.project.get_user_info(user, 'glob_params')
        resp = self.project.localFs.read_user_file(user, path_glob)

        if resp.startswith('*ERROR*'):
            logWarning(resp)
            return {}

        gparams = self._parse_common(etree.fromstring(resp), {})
        return gparams


    def _parse_configs(self, user, globs_file):
        """
        Returns a dictionary containing all config parameters.
        """
        resp = self.read_config_file(user, globs_file)

        if resp.startswith('*ERROR*'):
            logWarning(resp)
            return {}

        gparams = self._parse_common(etree.fromstring(resp), {})
        return gparams


    def _find_global_variable(self, user, node_path, globs_file=False):
        """
        Helper function.
        """
        if not globs_file:
            var_pointer = self.project.users[user]['global_params']
        else:
            var_pointer = self._parse_configs(user, globs_file)
        if not var_pointer:
            return False

        for node in node_path:
            if node in var_pointer:
                var_pointer = var_pointer[node]
            else:
                # Invalid variable path
                return False

        return var_pointer


    def get_global_variable(self, user, variable, globs_file=False):
        """
        Sending a global variable, using a path.
        """
        r = self.project.authenticate(user)
        if not r:
            return False

        try:
            node_path = [v for v in variable.split('/') if v]
        except Exception:
            logWarning('Global Variable: Invalid variable type `{}`, for user `{}`!'.format(variable, user))
            return False

        var_pointer = self._find_global_variable(user, node_path, globs_file)

        if not var_pointer:
            node_path = '/'.join(node_path)
            if globs_file:
                logWarning('Global Variable: Invalid variable path `{}` in \
                file `{}`, for user `{}`!'.format(node_path, globs_file, user))
            else:
                logWarning('Global Variable: Invalid variable path `{}`, for user `{}`!'.format(node_path, user))
            return False

        return var_pointer


    def set_global_variable(self, user, variable, value):
        """
        Set a global variable path, for a user.\n
        The change is not persistent.
        """
        r = self.project.authenticate(user)
        if not r:
            return False

        try:
            node_path = [v for v in variable.split('/') if v]
        except Exception:
            logWarning('Global Variable: Invalid variable type `{}`, for user `{}`!'.format(variable, user))
            return False

        if (not value) or (not str(value)):
            logWarning('Global Variable: Invalid value `{}`, for global variable `{}` from user `{}`!'\
                ''.format(value, variable, user))
            return False

        # If the path is in ROOT, it's a root variable
        if len(node_path) == 1:
            with self.glb_lock:
                self.project.users[user]['global_params'][node_path[0]] = value
            return True

        # If the path is more complex, the pointer here will go to the parent
        var_pointer = self._find_global_variable(user, node_path[:-1])

        if not var_pointer:
            logWarning('Global Variable: Invalid variable path `{}`, for user `{}`!'.format(node_path, user))
            return False

        with self.glb_lock:
            var_pointer[node_path[-1]] = value

        logDebug('Global Variable: Set variable `{} = {}`, for user `{}`!'.format(value, variable, user))
        return True


    def is_lock_config(self, user, fpath):
        """
        Complete path from tree - returns True/ False
        """
        if fpath in self.config_locks:
            logDebug('Config file `{}` is locked by `{}`.'.format(fpath, user))
        else:
            logDebug('Config file `{}` is not locked.'.format(fpath))
        return self.config_locks.get(fpath, False)


    def lock_config(self, user, fpath):
        """
        Complete path from tree - returns True/ False
        """
        # If already locked, return False
        if fpath in self.config_locks:
            err = '*ERROR* Config file `{}` is already locked by `{}`! \
            Cannot lock!'.format(fpath, self.config_locks[fpath])
            logDebug(err)
            return err
        with self.cfg_lock:
            self.config_locks[fpath] = user
            logDebug('User `{}` is locking config file `{}`.'.format(user, fpath))
            return True


    def unlock_config(self, user, fpath):
        """
        Complete path from tree - returns True/ False
        """
        # If not locked, return False
        if fpath not in self.config_locks:
            err = '*ERROR* Config file `{}` is not locked'.format(fpath)
            logDebug(err)
            return err
        # If not locked by this user, return False
        if self.config_locks[fpath] != user:
            err = '*ERROR* Config file `{}` is locked by `{}`! Cannot unlock!'.format(fpath, self.config_locks[fpath])
            logDebug(err)
            return err
        with self.cfg_lock:
            del self.config_locks[fpath]
            logDebug('User `{}` is releasing config file `{}`.'.format(user, fpath))
            return True


    def read_config_file(self, user, fpath):
        """
        Read a config file.
        """
        # Auto detect if ClearCase Test Config Path is active
        ccConfig = self.project.get_clearcase_config(user, 'tcfg_path')
        if ccConfig:
            view = ccConfig['view']
            actv = ccConfig['actv']
            path = ccConfig['path'].rstrip('/')
            if not path:
                return '*ERROR* You did not set ClearCase Config Path!'
            user_view_actv = '{}:{}:{}'.format(user, view, actv)
            return self.project.clearFs.read_user_file(user_view_actv, path +'/'+ fpath)
        else:
            dirpath = self.project.get_user_info(user, 'tcfg_path')
            return self.project.localFs.read_user_file(user, dirpath + '/' + fpath)


    def save_config_file(self, user, fpath, content):
        """
        Save config file - returns a True/ False.
        """
        lock = self.config_locks.get(fpath, False)
        if lock and lock != user:
            err = '*ERROR* Config file `{}` is locked by `{}`! Cannot save!'.format(fpath, lock)
            logWarning(err)
            return err

        logDebug('Update config file `{}`, for user `{}`.'.format(fpath, user))

        # Auto detect if ClearCase Test Config Path is active
        ccConfig = self.project.get_clearcase_config(user, 'tcfg_path')
        if ccConfig:
            view = ccConfig['view']
            actv = ccConfig['actv']
            path = ccConfig['path'].rstrip('/')
            if not path:
                return '*ERROR* You did not set ClearCase Config Path!'
            user_view_actv = '{}:{}:{}'.format(user, view, actv)
            # If file exists
            fsz = self.project.clearFs.file_size(user_view_actv, path + '/' + fpath)
            # Cannot save an existing file that isn't locked!
            if isinstance(fsz, long) and not lock:
                err = '*ERROR* Cannot save CC config file `{}`, because it\'s not locked!'.format(fpath)
                logWarning(err)
                return err
            return self.project.write_file(user, path + '/' + fpath, content,\
            type='clearcase:{}:{}'.format(view, actv))
        else:
            dirpath = self.project.get_user_info(user, 'tcfg_path')
            # If file exists
            fsz = self.project.localFs.file_size(user, dirpath + '/' + fpath)
            # Cannot save an existing file that isn't locked!
            if isinstance(fsz, long) and not lock:
                err = '*ERROR* Cannot save config file `{}`, because it\'s not locked!'.format(fpath)
                logWarning(err)
                return err
            return self.project.write_file(user, dirpath + '/' + fpath, content)


    def delete_config_file(self, user, fpath):
        """
        Delete config file - returns a True/ False.
        """
        lock = self.config_locks.get(fpath, False)
        # Cannot Delete a locked file!
        if lock:
            err = '*ERROR* Config file `{}` is locked by `{}`! Cannot delete!'.format(fpath, lock)
            logWarning(err)
            return err

        logDebug('Delete config file `{}`, for user `{}`.'.format(fpath, user))

        # Unbind the config file, if it was binded
        self.project.parsers[user].del_binding(fpath)

        # Auto detect if ClearCase Test Config Path is active
        ccConfig = self.project.get_clearcase_config(user, 'tcfg_path')
        if ccConfig:
            view = ccConfig['view']
            actv = ccConfig['actv']
            path = ccConfig['path'].rstrip('/')
            if not path:
                return '*ERROR* You did not set ClearCase Config Path!'
            user_view_actv = '{}:{}:{}'.format(user, view, actv)
            return self.project.clearFs.delete_user_file(user_view_actv, path + '/' + fpath)
        else:
            dirpath = self.project.get_user_info(user, 'tcfg_path')
            return self.project.localFs.delete_user_file(user, dirpath + '/' + fpath) class CeDbManager(object):
    """
    Central Engine database manager class.
    """

    project = None   # Pointer to Project instance
    connections = {} # Saved connections
    db_lock = allocate_lock() # Global database lock


    def __init__(self, project):
        """
        Initialize database manager.
        """
        self.project = project


    def connect_db(self, user, db_server='', db_name='', db_user='', db_passwd='', shared_db=False):
        """
        Connect to database.
        Must know the IP + Database, the user and if the DB is shared or not,
        to be able to decrypt the password.
        """
        # Get the path to DB.XML
        db_file = self.project.get_user_info(user, 'db_config')
        if not db_file:
            logError('Database: Null DB.XML file for user `{}`! Cannot connect!'.format(user))
            return False

        c_time = time.time()
        usr_server = '{}_{}_{}'.format(user, db_server, db_name)

        # Existing connection ?
        if self.connections.get(usr_server):
            # This is a very fresh connection !
            if c_time - self.connections[usr_server]['dt'] < 1.0:
                return self.connections[usr_server]['conn']

        # Use shared DB or not ?
        use_shared_db = self.project.get_user_info(user, 'use_shared_db')
        if use_shared_db and use_shared_db.lower() in ['true', 'yes']:
            use_shared_db = True
        else:
            use_shared_db = False

        # DB.xml + Shared DB parser
        users_groups = self.project._parse_users_and_groups()
        shared_db_path = users_groups['shared_db_cfg']
        db_config = DBParser(user, db_file, shared_db_path, use_shared_db).db_config

        # Try to use the default pair
        if not db_server and not db_name:
            db_server, db_name, db_user, db_passwd, _ = db_config['default_server']

        # Need to magically identify the correct key; the first pair is from private DB.xml;
        if shared_db:
            # Shared DB is True
            encr_key = users_groups.get('shared_db_key', 'Luxoft')
        else:
            # Fallback to shared DB
            encr_key = None

        # Decode database password
        db_password = self.project.decrypt_text(user, db_passwd, encr_key)

        logDebug('User `{}` connecting to {} MySQL `{} @ {} / {} : {}[...]`.'.\
        format(user, 'shared' if encr_key else 'user', db_user, db_server,\
        db_name, db_passwd[:3]))

        try:
            conn = MySQLdb.connect(host=db_server,
                                   db=db_name,
                                   user=db_user,
                                   passwd=db_password,
                                   charset='utf8',
                                   use_unicode=True)
            conn.autocommit = False
        except MySQLdb.Error as exp_err:
            logError('MySQL error for user `{}`: `{} - {}`!'.\
            format(user, exp_err.args[0], exp_err.args[1]))
            return False

        # Keep connection
        self.connections[usr_server] = {'conn': conn, 'dt': c_time}

        return conn


    def static_project_data(self, user):
        """
        Collect all files data, for a user.
        """
        # Central Engine variables ...
        system = platform.machine() + ' ' + platform.system() + ', ' + \
             ' '.join(platform.linux_distribution())

        # Timezone. Read from etc/timezone 1st, etc/sysconfig/clock 2nd, or from Time module
        try:
            timezone = open('/etc/timezone').read().strip()
        except Exception:
            try:
                timezone = subprocess.\
                check_output('cat /etc/sysconfig/clock | grep ^ZONE=',\
                shell=True)
                timezone = timezone.strip().replace('"', '').replace('ZONE=', '')
            except Exception:
                timezone = time.strftime('%Z')

        ce_host = socket.gethostname()
        try:
            ce_ip = socket.gethostbyname(ce_host)
        except Exception:
            ce_ip = ''

        # Default substitute dict
        default_subst = copy.deepcopy(DATABASE_EXPORT_VARS)
        default_subst['twister_user'] = user
        default_subst['twister_ce_type'] = self.project.server_init['ce_server_type'].lower()
        default_subst['twister_server_location'] = self.project.server_init.get('ce_server_location', '')

        default_subst['twister_rf_fname'] = '{}/config/resources.json'.format(TWISTER_PATH)
        default_subst['twister_pf_fname'] = self.project.users[user].get('proj_xml_name', '')

        default_subst['twister_ce_os'] = system
        default_subst['twister_ce_timezone'] = timezone
        default_subst['twister_ce_hostname'] = ce_host
        default_subst['twister_ce_ip'] = ce_ip
        default_subst['twister_ce_python_revision'] = '.'.join([str(v) for v in sys.version_info])

        def fix_log(tc_log):
            tc_log = tc_log.replace('\n', '<br>\n')
            tc_log = MySQLdb.escape_string(tc_log)
            tc_log = tc_log.replace('<div', '&lt;div')
            tc_log = tc_log.replace('</div', '&lt;/div')
            return tc_log

        # Pre-calculated data
        all_data = []

        for epname, ep_info in self.project.users[user]['eps'].iteritems():
            # If the suites key is not found continue to the next
            if ep_info.get('suites') is None:
                continue

            suites_manager = ep_info['suites']

            for file_id in suites_manager.get_files():

                # Default substitute data
                subst_data = dict(default_subst)
                # Add EP info
                subst_data.update(ep_info)
                del subst_data['suites']

                # Add Suite info
                file_info = suites_manager.find_id(file_id)
                if 'custom_vars' in file_info:
                    custom_vars = file_info.pop('custom_vars')
                    for var in custom_vars:
                        file_info[var] = custom_vars[var]
                suite_id = file_info['suite']
                suite_info = suites_manager.find_id(suite_id)

                # This is the root suite data
                root_suite = dict(suite_info)

                while 1:
                    root_suite_id = root_suite['suite']
                    root_suite = suites_manager.find_id(root_suite_id)
                    if not root_suite:
                        root_suite = {}
                        break
                    if not root_suite['suite']:
                        break

                # Add root suite data
                subst_data.update(root_suite)
                # Add current suite data
                subst_data.update(suite_info)
                del subst_data['children']

                # Add file info
                subst_data.update(file_info)

                # Insert/ fix DB variables
                subst_data['twister_ep_name'] = epname
                subst_data['twister_suite_id'] = suite_id
                subst_data['twister_suite_name'] = suite_info['name']
                subst_data['twister_tc_reason'] = file_info.get('_reason', '')
                subst_data['twister_tc_iteration'] = file_info.get('iterationNr', '')
                subst_data['twister_tc_full_path'] = file_info['file']
                subst_data['twister_tc_name'] = os.path.split(file_info['file'])[1]
                subst_data['twister_tc_id'] = file_id

                # Delete obsolete keys
                try:
                    del subst_data['name']
                except Exception:
                    pass
                try:
                    del subst_data['status']
                except Exception:
                    pass
                try:
                    del subst_data['type']
                except Exception:
                    pass
                try:
                    del subst_data['pd']
                except Exception:
                    pass

                # Log CLI for this EP - Suite - Test
                try:
                    tc_log = self.project.find_log(user, ltype='logCli',\
                    epname=epname, file_id=file_id, file_name=file_info['file'])
                    subst_data['twister_tc_log'] = fix_log(tc_log)
                except Exception:
                    subst_data['twister_tc_log'] = '*no log*'

                # The rest of the logs
                try:
                    tc_log = self.project.find_log(user, ltype='logRunning',\
                    epname=epname, file_id=file_id, file_name=file_info['file'])
                    subst_data['twister_tc_log_running'] = fix_log(tc_log)
                except Exception:
                    subst_data['twister_tc_log_running'] = '*no log*'

                try:
                    tc_log = self.project.find_log(user, ltype='logDebug',\
                    epname=epname, file_id=file_id, file_name=file_info['file'])
                    subst_data['twister_tc_log_debug'] = fix_log(tc_log)
                except Exception:
                    subst_data['twister_tc_log_debug'] = '*no log*'

                try:
                    tc_log = self.project.find_log(user, ltype='logTest', \
                    epname=epname, file_id=file_id, file_name=file_info['file'])
                    subst_data['twister_tc_log_test'] = fix_log(tc_log)
                except Exception:
                    subst_data['twister_tc_log_test'] = '*no log*'

                # :: Debug ::
                # import pprint ; pprint.pprint(subst_data)

                # Append all data for current file
                all_data.append(subst_data)

        all_data = sorted(all_data, key=lambda file_info: file_info['twister_tc_id'])

        return all_data


    def project_data(self, user, save_to_db=False):
        """
        Collect all data from a user, using the DB.XML for the current project.
        If save to DB is active, the function will also save.
        """
        # Get the path to DB.XML
        db_file = self.project.get_user_info(user, 'db_config')
        if not db_file:
            logError('Database: Null DB.XML file for user `{}`! Nothing to do!'.format(user))
            return False

        # DB.xml + Shared DB parser
        users_groups = self.project._parse_users_and_groups()
        shared_db_path = users_groups['shared_db_cfg']
        db_cfg_role = 'CHANGE_DB_CFG' in users_groups['users'][user]['roles']

        # Use shared DB or not ?
        use_shared_db = self.project.get_user_info(user, 'use_shared_db')
        if use_shared_db and use_shared_db.lower() in ['true', 'yes']:
            use_shared_db = True
        else:
            use_shared_db = False

        dbp = DBParser(user, db_file, shared_db_path, use_shared_db)
        all_inserts = dbp.get_inserts(db_cfg_role)
        del dbp

        if not all_inserts:
            logWarning('Database: Cannot use inserts defined for user `{}`!'.\
            format(user))
            return False

        # UserScript cache
        usr_script_cache_s = {} # Suite
        usr_script_cache_p = {} # Project

        # DbSelect cache
        db_select_cache_s = {} # Suite
        db_select_cache_p = {} # Project

        conn, curs = None, None

        # Pre-calculated data
        all_data = []

        for subst_data in self.static_project_data(user):

            # Setup and Teardown files will not be saved to database!
            if subst_data.get('setup_file') == 'true' or \
            subst_data.get('teardown_file') == 'true':
                logDebug("Ignoring `{}` file, because it's setup or teardown.".\
                format(subst_data['twister_tc_name']))
                continue
            # Pre-Suite or Post-Suite files will not be saved to database
            if subst_data.get('Pre-Suite') or subst_data.get('Post-Suite'):
                continue

            # If file has iterators, the iteration save==Failed and the status was Failed
            if subst_data.get('_cfg_files') and subst_data.get('iterationNr') \
            and subst_data.get('iterationSave') == 'failed' and \
            subst_data['twister_tc_status'] == 'PASS':
                continue

            # For every host, build correct data...
            for host_db in all_inserts:

                c_inserts = all_inserts[host_db]['inserts']
                c_fields = all_inserts[host_db]['fields']
                shared_db = all_inserts[host_db]['shared_db']

                db_server, db_name, db_user, db_passwd, _ = host_db
                conn = self.connect_db(user, db_server, db_name, db_user, db_passwd, shared_db=shared_db)
                if not conn:
                    continue
                curs = conn.cursor()

                # Escape all unicodes variables before SQL Statements!
                subst_data = {k: MySQLdb.escape_string(v) if isinstance(v, unicode) else v for \
                    k, v in subst_data.iteritems()}

                # For every query of the current host
                for query in c_inserts:

                    # All variables of type `UserScript` must be replaced
                    # with the script result
                    try:
                        user_script_fields = re.findall('(\$.+?)[,\.\'"\s]', query)
                    except Exception:
                        user_script_fields = []

                    for field in user_script_fields:
                        field = field[1:]

                        # Invalid field ?
                        if field not in c_fields:
                            continue

                        # If the field is not `UserScript`, ignore it
                        if c_fields.get(field, {}).get('type') != 'UserScript':
                            continue

                        # Field level: Suite or Project
                        lvl = c_fields.get(field)['level']

                        # Get Script Path, or null string
                        u_script = subst_data.get(field, '')

                        if not u_script:
                            query = query.replace('$'+field, '')
                            continue

                        # Execute this script based on level
                        if lvl == 'Project':
                            if u_script not in usr_script_cache_p:
                                # Execute script and use result
                                res = execScript(u_script)
                                # logDebug('Database: UserScript for `{}` was executed at '\
                                #     'LVL `{}`.'.format(user, lvl))
                                # Save result in cache
                                usr_script_cache_p[u_script] = res
                            else:
                                # Get script result from cache
                                res = usr_script_cache_p[u_script]
                        # Execute for every suite
                        else:
                            suite_id = subst_data['twister_suite_id']
                            if suite_id not in usr_script_cache_s:
                                usr_script_cache_s[suite_id] = {}
                            if u_script not in usr_script_cache_s[suite_id]:
                                # Execute script and use result
                                res = execScript(u_script)
                                # logDebug('Database: UserScript for `{}` was executed at '\
                                #     'LVL `{}`.'.format(user, lvl))
                                # Save result in cache
                                usr_script_cache_s[suite_id][u_script] = res
                            else:
                                # Get script result from cache
                                res = usr_script_cache_s[suite_id][u_script]

                        # Replace UserScript with with real Script results
                        if not res:
                            res = ''
                        query = query.replace('$'+field, res)

                        # Adding user script fields
                        subst_data[field] = res

                    # All variables of type `DbSelect` must be replaced with the SQL result
                    try:
                        auto_insert_fields = re.findall('(@.+?@)', query)
                    except Exception:
                        auto_insert_fields = []

                    for field in auto_insert_fields:
                        # Delete the @ character
                        field = field[1:-1]

                        # Invalid field ?
                        if field not in c_fields:
                            continue

                        # Get Auto Query, or null string
                        u_query = c_fields.get(field, {}).get('query', '')

                        # Field level: Suite, Project, or Testcase
                        lvl = c_fields.get(field)['level']

                        if not u_query:
                            logError('User `{}`, file `{}`: Cannot build query! Field `{}` '\
                                'is not defined in the fields section!'.format(user, subst_data['file'], field))
                            return False

                        # Execute User Query based on level
                        if lvl == 'Project':
                            if u_query not in db_select_cache_p:
                                # Execute User Query
                                curs.execute(u_query)
                                q_value = curs.fetchone()[0]
                                # logDebug('Database: DbSelect for `{}` was executed at '\
                                #     'LVL `{}`.'.format(user, lvl))
                                # Save result in cache
                                db_select_cache_p[u_query] = q_value
                            else:
                                # Get script result from cache
                                q_value = db_select_cache_p[u_query]
                        # Execute User Query for every suite
                        elif lvl == 'Suite':
                            suite_id = subst_data['twister_suite_id']
                            if suite_id not in db_select_cache_s:
                                db_select_cache_s[suite_id] = {}
                            if u_query not in db_select_cache_s[suite_id]:
                                # Execute User Query
                                curs.execute(u_query)
                                q_value = curs.fetchone()[0]
                                # logDebug('Database: DbSelect for `{}` was executed at '\
                                #     'LVL `{}`.'.format(user, lvl))
                                # Save result in cache
                                db_select_cache_s[suite_id][u_query] = q_value
                            else:
                                # Get script result from cache
                                q_value = db_select_cache_s[suite_id][u_query]
                        else:
                            # Execute User Query
                            curs.execute(u_query)
                            q_value = curs.fetchone()[0]
                            # logDebug('Database: DbSelect for `{}` was executed at '\
                            #     'LVL `TestCase`.'.format(user))

                        # Replace @variables@ with real Database values
                        query = query.replace('@'+field+'@', str(q_value))
                        # Adding auto insert fields
                        subst_data[field] = str(q_value)

                    # String Template
                    tmpl = Template(query)

                    # Fix None values to NULL
                    for k, v in subst_data.iteritems():
                        if k == 'twister_tc_log':
                            try:
                                subst_data[k] = unicode(v, errors='ignore')
                            except Exception as err:
                                subst_data[k] = v
                                logDebug('ERROR converting to unicode: {} | {}'.format(err, v))
                        else:
                            subst_data[k] = 'NULL' if v is None else v
                    logDebug('---- SUBST_DATA: {}'.format(subst_data))
                    # Build complete query
                    try:
                        query = tmpl.substitute(subst_data)
                    except Exception as exp_err:
                        logError('User `{}`, file `{}`: Cannot build query! \
                        Error on `{}`!'.format(user, subst_data['file'], exp_err))
                        return False

                    # Save query in database ?
                    if save_to_db:
                        # Execute MySQL Query!
                        try:
                            curs.execute(query)
                            logDebug('Executed query\n\t``{}``\n\t on {} OK.'.format(query.strip(), host_db))
                            conn.commit()
                        except MySQLdb.Error as exp_err:
                            logError('Error in query ``{}`` , for user \
                            `{}`!\n\t MySQL Error {}: {}!'.\
                            format(query, user, exp_err.args[0],\
                            exp_err.args[1]))
                            conn.rollback()
                            return False

            # :: Debug ::
            # import pprint ; pprint.pprint(subst_data)

            # Append all data for current file
            all_data.append(subst_data)

        return all_data


    def save_to_database(self, user):
        """
        Save all data from a user: Ep, Suite, File, into database,
        using the DB.XML files for the current project.
        """
        with self.db_lock:
            return self.project_data(user, True)
import traceback
import os
import sys
import re
import time
import datetime
import copy
import subprocess
import socket
import smtplib
import binascii
import threading
import paramiko
import cherrypy
    import simplejson as json
    import json
from collections import OrderedDict
from thread import allocate_lock
from thread import start_new_thread
from string import Template
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from common.constants  import STATUS_WORKING, STATUS_PENDING, testStatus
from common.constants  import STATUS_NOT_EXEC, STATUS_STOP, STATUS_INVALID
from common.constants  import STATUS_INTERACT, STATUS_PAUSED, STATUS_RUNNING
from common.constants  import STATUS_RESUME, EXEC_STATUS, ROLES
from common.helpers    import userHome, execScript, decrypt, encrypt
from common.tsclogging import getLogLevel
from common.tsclogging import logDebug, logFull, logError, logWarning
from common.tsclogging import logInfo, logCritical
from common.xmlparser  import PluginParser, ClearCaseParser, TSCParser
from common.suitesmanager import SuitesManager
from common import iniparser
from server.CeParser import CeXmlParser
from server.CeServices import ServiceManager
from server.CeWebUi import WebInterface
from server.CeReports import ReportingServer
from server.CeSuts import Suts
from server.CeTestBeds import TestBeds
from server.CeFs import LocalFS
from server.CeClearCaseFs import ClearCaseFs
from server.CeConfigs import CeConfigs
from server.CeDatabase import CeDbManager
                for file_id in files:
                    current_status = self.get_file_info(user, epname, file_id).get('status', -1)
                    # Change the files with PENDING status, to NOT_EXEC
                    if current_status in [STATUS_PENDING, -1]:
                        self.set_file_info(user, epname, file_id, 'status', STATUS_NOT_EXEC)
                        statuses_changed += 1

            if statuses_changed:
                logDebug('User `{}` changed `{}` file statuses from \
                "Pending" to "Not executed".'.format(user, statuses_changed))

        # Pause or other statuses ?
        else:
            # Change status on all project and real EPs !
            project_eps = self.parsers[user].getActiveEps()
            for epname in project_eps:
                if epname not in real_eps:
                    continue
                # Set the NEW EP status
                self.set_ep_info(user, epname, 'status', new_status)

        # All active EPs for this project, refresh after all settings...
        project_eps = self.parsers[user].getActiveEps()
        intersect_eps = sorted(set(real_eps) & set(project_eps))

        # Update status for User
        self.set_user_info(user, 'status', new_status)

        if msg and msg != ',':
            logInfo('Status changed for `{} {}` - {}.\n\tMessage: `{}`.'.format(
                user, intersect_eps, reversed[new_status], msg))
        else:
            logInfo('Status changed for `{} {}` - {}.'.format(
                user, intersect_eps, reversed[new_status]))

        return reversed[new_status]


    def get_file_status_all(self, user, epname=None, suite_id=None):
        """
        Return the status of all files, in order.
        This can be filtered for an EP and a Suite.
        """
        logFull('CeProject:get_file_status_all user `{}`.'.format(user))
        res = self.authenticate(user)
        if not res:
            return []

        if suite_id and not epname:
            logError('Must provide both EP and Suite, user `{}`!'.format(user))
            return []

        statuses = {} # Unordered
        final = []    # Ordered

        # There are no EPs registered yet !
        if user not in self.test_ids:
            return final

        # Lock resource
        with self.stt_lock:
            eps = self.users[user].get('eps')

            if not eps:
                return []

            if epname:
                suites_p = eps[epname].get('suites')
                if not suites_p:
                    files = []
                elif suite_id:
                    files = suites_p.get_files(suite_id=suite_id, recursive=True)
                else:
                    files = suites_p.get_files(suite_id=None, recursive=True)

                for file_id in files:
                    f_status = self.get_file_info(user, epname, file_id)
                    if f_status:
                        f_status = f_status.get('status', 10)
                    else:
                        f_status = 10
                    # Unordered
                    statuses[file_id] = str(f_status)
            # Default case, no EP and no Suite
            else:
                for epname in eps:
                    if not 'suites' in eps[epname]:
                        continue
                    files = eps[epname]['suites'].get_files(suite_id=None, recursive=True)
                    for file_id in files:
                        f_status = self.get_file_info(user, epname, file_id)
                        if f_status:
                            f_status = f_status.get('status', 10)
                        else:
                            f_status = 10
                        # Unordered
                        statuses[file_id] = str(f_status)

        for tcid in sorted(statuses.keys()):
            final.append(statuses[tcid])

        return final


    def set_file_status(self, user, epname, file_id, new_status=10, time_elapsed=0.0):
        """
        Set status for one file and write in log summary.
        """
        logFull('CeProject:set_file_status user `{}`.'.format(user))
        res = self.authenticate(user)
        if not res:
            return False
        eps = self.users[user]['eps']

        if epname not in eps:
            logDebug('Project: Invalid EP name `{}` !'.format(epname))
            return False
        if new_status not in testStatus.values():
            logError('Project: Status value `{}` is not in the list of \
            defined statuses: `{}`!' ''.format(new_status,\

            sorted(testStatus.values())))