class HPCStatsSourceError(HPCStatsException):

    """Class for source errors exceptions in HPCStats"""

    def __init__(self, msg):

        super(HPCStatsSourceError, self).__init__(msg) class Cluster(object):
    """Model class for Cluster table"""

    def __init__(self, name, cluster_id=None):

        self.cluster_id = cluster_id
        self.name = name
    
    def __str__(self):

        return self.name

    def __eq__(self, other):

        return self.name == other.name

    def find(self, db):
        """Search the Cluster in the database based on its name. If exactly
           one cluster matches in database, set cluster_id attribute properly
           and returns its value. If more than one cluster matches, raises
           HPCStatsDBIntegrityError. If no cluster is found, returns None.
        """

        req = """
                SELECT cluster_id
                  FROM Cluster
                 WHERE cluster_name = %s
              """
        params = ( self.name, )
        db.execute(req, params)
        nb_rows = db.cur.rowcount
        if nb_rows == 0:
            logger.debug("cluster %s not found in DB", str(self))
            return None
        elif nb_rows > 1:
            raise HPCStatsDBIntegrityError(
                    "several cluster_id found in DB for cluster %s" \
                      % (str(self)))
        else:
            self.cluster_id = db.cur.fetchone()[0]
            logger.debug("cluster %s found in DB with id %d",
                         str(self),
                         self.cluster_id )
            return self.cluster_id

    def save(self, db):
        """Insert Cluster in database. You must make sure that the Cluster does
           not already exist in database yet (typically using Cluster.find()
           method else there is a risk of future integrity errors because of
           duplicated clusters. If cluster_id attribute is set, it raises
           HPCStatsRuntimeError.
        """

        if self.cluster_id is not None:
            raise HPCStatsRuntimeError(
                    "could not insert cluster %s since already existing in "\
                    "database" \
                      % (str(self)))

        req = """
                INSERT INTO Cluster ( cluster_name )
                VALUES ( %s )
                RETURNING cluster_id
              """
        params = ( self.name, )

        #print db.cur.mogrify(req, params)
        db.execute(req, params)
        self.cluster_id = db.cur.fetchone()[0]

    def get_nb_cpus(self, db):
        """Returns the total number of CPUs available on the cluster"""

        if self.cluster_id is None:
            raise HPCStatsRuntimeError(
                    "could not search for data with cluster %s since not " \
                    "found in database" \
                      % (str(self)))

        req = """
                SELECT SUM(node_nbCpu)
                  FROM Node
                 WHERE cluster_id = %s
              """
        params = ( self.cluster_id, )

        #print db.cur.mogrify(req, params)
        db.execute(req, params)
        return db.cur.fetchone()[0]

    def get_min_datetime(self, db):
        """Returns the start datetime of the oldest started and unfinished
           job on the cluster.
        """

        if self.cluster_id is None:
            raise HPCStatsRuntimeError(
                    "could not search for data with cluster %s since not " \
                    "found in database" \
                      % (str(self)))

        req = """
                SELECT MIN(job_start)
                  FROM Job
                 WHERE cluster_id = %s
                   AND job_state NOT IN ('CANCELLED', 'NODE_FAIL', 'PENDING')
              """
        params = ( self.cluster_id, )

        #print db.cur.mogrify(req, params)
        db.execute(req, params)
        return db.cur.fetchone()[0]

   
    def get_nb_accounts(self, db, creation_date):
        """Returns the total of users on the cluster whose account have been
           created defore date given in parameter.
        """

        if self.cluster_id is None:
            raise HPCStatsRuntimeError(
                    "could not search for data with cluster %s since not " \
                    "found in database" \
                      % (str(self)))

        req = """
                SELECT COUNT (userhpc_id)
                  FROM Userhpc,
                       Account
                 WHERE Account.userhpc_id = Userhpc.userhpc_id
                   AND Account.account_creation < %s
                   AND Account.cluster_id = %s
              """
        params = (creation_date, self.cluster_id )

        #print db.cur.mogrify(req, params)
        db.execute(req, params)
        
        return db.cur.fetchone()[0]

    def get_nb_active_users(self, db, start, end):
        """Returns the total number of users who have run job(s) on the cluster
           between start and end datetimes in parameters.
        """

        if self.cluster_id is None:
            raise HPCStatsRuntimeError(
                    "could not search for data with cluster %s since not " \
                    "found in database" \
                      % (str(self)))

        req = """
                SELECT COUNT(DISTINCT userhpc_id)
                  FROM Job
                 WHERE Job.cluster_id = %s,
                   AND ((job_start BETWEEN %s AND %s)
                        OR (job_end BETWEEN %s AND %s)
                        OR (job_start <= %s AND job_end >= %s))
              """
        params = (self.cluster_id, start, end, start, end, start, end)
        #print db.cur.mogrify(req, params)
        db.execute(req, params)
        
        return db.cur.fetchone()[0] class UserImporterLdap(UserImporter):

    """This UserImporter is a bit tricky since it also has to calculate start
       and end datetime of users' Accounts. For this purpose, the load() method
       must first get all user accounts that are currently defined w/o end
       datetime in database to find out if they have disapeared or not in LDAP
       directory. If yes, update the account end datetime with todays date.
    """
    def __init__(self, app, db, config, cluster):

        super(UserImporterLdap, self).__init__(app, db, config, cluster)

        ldap_section = self.cluster.name + '/ldap'

        self._ldapurl = config.get(ldap_section, 'url')
        self._ldapbase = config.get(ldap_section, 'basedn')
        self._ldapdn = config.get(ldap_section, 'dn')
        self._ldaphash = config.get(ldap_section, 'phash')
        self.ldap_password = \
          base64.b64decode(decypher(base64.b64decode(self._ldaphash)))
        self._ldapcert = config.get_default(ldap_section, 'cert', None)

        self._ldapgroups = []
        group_list = config.get_default(ldap_section, 'groups', None)
        if group_list is not None:
            self._ldapgroups = group_list.split(',')
        single_ldapgroup = config.get_default(ldap_section, 'group', None)
        if single_ldapgroup is not None:
            self.log.warn(Errors.E_U0005,
                          "Deprecated config option,"
                          "'group' is replaced by 'groups'")
            self._ldapgroups += [single_ldapgroup]

        self.ldap_rdn_people = config.get_default(ldap_section,
                                                  'rdn_people',
                                                  'ou=people')
        self.ldap_rdn_groups = config.get_default(ldap_section,
                                                  'rdn_groups',
                                                  'ou=groups')

        self.ldap_dn_people = self.ldap_rdn_people + ',' + self._ldapbase
        self.ldap_dn_groups = self.ldap_rdn_groups + ',' + self._ldapbase
        self.group_dpt_search = config.get(ldap_section, 'group_dpt_search')
        self.group_dpt_regexp = config.get(ldap_section, 'group_dpt_regexp')
        self.default_subdir = config.get_default(ldap_section,
                                                 'default_subdir',
                                                 'unknown')

        self.strict_user_membership = config.get_default( \
                                               'constraints',
                                               'strict_user_membership',
                                               True,
                                               bool)

        self.groups_alias_file = config.get_default(ldap_section,
                                                    'groups_alias_file',
                                                    None)
        self.groups_alias = {} # hash of aliases

        self.users_acct_ldap = None
        self.users_acct_db = None

        self.ldap_conn = None

    def connect_ldap(self):
        """Connect to LDAP directory and set ldap_conn attribute accordingly.
           Raises HPCStatsSourceError in case of error.
        """

        if self._ldapcert is not None:
            ldap.set_option(ldap.OPT_X_TLS_CACERTFILE, self._ldapcert)
        try:
            self.ldap_conn = ldap.initialize(self._ldapurl)
            self.ldap_conn.simple_bind(self._ldapdn, self.ldap_password)
        except ldap.SERVER_DOWN, err:
            raise HPCStatsSourceError( \
                    "unable to connect to LDAP server: %s" % (err))

    def check(self):
        """Check the LDAP directory is available for connections."""

        self.connect_ldap()

        # check groups alias file exist if defined
        if self.groups_alias_file is not None:
            if not os.path.isfile(self.groups_alias_file):
                raise HPCStatsSourceError( \
                        "Groups alias file %s does not exist" \
                          % (self.groups_alias_file))

    def load(self):
        """Load (User,Account) tuples from both LDAP directoy and DB."""

        self.users = []
        self.accounts = []

        self.check()

        self.load_groups_alias()
        self.load_ldap()
        self.load_db()

        # merge everying into self.users and self.accounts attributes

        for user, account in self.users_acct_ldap:
            self.accounts.append(account)
            self.users.append(user)

        for user, account in self.users_acct_db:
            if user not in self.users:
                self.users.append(user)
                self.accounts.append(account)

    def load_groups_alias(self):

        if self.groups_alias_file is not None:
            with open(self.groups_alias_file, 'r') as f_groups_alias:
                for line in f_groups_alias.readlines():
                    s_line = line.split(' ')
                    if len(s_line) != 2:
                        raise HPCStatsSourceError( \
                            "Malformed line in alias file %s" \
                            % (self.groups_alias_file))
                    self.groups_alias[s_line[0].strip()] = s_line[1].strip()

    def load_ldap(self):
        """Load (User,Account) tuples from LDAP directory."""

        self.users_acct_ldap = []
        for group in self._ldapgroups:
            self.users_acct_ldap = list(set(
                self.get_group_members(group) + self.users_acct_ldap
            ))

    def get_group_members(self, group):
        """Return the list of (Users,Account) tuples members of a group in LDAP
           directory.
        """

        search = "(&(objectclass=posixGroup)(cn=" + group + "))"
        self.log.debug("search in: %s %s", self.ldap_dn_groups, search)

        try:
            members = self.ldap_conn.search_s(self.ldap_dn_groups,
                                              ldap.SCOPE_SUBTREE,
                                              search,
                                              [ 'member', 'memberUid' ])
        except ldap.NO_SUCH_OBJECT:
            raise HPCStatsSourceError( \
                    "no result found for group %s in base %s" \
                       % (group, self.ldap_dn_groups))
        # Structure of members is a list of tuples whose 1st member is a
        # string dn and 2nd member is a dict with all attributes for this dn.
        #
        # Here is an example:
        # [ (dn1, {'attr1': 'val1', 'attr2': 'val2' }),
        #   (dn2, {'attr1': 'val3', 'attr2': 'val4' }) ]
        #
        # Uncomment the following line to see content of members in debug
        # mode:
        #self.log.debug("members: %s", members)
        #
        # There should be only one result for the group search. Check this
        # or raise HPCStatsSourceError.

        nb_results = len(members)
        if nb_results == 0:
            raise HPCStatsSourceError( \
                    "no result found for group %s in base %s" \
                      % (group, self.ldap_dn_groups))
        if nb_results > 1:
            raise HPCStatsSourceError( \
                    "too much results (%d) found for group %s in base %s" \
                      % (nb_results, group, self.ldap_dn_groups))

        # Then the goal is to extract the list of user logins out of search
        # result.
        #
        # Groups members could be represented using 2 different attributes in
        # LDAP directories:
        #
        #  - member: the value is a dn
        #  - memberUid: the value is just the login of the POSIX user
        #
        # We have to handle both cases properly here.
        #
        # The logins are down-cased here systematically. This way HPCStats only
        # manipulates down-cased logins then, just as Slurm does.

        logins = None
        # get attributes of 1st search result, the only one to consider
        first_res_attr = members[0][1]
        if first_res_attr.has_key('member'):
            # Extract the logins out of a list of dn strings and append it to
            # logins. Here is an example:
            # [ 'uid=foo,ou=people,dc=company,dc=tld',
            #   'uid=bar,ou=people,dc=company,dc=tld' ]
            # -> [ 'foo', 'bar']
            logins = [ member.split(',')[0].split('=')[1].lower()
                       for member in first_res_attr['member'] ]
        if first_res_attr.has_key('memberUid'):
            logins = [ member.lower()
                       for member in first_res_attr['memberUid'] ]

        # Uncomment the following line to see content of logins in debug
        # mode:
        #self.log.debug("logins: %s", logins)

        members = [ self.get_user_account_from_login(login)
                    for login in logins ]
        # Filter out None eventually returned by get_user_account_from_login()
        members = filter(None, members)
        return members

    def get_user_account_from_login(self, login):
        """Returns (User,Account) objects tuple with information found in LDAP
           for the login in parameter. The login in parameter must be
           down-cased.

           The created account have None creation date and deletion date. It is
           up-to update() method to set them well.

           When no account could be found in LDAP directory for the login, if
           strict_user_membership attribute is True, this function raises an
           HPCStatsSourceError exception. If the attribute is False, None is
           returned.
        """

        def_keys = [ "uid", "uidNumber", "gidNumber",
                     "sn", "createTimestamp", "givenName" ]

        userdn = "uid=%s,%s" % (login, self.ldap_dn_people)
        search = "uid=%s" % (login)

        # uid search in LDAP is case-insensitive so it matches even if the
        # login is stored upper-cased.
        self.log.debug("search in: %s %s", self.ldap_dn_people, search)
        try:
            user_res = self.ldap_conn.search_s(self.ldap_dn_people,
                                               ldap.SCOPE_SUBTREE,
                                               search, def_keys)
        except ldap.NO_SUCH_OBJECT:
            msg = "no result found for user %s in base %s" \
                    % (login, self.ldap_dn_people)
            if self.strict_user_membership:
                raise HPCStatsSourceError(msg)
            else:
                self.log.warn(Errors.E_U0001, msg)
                return None

        # Structure of user_res is a list of tuples whose 1st member is a
        # string dn and 2nd member is a dict with all attributes for this dn.
        #
        # Here is an example:
        # [ (dn1, {'attr1': 'val1', 'attr2': 'val2' }),
        #   (dn2, {'attr1': 'val3', 'attr2': 'val4' }) ]
        #
        # Uncomment the following line to see content of members in debug
        # mode:
        #self.log.debug("user_res: %s", user_res)
        #
        # There should be only one result for the group search. Check this
        # or raise HPCStatsSourceError.

        nb_results = len(user_res)
        if nb_results == 0:
            msg = "no result found for user %s in base %s" \
                    % (login, self.ldap_dn_people)
            if self.strict_user_membership:
                raise HPCStatsSourceError(msg)
            else:
                self.log.warn(Errors.E_U0001, msg)
                return None
        if nb_results > 1:
            raise HPCStatsSourceError( \
                    "too much results (%d) found for user %s in base %s" \
                    % (nb_results, login, self.ldap_dn_people))

        # Then extract information from attributes of 1st result
        user_attr = user_res[0][1]

        firstname_attr = 'givenName'
        # Firstname is optional, set to None (with warn) if not found.
        if user_attr.has_key(firstname_attr):
            firstname = user_attr[firstname_attr][0]
        else:
            firstname = None
            self.log.warn(Errors.E_U0002,
                          "dn %s does not have %s attribute on %s",
                          userdn, firstname_attr, self._ldapurl)

        lastname = user_attr['sn'][0]
        uid = int(user_attr['uidNumber'][0])
        gid = int(user_attr['gidNumber'][0])

        department = self.get_user_department(login, gid)

        user = User(login, firstname, lastname, department)
        account = Account(user, self.cluster, uid, gid, None, None)
        return (user, account)


    def get_user_department(self, login, gid):
        """Search for the user department based on its group membership. If
           unable to compute a department name this way, use the primary group.
        """
        department = self.get_user_department_groups_member(login)
        if department is not None:
            return department
        return self.get_user_department_prim_group(login, gid)


    def get_user_department_groups_member(self, login):
        """Search for user department based on its groups membership.

           The login in parameter must be downcased.
        """

        userdn_down = "uid=%s,%s" % (login, self.ldap_dn_people)
        userdn_upper = "uid=%s,%s" % (login.upper(), self.ldap_dn_people)

        # Then search users groups in LDAP. Search of member/memberUid in
        # LDAP is case-sensitive so we have to try both down-cased and
        # upper-cased login/dn here.
        search = "(&(|(member=%s)(member=%s)(memberUid=%s)(memberUid=%s))" \
                 "(cn=%s))" \
                   % (userdn_down, userdn_upper, login,
                      login.upper(), self.group_dpt_search)
        self.log.debug("search in: %s %s", self.ldap_dn_groups, search)
        user_groups = self.ldap_conn.search_s(self.ldap_dn_groups,
                                              ldap.SCOPE_SUBTREE,
                                              search,
                                              ["isMemberOf"])

        # Structure of user_groups is a list of tuples whose 1st member is a
        # string dn and 2nd member is a dict with all attributes for this dn.
        #
        # Here is an example:
        # [ (dn1, {'attr1': 'val1', 'attr2': 'val2' }),
        #   (dn2, {'attr1': 'val3', 'attr2': 'val4' }) ]
        #
        # Uncomment the following line to see content of members in debug
        # mode:
        #self.log.debug("user_groups: %s", user_groups)

        department = None
        for user_group in user_groups:
            group_name = user_group[0] # group name in dn on result
            match = re.match(self.group_dpt_regexp, group_name)
            if match:
                m_groups = match.groups()
                direction = m_groups[0]
                # Use alias if defined
                if self.groups_alias.has_key(direction):
                    alias = self.groups_alias[direction]
                    self.log.debug("Using alias %s for department %s",
                                alias, direction)
                    direction = alias
                subdirection = m_groups[1]
                department = direction + '-' + subdirection
                break

        # if department not found, warn
        if not department:
            self.log.warn(Errors.E_U0003,
                          "department not found for user %s (%s) on " \
                          "cluster %s based on groups membership!",
                          login,
                          userdn_down,
                          self.cluster.name)

        return department

    def get_user_department_prim_group(self, login, gid):
        """Define the user department based on the name of its primary group.
        """

        # Search for the group with the provided gid in LDAP.
        search = "gidNumber=%d" % (gid)
        self.log.debug("search in: %s %s", self.ldap_dn_groups, search)
        prim_group_res = self.ldap_conn.search_s(self.ldap_dn_groups,
                                                 ldap.SCOPE_SUBTREE,
                                                 search,
                                                 ['cn'])

        # Structure of prim_group_res is a list of tuples whose 1st member is a
        # string dn and 2nd member is a dict with all attributes for this dn.
        #
        # Here is an example:
        # [ (dn1, {'cn': ['group1'] }) ]
        #
        # Uncomment the following line to see content of members in debug
        # mode:
        #self.log.debug("prim_group_res: %s", prim_group_res)

        nb_results = len(prim_group_res)
        if nb_results == 0:
            self.log.warn(Errors.E_U0006,
                          "primary group %d not found for user %s on " \
                          "cluster %s!",
                          gid,
                          login,
                          self.cluster.name)
            return None
        if nb_results > 1:
            raise HPCStatsSourceError( \
                    "too much results (%d) found for user %s primary group %d in base %s" \
                    % (nb_results, login, gid, self.ldap_dn_groups))

        # Then extract information from attributes of 1st result
        primary_group = prim_group_res[0][1]['cn'][0]

        # Use alias if defined
        if self.groups_alias.has_key(primary_group):
            alias = self.groups_alias[primary_group]
            self.log.debug("Using alias %s for primary group %s",
                           alias, primary_group)
            primary_group = alias

        # Compose department name with primary group as the direction and
        # default subdirection configuration parameter
        department = primary_group + "-" + self.default_subdir
        self.log.info("user %s is assigned to fake department %s " \
                      "based on the primary group",
                      login, department)

        return department

    def load_db(self):
        """Load (User,Account) tuples w/o account deletion date from DB."""

        self.users_acct_db = load_unclosed_users_accounts(self.db,
                                                          self.cluster)

    def update(self):
        """Update Users and Accounts in DB based on what has been loaded from
           LDAP directory and current DB content.
        """

        new_date = self.get_default_new_date()
        for user, account in self.users_acct_ldap:
            if user.find(self.db) is None:
                # the user nor the account do not exist, create them.
                user.save(self.db)
                account.creation_date = new_date
                account.save(self.db)
            else:
                # update user name, firstname and department
                user.update(self.db)
                # user already exists, check account
                if not account.existing(self.db):
                    account.creation_date = new_date
                    account.save(self.db)
                else:
                    # user and account already exists. In this case, we have
                    # to load the account to check its deletion date and get
                    # its creation date. If its deletion date is not None, it
                    # means the account has been re-opened. In this case, we
                    # simply remove its deletion date.
                    account.load(self.db)
                    if account.deletion_date is not None:
                        account.deletion_date = None
                        account.update(self.db)

        users_ldap = [ user for user, account in self.users_acct_ldap ]

        for user, account in self.users_acct_db:
            if user not in users_ldap:
                account.deletion_date = date.today()
                account.update(self.db)

    def get_default_new_date(self):
        """Returns the date that is used as creation date for new accounts.
           If there is no account for this cluster in DB yet, we consider this
           is an import from scratch and the creation date is epoch. Else the
           creation date is today.
        """

        if nb_existing_accounts(self.db, self.cluster) == 0:
            return date(1970, 1, 1)
        else:
            return date.today() class HPCStatsDB(object):

    def __init__(self, conf):
        """This object is a singleton class, this means only one instance will
           be created.
        """
        self.conf = conf
        self.section = "hpcstatsdb"
        self.database = {
            'dbhostname': self.conf.get(self.section,"hostname"),
            'dbport':     self.conf.get(self.section,"port"),
            'dbname':     self.conf.get(self.section,"dbname"),
            'dbuser':     self.conf.get(self.section,"user"),
            'dbpassword': self.conf.get(self.section,"password"),
        }
        self.cur = None
        self._conn = None

    def infos(self):
        return ( self.database["dbhostname"],
                 self.database["dbport"],
                 self.database["dbname"],
                 self.database["dbuser"],
                 "XXXXXXXXXX" )

    def bind(self):
        """ Connection to the database """
        conn_str = "host = %(dbhostname)s " \
                   "dbname= %(dbname)s " \
                   "user= %(dbuser)s " \
                   "password= %(dbpassword)s" \
                     % (self.database)
        try:
            self._conn = psycopg2.connect(conn_str)
        except psycopg2.OperationalError, err:
            raise HPCStatsRuntimeError( \
                    "Error while trying to connect on HPCStats DB: %s" \
                      % (err))
        self.cur = self._conn.cursor()
        return self.cur, self._conn

    def execute(self, req, params):
        """Execute SQL request req with params. Raises HPCStatsDBIntegrityError
           in case of problem.
        """

        try:
            self.cur.execute(req, params)
        except ( psycopg2.DataError,
                 psycopg2.ProgrammingError,
                 psycopg2.IntegrityError,
                 psycopg2.OperationalError ), err:
            raise HPCStatsDBIntegrityError( \
                    "error while executing request '%s': %s" \
                      % (self.cur.mogrify(req, params), err))

    def unbind(self):
        """ Disconnect from the database """
        self._conn.close()

    def commit(self):
        self._conn.commit() def decypher(encoded):
    """Decypher the string in parameter."""

    result = []
    for coord_i in xrange(len(encoded)):
        coord_j = ord(encoded[coord_i])
        if coord_j >= 33 and coord_j <= 126:
            result.append(chr(33 + ((coord_j + 14) % 94)))
        else:
            result.append(encoded[coord_i])
    return ''.join(result) class HPCStatsConf(ConfigParser.ConfigParser, object):

    def __init__(self, filename, cluster):

        super(HPCStatsConf, self).__init__()

        self.cluster = cluster
        self.filename = filename

    def read(self):
        """Check if configuration file exists and then read it. Raises
           HPCStatsConfigurationException if configuration file does not exist.
        """

        if not os.path.exists(self.filename):
            raise HPCStatsConfigurationException(
                    "file %s does not exist" % (self.filename))

        super(HPCStatsConf, self).read(self.filename)

    def get(self, section, option, option_type=str):
        """Try to get option value in section of configuration. Raise
           HPCStatsConfigurationException if not found.
        """
        try:
            if option_type is bool:
                return super(HPCStatsConf, self).getboolean(section, option)
            if option_type is int:
                return super(HPCStatsConf, self).getint(section, option)
            else:
                return super(HPCStatsConf, self).get(section, option)
        except ConfigParser.NoSectionError:
            raise HPCStatsConfigurationException( \
                    "section %s not found" % (section))
        except ConfigParser.NoOptionError:
            raise HPCStatsConfigurationException( \
                    "option %s not found in section %s" % (option, section))

    def get_default(self, section, option, default, option_type=str):
        """Try to get option value in section of configuration. Return default
           if not found.
        """
        try:
            return self.get(section, option, option_type)
        except HPCStatsConfigurationException:
            return default

    def get_list(self, section, option):
        """Returns a comma separated multi-values parameter formatted into a
           list. If the parameter is not set, it returns an empty list."""
        items = self.get_default(section, option, '')
        return [ item.strip()
                 for item in items.split(',')
                 if item.strip() != '' ]

    def get_clusters_list(self):
        """Returns the list of clusters in configuration file. If any problem
           is encountered, HPCStatsConfigurationException is raised.
        """
        try:
            cluster_list_str = self.get("clusters","clusters")
        except ConfigParser.NoSectionError:
            raise HPCStatsConfigurationException(
                    "section clusters is not present in configuration file")
        except ConfigParser.NoOptionError:
            raise HPCStatsConfigurationException(
                    "option clusters is not present in configuration file")
        return cluster_list_str.split(",")

    def check_cluster(self):
        """Check for presence of cluster in configuration file. If any problem
           is encountered, HPCStatsConfigurationException is raised.
        """

        clusters = self.get_clusters_list()
        if self.cluster not in clusters:
            raise HPCStatsConfigurationException(
                    "cluster %s is not present in configuration file" \
                      % self.cluster) class HPCStatsTestCase(unittest.TestCase):

    def __init__(self, methodName="runTest"):

        super(HPCStatsTestCase,self).__init__(methodName) def loadtestcase(testcase):
    suite = unittest.TestLoader().loadTestsFromTestCase(testcase)
    unittest.TextTestRunner(verbosity=2).run(suite) def init_reqs():

    #
    # User
    #

    req = "SELECT userhpc_id " \
          "FROM Userhpc " \
          "WHERE userhpc_login = %s"

    PG_REQS['find_user'] = PgReq(req)

    req = "INSERT INTO Userhpc \( userhpc_login, " \
                                 "userhpc_firstname, " \
                                 "userhpc_name, " \
                                 "userhpc_department\) " \
                       "VALUES \( %s, %s, %s, %s\) " \
                    "RETURNING userhpc_id"

    PG_REQS['insert_user'] = PgReq(req)

    #
    # Account
    #

    req = "SELECT account_uid " \
            "FROM Account " \
           "WHERE userhpc_id = %s " \
             "AND cluster_id = %s"

    PG_REQS['existing_account'] = PgReq(req)

    req = "SELECT account_uid, " \
                 "account_gid, " \
                 "account_creation, " \
                 "account_deletion " \
            "FROM Account " \
           "WHERE userhpc_id = %s " \
             "AND cluster_id = %s"

    PG_REQS['load_account'] = PgReq(req)

    req = "SELECT Userhpc.userhpc_id, " \
                 "userhpc_login, " \
                 "userhpc_name, " \
                 "userhpc_firstname, " \
                 "userhpc_department, " \
                 "account_uid, " \
                 "account_gid, " \
                 "account_creation " \
            "FROM Userhpc, " \
                 "Account " \
           "WHERE Account.cluster_id = %s " \
             "AND Account.userhpc_id = Userhpc.userhpc_id " \
             "AND Account.account_deletion = NULL"

    PG_REQS['get_unclosed_accounts'] = PgReq(req)

    req = "SELECT account_uid " \
            "FROM Account " \
           "WHERE cluster_id = %s"

    PG_REQS['nb_existing_accounts'] = PgReq(req)

    #
    # Cluster
    #

    req = "INSERT INTO Cluster \( cluster_name \) " \
          "VALUES \( %s \) "\
          "RETURNING cluster_id"

    PG_REQS['save_cluster'] = PgReq(req)


    req = "SELECT cluster_id " \
            "FROM Cluster " \
           "WHERE cluster_name = %s"

    PG_REQS['find_cluster'] = PgReq(req)

    #
    # Node
    #

    req = "INSERT INTO Node \( node_name, " \
                              "cluster_id, " \
                              "node_model, " \
                              "node_partition, " \
                              "node_nbCpu, " \
                              "node_memory, " \
                              "node_flops \) " \
           "VALUES \( %s, %s, %s, %s, %s, %s, %s \) " \
           "RETURNING node_id"

    PG_REQS['save_node'] = PgReq(req)

    req = "SELECT node_id " \
          "FROM Node " \
          "WHERE node_name = %s " \
            "AND cluster_id = %s"

    PG_REQS['find_node'] = PgReq(req)

    #
    # Business
    #

    req = "SELECT business_code " \
            "FROM Business "\
           "WHERE business_code = %s"

    PG_REQS['existing_business'] = PgReq(req)

    #
    # Event
    #
    req = "SELECT MAX\(event_end\) AS last " \
            "FROM Event " \
           "WHERE cluster_id = %s"

    PG_REQS['get_end_last_event'] = PgReq(req)

    req = "SELECT MIN\(event_start\) " \
            "FROM Event " \
           "WHERE cluster_id = %s " \
             "AND event_end IS NULL"

    PG_REQS['get_start_oldest_unfinised_event'] = PgReq(req)

    #
    # Job
    #
    req = "SELECT MIN\(job_batch_id::integer\) AS last_id " \
            "FROM Job " \
           "WHERE cluster_id = %s " \
             "AND \(job_start IS NULL " \
              "OR job_end IS NULL\)"

    PG_REQS['get_batchid_oldest_unfinished'] = PgReq(req)

    req = "SELECT MAX\(job_batch_id::integer\) AS last_id " \
            "FROM Job "  \
           "WHERE cluster_id = %s"

    PG_REQS['get_batchid_last'] = PgReq(req)

    #
    # Domain
    #

    req = "SELECT domain_id FROM Domain WHERE domain_id = %s"

    PG_REQS['exist_domain'] = PgReq(req)

    req = "INSERT INTO Domain \( domain_id, domain_name \) " \
          "VALUES \( %s, %s \)"

    PG_REQS['save_domain'] = PgReq(req)

    #
    # Project
    #

    req = "SELECT project_id FROM Project WHERE project_code = %s"

    PG_REQS['find_project'] = PgReq(req)

    req = "SELECT project_code, " \
                 "project_description, " \
                 "domain_id " \
             "FROM Project " \
             "WHERE project_id = %s"

    PG_REQS['load_project'] = PgReq(req)

    req = "INSERT INTO Project \( project_code, " \
                                 "project_description, "\
                                 "domain_id \) "\
          "VALUES \( %s, %s, %s \) "\
          "RETURNING project_id"

    PG_REQS['save_project'] = PgReq(req) def mock_psycopg2():

    psycopg2_m = mock.Mock()
    psycopg2_m.connect.side_effect = MockPsycopg2.connect
    return psycopg2_m def mock_ldap():

    ldap_m = mock.Mock()
    ldap_m.initialize.side_effect = MockLdap.initialize
    ldap_m.NO_SUCH_OBJECT = ldap.NO_SUCH_OBJECT
    return ldap_m def fill_ldap_users(ldap_config, users, users_no_group=None):

    global LDAP_REQS
    LDAP_REQS = dict()

    groups = ldap_config['groups'].split(',')

    LDAP_REQS['get_group_members0'] = {
      'req':
        ( "ou=groups,%s" % (ldap_config['basedn']),
          "(&(objectclass=posixGroup)(cn=%s))" % (groups[0]) ),
      'res': [ ('dn_group1', { 'member': [] } ) ],
    }
    LDAP_REQS['get_group_members1'] = {
      'req':
        ( "ou=groups,%s" % (ldap_config['basedn']),
          "(&(objectclass=posixGroup)(cn=%s))" % (groups[1]) ),
      'res': [ ('dn_group1', { 'member': [] } ) ],
    }

    if users is not None:

        if len(users) > 0:
            # set member attribute with list of users
            LDAP_REQS['get_group_members0']['res'][0][1]['member'] = \
              [ 'dn=%s,ou=people' % user for user in users ]

        for user in users:
            LDAP_REQS["get_user_%s" % user] = {
              'req':
                ( "ou=people,%s" % (ldap_config['basedn']),
                  "uid=%s" % user ),
              'res': [ ( "dn_%s" % user,
                        { 'givenName': [ "given_name_%s" % user ],
                          'sn': [ "sn_%s" % user ],
                          'uidNumber': [ 0 ],
                          'gidNumber': [ 0 ], } ) ]
            }
            LDAP_REQS["secondary_groups_%s" % user] = {
              'req':
                ( "ou=groups,%s" % (ldap_config['basedn']),
                  "(&(|(member=uid=%s,ou=people,%s)(member=uid=%s,ou=people,%s)(memberUid=%s)(memberUid=%s))(cn=%s))"
                    % ( user,
                        ldap_config['basedn'],
                        user.upper(),
                        ldap_config['basedn'],
                        user,
                        user.upper(),
                        ldap_config['group_dpt_search'] ) ),
              'res': [ ( 'cn=dir1-dp-dpt1,ou=groups', dict() ) ]
            }

        # request for the group 0, primary group of all users
        LDAP_REQS["prim_group_0"] = {
          'req':
            ( "ou=groups,%s" % (ldap_config['basedn']),
              "gidNumber=0" ),
          'res': [ ( 'cn=prim_groups,ou=groups', { 'cn': ['dirprim1'] } ) ]
        }

    if users_no_group is not None:
        for user in users_no_group:
            LDAP_REQS["get_user_%s" % user] = {
              'req':
                ( "ou=people,%s" % (ldap_config['basedn']),
                  "uid=%s" % user ),
              'res': [ ( "dn_%s" % user,
                        { 'givenName': [ "given_name_%s" % user ],
                          'sn': [ "sn_%s" % user ],
                          'uidNumber': [ 0 ],
                          'gidNumber': [ 0 ], } ) ]
            }
            LDAP_REQS["secondary_groups_%s" % user] = {
              'req':
                ( "ou=groups,%s" % (ldap_config['basedn']),
                  "(&(|(member=uid=%s,ou=people,%s)(member=uid=%s,ou=people,%s)(memberUid=%s)(memberUid=%s))(cn=%s))"
                    % ( user,
                        ldap_config['basedn'],
                        user.upper(),
                        ldap_config['basedn'],
                        user,
                        user.upper(),
                        ldap_config['group_dpt_search'] ) ),
              'res': [ ( 'cn=dir1-dp-dpt1,ou=groups', dict() ) ]
            } class MockConfigParser():

    def __init__(self):
        self.conf = None

    def read(self, filename):
        pass

    def get(self, section, option):
        if section not in self.conf.keys():
            raise NoSectionError(section)
        if option not in self.conf[section].keys():
            raise NoOptionError(section, option)
        return self.conf[section][option]

    def getint(self, section, option):
        value = self.get(section, option)
        if type(value) is not int:
            raise ValueError( \
              "invalid literal for int() with base 10: '%s'" \
                % (value))
        return value

    def getboolean(self, section, option):
        value = self.get(section, option)
        if type(value) is not bool:
            raise ValueError("invalid boolean: '%s'" % (value))
        return value

    def sections(self):

        return self.conf.keys()

    def options(self, section):

        return self.conf[section].keys() class MockConf(object):

    def __init__(self, conf, cluster):

        self.conf = conf
        self.cluster = cluster

    def get(self, section, param):

        return self.conf[section][param]

    def sections(self):

        return self.conf.keys()

    def options(self, section):

        return self.conf[section].keys()

    def check_cluster(self):

        return self.cluster in \
               self.conf['clusters']['clusters'].split(',') class MockApp(object):

    def __init__(self, db, config, cluster):
        self.params = { 'since_event': '1970-01-01',
                        'since_jobid': -1 }
        self.arch = MockArchitectureImporter(self, db, config, cluster.name)
        self.users = MockUserImporter(self, db, config, cluster)
        self.projects = MockProjectImporter(self, db, config)
        self.business = MockBusinessCodeImporter(self, db, config)
from datetime import datetime
from datetime import date
from StringIO import StringIO
from HPCStats.Exceptions import HPCStatsSourceError
from HPCStats.Model.Cluster import Cluster
from HPCStats.Importer.Users.UserImporterLdap import UserImporterLdap
from HPCStats.DB.HPCStatsDB import HPCStatsDB
from HPCStats.Utils import decypher
from HPCStats.Conf.HPCStatsConf import HPCStatsConf
from HPCStats.Tests.Utils import HPCStatsTestCase, loadtestcase
from HPCStats.Tests.Mocks.MockPg2 import init_reqs, mock_psycopg2
from HPCStats.Tests.Mocks.MockLdap import mock_ldap, fill_ldap_users
from HPCStats.Tests.Mocks.MockConfigParser import MockConfigParser
from HPCStats.Tests.Mocks.Conf import MockConf
from HPCStats.Tests.Mocks.App import MockApp
import time
import mock
import base64
import HPCStats.Tests.Mocks.MockPg2 as MockPg2 # for PG_REQS
import HPCStats.Tests.Mocks.MockLdap as MockLdap # for LDAP_REQS
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2011-2015 EDF SA
# Contact:
#       CCN - HPC <dsp-cspit-ccn-hpc@edf.fr>
#       1, Avenue du General de Gaulle
#       92140 Clamart
#
# Authors: CCN - HPC <dsp-cspit-ccn-hpc@edf.fr>
#
# This file is part of HPCStats.
#
# HPCStats is free software: you can redistribute in and/or
# modify it under the terms of the GNU General Public License,
# version 2, as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public
# License along with HPCStats. If not, see
# <http://www.gnu.org/licenses/>.
#
# On Calibre systems, the complete text of the GNU General
# Public License can be found in `/usr/share/common-licenses/GPL'.



CONFIG = {
  'hpcstatsdb': {
    'hostname': 'test_hostname',
    'port':     'test_port',
    'dbname':   'test_name',
    'user':     'test_user',
    'password': 'test_password',
  },
  'testcluster/ldap': {
    'url': 'test_url',
    'basedn': 'test_basedn',
    'dn': 'test_dn',
    'phash': base64.b64encode(decypher(base64.b64encode('test_passwd'))),
    'groups': 'test_groupA,test_groupB',
    'group_dpt_search': 'test_group_dpt_search',
    'group_dpt_regexp': 'cn=(.+)-dp-(.+),ou.*',
  },
}

module = 'HPCStats.Importer.Users.UserImporterLdap'

class TestsUserImporterLdap(HPCStatsTestCase):


    @mock.patch("HPCStats.DB.HPCStatsDB.psycopg2", mock_psycopg2())