class ScannerFile(ScannerBase):

    suf_list = ['.conf', '.cfg', '.ini']
    
    def scan(self):
        """
        scan config files
        """

        logging.info('searching for config files')

        # scan the system directories
        self.scandir(Config.files_path)
        
    # scan directory and add config files
    def scandir(self, pathdir): 
        # Visit every file in pathdir except those on the exclusion list above.
        pathdirs = re.split(":", pathdir)
        for p in pathdirs:
            if not p: continue
            for dirpath, dirnames, filenames in os.walk(p, followlinks=True):
                for filename in filenames:
                    self.addfile(os.path.join(dirpath, filename))

    # add per config file
    def addfile(self, pathname):
        # only plane text file
        if valid_txtfile(pathname) == False:     
            return

#        # only include above suffix config file
#        suf = os.path.splitext(pathname)[1]
#        if suf is None or suf not in self.suf_list:
#            return

        # get owner, group and mode
        s = get_stat(pathname)
        
        # read the config file's content
        c = get_content(pathname)

        # add the config file:
        # checksum, content, group, mode, owner, path, force=False, provider=None, 
        # recurse=None, recurselimit=None, source=None
        self.add_file('md5', c, s[0], s[1], s[2], pathname) class ScannerGem(ScannerBase):

    def scan(self):
        """
        scan gem
        """
        logging.info('searching for Ruby gems')

        # Precompile a pattern for extracting the version of Ruby that was used
        # to install the gem.
        pattern = re.compile(r'gems/([^/]+)/gems')

        # Look for gems in all the typical places.  This is easier than looking
        # for `gem` commands, which may or may not be on `PATH`.
        for globname in ('/usr/lib/ruby/gems/*/gems',
                         '/usr/lib64/ruby/gems/*/gems',
                         '/usr/local/lib/ruby/gems/*/gems',
                         '/var/lib/gems/*/gems'):
            for dirname in glob.glob(globname):
                # The `ruby1.9.1` (really 1.9.2) package on Maverick begins
                # including RubyGems in the `ruby1.9.1` package and marks the
                # `rubygems1.9.1` package as virtual.  So for Maverick and
                # newer, the manager is actually `ruby1.9.1`.
                match = pattern.search(dirname)
                if '1.9.1' == match.group(1) and rubygems_virtual():
                    manager = 'ruby{0}'.format(match.group(1))

                # Oneiric and RPM-based distros just have one RubyGems package.
                elif rubygems_unversioned():
                    manager = 'rubygems'

                # Debian-based distros qualify the package name with the version
                # of Ruby it will use.
                else:
                    manager = 'rubygems{0}'.format(match.group(1))

                for entry in os.listdir(dirname):
                    try:
                        package, version = entry.rsplit('-', 1)

                    except ValueError:
                        logging.warning('skipping questionably named gem {0}'.
                                        format(entry))
                        continue

                    self.add_package(package, manager=manager, version=version, provider='gem') class ScannerNpm(ScannerBase):
    def scan(self):
        """
        scan apt
        """
        logging.info('searching for npm packages')

        # Precompile a pattern for parsing the output of `npm list -g`.
        pattern = re.compile(r'^\S+ (\S+)@(\S+)$')

        lines = self.subprocess(['npm', 'ls', '-g'])    # only list global packages
        for line in lines:
            match = pattern.match(line.rstrip())
            if match is None:
                continue
            package, version = match.group(1), match.group(2)
            manager='nodejs'

            self.add_package(package, manager=manager, version=version, provider='npm') class ScannerPhp(ScannerBase):
    def scan(self):
        """
        scan php
        """
        logging.info('searching for PEAR/PECL packages')

        # Precompile a pattern for parsing the output of `{pear,pecl} list`.
        pattern = re.compile(r'^([0-9a-zA-Z_]+)\s+([0-9][0-9a-zA-Z\.-]*)\s')

        # PEAR packages are managed by `php-pear` (obviously).  PECL packages
        # are managed by `php5-dev` because they require development headers
        # (less obvious but still makes sense).
        if lsb_release_codename() is None:
            pecl_manager = 'php-devel'
        else:
            pecl_manager = 'php5-dev'
        for manager, progname in (('php-pear', 'pear'),
                                  (pecl_manager, 'pecl')):

            lines = self.subprocess([progname, 'list'])
            if lines==None:
                return
                
            for line in lines:
                match = pattern.match(line)
                if match is None:
                    continue
                package, version = match.group(1), match.group(2)
                
                self.add_package(package, manager=manager, version=version, provider=progname) class ScannerPypi(ScannerBase):

    def scan(self):
        """
        scan pypi
        """
        logging.info('searching for Python packages')

        # Look for packages in the typical places.
        globnames = ['/usr/lib/python*/dist-packages',
                     '/usr/lib/python*/site-packages',
                     '/usr/local/lib/python*/dist-packages',
                     '/usr/local/lib/python*/site-packages']
        virtualenv = os.getenv('VIRTUAL_ENV')
        if virtualenv is not None:
            globnames.extend(['{0}/lib/python*/dist-packages'.format(virtualenv),
                              '{0}/lib/python*/dist-packages'.format(virtualenv)])
        for globname in globnames:
            for dirname in glob.glob(globname):
                manager = PATTERN_MANAGER.search(dirname).group(1)
                for entry in os.listdir(dirname):
                    match = PATTERN.match(entry)
                    if match is None:
                        continue
                    package, version = match.group(1, 2)
                    
                    pathname = os.path.join(dirname, entry)

                    # Symbolic links indicate this is actually a system package
                    # that injects files into the PYTHONPATH.
                    if os.path.islink(pathname): continue

                    # check pathname
                    if not os.path.isdir(pathname):
                        pathname = os.path.join(dirname, package)
                        if not os.path.exists(pathname): continue

                    # installed via `easy_install`.
                    if PATTERN_EGG.search(entry):
                        self.add_package(package, manager='python', version=version, provider='pip')

                    # installed via `pip`.
                    elif PATTERN_EGGINFO.search(entry):
                        self.add_package(package, manager='pip', version=version, provider='pip')                     class ScannerService(ScannerBase):
    # Patterns for determining which Upstart services should be included, based
    # on the events used to start them.
    UPSTART_PATTERN1 = re.compile(r'start\s+on\s+runlevel\s+\[[2345]', re.S)
    UPSTART_PATTERN2 = re.compile(r'start\s+on\s+\([^)]*(?:filesystem|filesystems|local-filesystems|mounted|net-device-up|remote-filesystems|startup|virtual-filesystems)[^)]*\)', re.S)
    
    def scan(self):
        """
        scan service:
        search the service config files
        """
        logging.info('searching for system services')

        for dir in ['/etc/init', 
                    '/etc/init.d',
                    '/etc/rc.d/init.d']:
            for dirname, dirnames, filenames in os.walk(dir):
                for filename in filenames:
                    try:
                        pathname = os.path.join(dirname, filename)
                        dict = self.parse_service(pathname)

                        # add service
                        if dict != None:
                            # format (provider, name, enable, hasrestart)
                            self.add_service(enable=dict['enable'], hasrestart=dict['hasrestart'], 
                                name=dict['name'], path=dirname, provider=dict['provider'], hasstatus=dict['hasstatus'])
                    except ValueError:
                        pass
            
    def parse_service(self, pathname):
        """
        Parse a potential service init script or config file into the
        manager and service name or raise `ValueError`.  Use the Upstart
        "start on" stanzas and SysV init's LSB headers to restrict services to
        only those that start at boot and run all the time.
        
        ###Need to add systemd service parse.
        """
        
        dirname, basename = os.path.split(pathname)
        if '/etc/init' == dirname:
            service, ext = os.path.splitext(basename)
            
            # Ignore extraneous files in /etc/init.
            if '.conf' != ext:
                raise ValueError('not an Upstart config')

            # Ignore services that don't operate on the (faked) main runlevels.
            try:
                content = open(pathname).read()
            except IOError:
                raise ValueError('not a readable Upstart config')
                
            enable = False    
            if (self.UPSTART_PATTERN1.search(content) \
                    or self.UPSTART_PATTERN2.search(content)):
                enable = True

            return {'provider':'upstart', 'name':service, 'enable':enable, 
                'hasrestart':False, 'hasstatus':False}
                
        elif '/etc/init.d' == dirname or '/etc/rc.d/init.d' == dirname:
            #import pdb
            #pdb.set_trace()

            # Let Upstart handle its services.
            if os.path.islink(pathname) \
                and '/lib/init/upstart-job' == os.readlink(pathname):
                raise ValueError('proxy for an Upstart config')

            # Ignore services that don't operate on the main runlevels.
            try:
                content = open(pathname).read()
            except IOError:
                raise ValueError('not a readable SysV init script')
                
            enable = False    
            if re.search(r'(?:Default-Start|chkconfig):\s*[-2345]', content):
                enable = True
            
            hasrestart = False
            if re.search(r'\s*(?:restart|reload)\)\s*', content):
                hasrestart = True
            
            hasstatus = False
            if re.search(r'\s*status\)\s*', content):
                hasstatus = True
            
            return {'provider':'init', 'name':basename, 'enable':enable,
                'hasrestart':hasrestart, 'hasstatus':hasstatus}       ### change sysvinit to init
        else:
            raise ValueError('not a service') class ScannerHost(ScannerBase):

    def scan(self):
        """
        scan host:
        parse the host config file (usually /etc/hosts)
        """
        logging.info('searching for hosts')
        hostlst = self.parse_hostfile()

        if len(hostlst)<=0:
            return

        for dict in hostlst:
            self.add_host(ip=dict['ip'], name=dict['name'],
                target=dict['target'], host_aliases=dict['host_aliases'])

    def parse_hostfile(self):
        hostfile = Config.scan_host

        hosts = []
        try:
            for line in open(hostfile):
                # ignore blank line
                if not line.strip(): continue
                # ignore comment line
                elif line.strip().startswith("#"): continue

                itemlst = line.strip().split()
                if len(itemlst) <= 1: continue

                hosts.append({'ip':itemlst[0], 'name':itemlst[1], 'target':hostfile, 'host_aliases':itemlst[2:]})

                #ip = re.search( r'[0-9]+(?:\.[0-9]+){3}', itemlst[0] )
                #if ip==None:
                #    continue
                #aliases = []
                #if len(itemlst)>=3:
                #    for i in range(2, len(itemlst)-1):
                #        aliases.append(itemlst[i])
                #print "aliases=%s"%aliases
                #hosts.append({'ip':ip.group(), 'name':itemlst[1], 'target':hostfile, 'host_aliases':aliases})

        except IOError:
            return hosts

        return hosts class ScannerMount(ScannerBase):
    
    def scan(self):
        self.__disk_usage()
    
    def __disk_partitions(self,all=False):  
        """
        Return all mountd partitions as a dict.    
        """  
        phydevs = []  
        f = open("/proc/filesystems", "r")  
        for line in f:  
            if not line.startswith("nodev"):  
                phydevs.append(line.strip())  
      
        retlist = {} 
        f = open('/etc/mtab', "r")  
        for line in f:  
            if not all and line.startswith('none'):
                continue
            fields                 = line.split()  
            device                 = fields[0]  
            mountpoint             = fields[1]  
            fstype                 = fields[2]  
            if not all and fstype not in phydevs:  
                continue  
            if device == 'none':  
                device             = ''
            retlist[device]        = (device, mountpoint, fstype)
        return retlist  
    
    def __disk_usage(self):  
        """Return disk usage associated with path."""  
        disk                     = self.__disk_partitions()
        for device in disk.keys():
            st = os.statvfs(device)
            size = st.f_blocks * st.f_frsize
            self.add_mount(device=device, fstype=disk[device][2], name=disk[device][1], size=size) class ScannerCron(ScannerBase):
    def scan(self):

        users = self.get_users()
        res = self.subprocess(['crontab', '-l'])
        for line in res:
            # ignore empty and the comment lines
            if not line.strip() or line.strip().startswith("#"): continue

            ary = line.split()
            if ary[5] in users.keys():
                paths = os.path.split(ary[6])
                self.add_cron(
                              name=paths[1],
                              command=" ".join(ary[6:]),
                              environment=paths[0],
                              user=ary[5],
                              minute=ary[0],
                              hour=ary[1],
                              monthday=ary[2],
                              month=ary[3],
                              weekday=ary[4]
                              )
            else:
                paths = os.path.split(ary[5])
                self.add_cron(
                              name=paths[1],
                              command=" ".join(ary[5:]),
                              environment=paths[0],
                              minute=ary[0],
                              hour=ary[1],
                              monthday=ary[2],
                              month=ary[3],
                              weekday=ary[4]
                              ) class ScannerKey(ScannerBase):
    
    support_key_type = [
                        'ssh-rsa',
                        'ssh-dss',
                        'ecdsa-sha2-nistp256',
                        'ecdsa-sha2-nistp384',
                        'ecdsa-sha2-nistp521'
                        ]
    
    scan_file_type = [
                      '.pub',
                      '.pem'
                      ]
    
    re_pattern = "-----.+-----"

    path_dir = Config.key_path

    def scan(self):
        pathdirs = re.split(":", self.path_dir)
        for p in pathdirs:
            if not p: continue
            for dirpath, dirnames, filenames in os.walk(p):
                for filename in filenames:
                    try:
                        # key with specific name
                        if '.' in filename:
                            for file_type in self.scan_file_type:
                                if filename.endswith(file_type):
                                    full_path = os.path.join(dirpath, filename)
                                    mode = get_stat(full_path)[1]
                                    content = get_content(full_path)
                                    if content:
                                        _type = self.__get_type(content)
                                        self.add_key(key=content, name=filename, _type=_type, path=full_path, mode=mode)
                                        logging.debug('ScannerKey.scan(): Add key file %s' % filename)
                        # key without specific name
                        else:
                            full_path = os.path.join(dirpath, filename)
                            mode = get_stat(full_path)[1]
                            content = get_content(full_path)
                            if content and re.match(self.re_pattern, content):
                                _type = self.__get_type(content)
                                self.add_key(key=content, name=filename, _type=_type, path=full_path, mode=mode)
                                logging.debug('ScannerKey.scan(): Add key file %s' % filename)
                    except Exception, e:
                        #log
                        logging.error("ScannerKey.scan(): Add file %s failed, %s" % (filename, str(e)))

    def __get_type(self, content):
        for _type in self.support_key_type:
            if _type in content:
                return _type class ScannerUser(ScannerBase):
    def scan(self):
        for p in pwd.getpwall():
            name, password, uid, gid, gecos, home, shell = p

            groups = [] 
            # get the secondary groups
            for gr in grp.getgrall():
                gr_name, gr_pwd, gr_gid, gr_mem = gr
                # check whether the main group
                if gid == gr_gid:   
                    group = gr_name
                    continue
                # check whether the group member
                if name in gr_mem:
                    groups.append(gr_name)

            if group is None or not group: continue

            self.add_user(name=name, uid=uid, gid=gid, group=group, groups=groups, shell=shell, home=home) class ScannerGroup(ScannerBase):
    
    def scan(self):
        for g in grp.getgrall():
            name, password, gid, member = g
            self.add_group(name=name, gid=gid, member=member) class ScannerPackage(ScannerBase):
    def scan(self):
        """
        scan package
        first search with yum, next with apt and with subprocess at last
        """
        logging.info('searching for packages')

        self.core_pkgs = {}
        self.user_pkgs = {}
        self.ip_list = []

        if self.scan_yum()==False:
           if self.scan_rpm()==False:
                if self.scan_apt()==False:
                    self.scan_subprocess()
                else: Config.platform = "apt"
           else: Config.platform = "rpm"
        else: Config.platform = "yum"

        # add the user package list
        self.add_pkgs()

        # add ips
        for ip in self.ip_list:
            self.add_ip(ip)

    def scan_yum(self):
        """
        scan the redhat family platform with yum api
        """
        try:
            import yum
        except ImportError:
            return False

        logging.info('searching for yum packages')

        self.scan_mode = 'yum'

        yb = yum.YumBase()
        # get all installed packages and check each's dependency
        yb.conf.cache = 1
        for pkg in sorted(yb.rpmdb.returnPackages()):
            pkgtag = pkg.__str__()

            # skip the package which is in the core package list
            if pkgtag in self.core_pkgs:
                continue

            reqs = pkg.required_packages()
            # remove packages which is in the requiring list from user packages
            # and add packages which is not in core packages from dependency list
            for req in reqs:
                reqtag = req.__str__()
                if reqtag in self.user_pkgs:
                    del self.user_pkgs[reqtag]

                if reqtag not in self.core_pkgs:
                    self.core_pkgs[reqtag] = req

            thepkg = yb.pkgSack.searchNevra(pkg.name, pkg.epoch, pkg.version, pkg.release, pkg.arch)  #epoch=None, ver=None, rel=None, arch=None
            if len(thepkg)==0 or thepkg[0].verEQ(pkg)==False:
                continue

            self.user_pkgs[pkgtag] = pkg

        return True

    def scan_rpm(self):
        """
        scan the redhat family platform with rpm api
        """
        try:
            import rpm
        except ImportError:
            return False

        logging.info('searching for rpm packages')

        self.scan_mode = 'rpm'

        ts = rpm.TransactionSet()
        mi = ts.dbMatch()
        for h in mi:
            if h[rpm.RPMTAG_EPOCH]==None:
                epoch = '(none)'
            pkgtag = '%s-%s-%s.%s-%s' % (h[rpm.RPMTAG_NAME], h[rpm.RPMTAG_VERSION],
                h[rpm.RPMTAG_RELEASE], h[rpm.RPMTAG_ARCH], epoch)

            # skip the package in the core package list already
            if pkgtag in self.core_pkgs:
                continue

            # query the package dependency
            for req in h[rpm.RPMTAG_REQUIRENAME]:
                if req.find(' ')>=0:
                    req = req[0:req.find(' ')]
                if req.startswith('rpmlib'):    continue

                try:
                    p = subprocess.Popen(['rpm',
                                      '-q',
                                      '--qf=%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}-%{EPOCH}',
                                      '--whatprovides',
                                      req],
                                     close_fds=True,
                                     stdout=subprocess.PIPE)
                    reqtag, stderr = p.communicate()
                    # ignore the package itself
                    if reqtag==pkgtag: continue
                except OSError: continue

                if reqtag in self.user_pkgs:
                    del self.user_pkgs[reqtag]
                if reqtag not in self.core_pkgs:
                    self.core_pkgs[reqtag] = req   ### need to query the req_pkg's header

                self.user_pkgs[pkgtag] = h

        return True

    def scan_apt(self):
        """
        searching at the debian family platform with apt package
        http://apt.alioth.debian.org/python-apt-doc/
        """
        try:
            import apt
        except ImportError:
            return False

        logging.info('searching for Apt packages')

        self.scan_mode = 'apt'

        apt_cache = apt.Cache()
#        apt_cache.update()
#        apt_cache.open()

        DEP_TYPES = ['Depends', 'Recommends', 'Suggests', 'Replaces', 'Enhances']

        for package_name in apt_cache.keys():
            pkg = apt_cache[package_name]

            #Verify that the package is installed
            if hasattr(pkg, 'isInstalled'):
                if not pkg.isInstalled:    continue
            elif hasattr(pkg, 'is_installed'):
                if not pkg.is_installed:    continue
            else:
                continue


            pkgid = pkg.id

            for pkgv in pkg.versions:
                deps = self.ext_aptdeps(pkgv, 'Depends')

                for dep in deps:
                    # check the dep_pkg's name
                    if dep.name not in apt_cache: continue
                    # query the dep_pkg
                    dep_pkg = apt_cache[dep.name]

                    depid = dep_pkg.id

                    if hasattr(pkg, 'isInstalled'):
                        if not pkg.isInstalled:    continue
                    elif hasattr(pkg, 'is_installed'):
                        if not pkg.is_installed:    continue
                    else:
                        continue

                    if depid in self.user_pkgs:
                        del self.user_pkgs[depid]

                    if depid not in self.core_pkgs:
                        self.core_pkgs[depid] = dep_pkg

            self.user_pkgs[pkgid] = pkg

        return True

    def ext_aptdeps(self, pkg, dep_type):
        """
        generate the package's dependencies of particular dependency type
        """
        for dep_pkglst in pkg.get_dependencies(dep_type):
            for dep_pkg in dep_pkglst.or_dependencies:
                yield dep_pkg

    def scan_subprocess(self):
        """
        searching package with subprocess
        """
        logging.info('searching with subprocess')

        self.scan_mode = 'sub_rpm'
        Config.platform = 'rpm'
        lines = self.subprocess(['rpm', '--qf=%{NAME}\x1E%{VERSION}\x1E%{RELEASE}\x1E%{ARCH}\x1E%{EPOCH}\x1E%{SUMMARY}\x1E%{VENDOR}\n', '-qa'])
        if lines==None:
            # try apt of the Debian family system
            self.scan_mode = 'sub_dpkg'
            Config.platform = 'apt'

            lines = self.subprocess(['dpkg-query', '-W',
                '-f=${Status}\x1E${Package}\x1E${Version}\x1E${Architecture}\n'])

            if lines==None:
                logging.info('searching packages failed')
                return

        # parse the stdout [michael, 2013/03/28]
        for line in lines:
            if self.scan_mode=='sub_rpm':
                pkg, ver, rel, arch, epo, sum, ven = line.strip().split('\x1E')
                pkg_tag = pkg + '-' + ver + '-' + rel + '.' + arch + '-' +  epo
            elif self.scan_mode=='sub_dpkg':
                status, pkg, ver, arch = line.strip().split('\x1E')
                if 'install ok installed' != status: continue
                pkg_tag = pkg + '-' + ver + '-' + arch
            else:
                return

            if pkg_tag in self.core_pkgs:   continue

            # query the dependency
            self.query_deps(pkg)

            confs = []
            if self.scan_mode=='sub_rpm':
                # query config files
                conffiles = self.subprocess(['rpm', '-qc', pkg])
                for fi in conffiles:
                    file = fi.strip()
                    confs.append(file)

                self.user_pkgs[pkg_tag] = {
                    'name':pkg, 'version':ver, 'release':rel, 'arch':arch, 'epoch':epo, 'summary':sum, 'vendor':ven, 'configs':confs}

            if self.scan_mode=='sub_dpkg':
                conffiles = self.subprocess(['dpkg-query', '-W', '-f=${Conffiles}', pkg])
                for fi in conffiles:
                    file = fi.strip().split(' ')[0]
                    confs.append(file)

                self.user_pkgs[pkg_tag] = {'name':pkg, 'version':ver, 'arch':arch, 'configs':confs}

    def add_conffile(self, filepath):
        """
        check and add the package's config file
        """
        ### config file(in 'etc' directory)
        if filepath.startswith(('/etc/', '/home/'))==False:
            return False
        ### suffix list[conf, cfg, ini]
        filename, suffix = os.path.splitext(filepath)
        if suffix not in ['.conf', '.cfg', '.ini']:
            return False

        # get the config file's information and add it
        # only plane text file
        if valid_txtfile(filepath) == False:
            return False

        # get owner, group and mode
        s = get_stat(filepath)

        # read the config file's content
        c = get_content(filepath)

        # add the config file:
        # checksum, content, group, mode, owner, path, force=False, provider=None,
        # recurse=None, recurselimit=None, source=None
        self.add_file('md5', c, s[0], s[1], s[2], filepath)

        return True

    def query_deps(self, package):
        """
        query package's denpendency in the list and update the user/core package list
        """

        if self.scan_mode == 'sub_rpm':
            reqs = self.subprocess(['rpm', '-qR', package])

            for req in reqs:
                req = req.strip()
                # query the dependency package
                infos = self.subprocess(['rpm', '-q',
                    '--qf=%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}-%{EPOCH}\n',
                    '--whatprovides', req])

                for info in infos:
                    dep_tag = info.strip().split('\x1E')[0]

                    # check dependency package whether in the user package list
                    if dep_tag in self.user_pkgs: del self.user_pkgs[dep_tag]

                    # check dependency package whether in the core package list
                    if dep_tag not in self.core_pkgs: self.core_pkgs[dep_tag] = dep_tag

        elif self.scan_mode == 'sub_dpkg':
            # query the dependency
            deplines = self.subprocess(['dpkg-query', '-W', '-f=${Depends}\n', package])

            for line in deplines:
                deps = re.split(', |\|', line.strip())

                for dep in deps:
                    if dep.find(' '):
                        dep = dep[0:dep.find(' ')]

                    infos = self.subprocess(['dpkg-query', '-W',
                        '-f=${Status}\x1E${Package}-${Version}-${Architecture}\n', dep])
                    for line in infos:
                        status, dep_tag = line.strip().split('\x1E')

                        if 'install ok installed' != status:    continue

                        if dep_tag in self.user_pkgs:
                            del self.user_pkgs[dep_tag]

                        if dep_tag not in self.core_pkgs:
                            self.core_pkgs[dep_tag] = dep_tag

        else:
            return

    @property
    def user_pkgs(self):
        return self.user_pkgs

    @property
    def core_pkgs(self):
        return self.core_pkgs

    @property
    def ip_list(self):
        return self.ip_list

    def add_user_pkg(self, pkg_tag, pkg):
        self.user_pkgs[pkg_tag] = pkg

    def add_core_pkg(self, pkg_tag, pkg):
        self.core_pkgs[pkg_tag] = pkg

    def add_ips(self, ex_ips):
        self.ip_list = list(set(self.ip_list+ex_ips))

    @property
    def scan_mode(self):
        return self.scan_mode

    def set_scan_mode(self, mode):
        if mode not in scan_modes:
            self.scan_mode = 'yum'
        else:
            self.scan_mode = mode

    def add_pkgs(self):
        """
        add packages
        """
        if self.scan_mode=='yum' or self.scan_mode=='rpm':
            try:
                import rpm
            except ImportError:
                return

            ts = rpm.TransactionSet()
            for (pkg_tag, pkg) in self.user_pkgs.items():
                if self.scan_mode == 'yum':
                    name = pkg.name
                if self.scan_mode == 'rpm':
                    name = pkg[rpm.RPMTAG_NAME]

                ### search the yum api of searching package's config files
                mi = ts.dbMatch('name', name)
                for h in mi:
                    # save the config files
                    confs = []

                    fi = h.fiFromHeader()
                    for file in fi:
                        fipath = file[0]

                        if fipath in confs:
                            continue

                        if self.add_conffile(fipath)==False:
                            continue

                        tmpips = get_ips(fipath)
                        self.add_ips(tmpips)

                        confs.append(fipath)

                    # get the epoch
                    #epoch = h['epoch'] if h['epoch'] else 0

                    # add package
                    self.add_package(name, manager='yum', provider='yum', files=confs,
                                description=h['summary'], platform=h['arch'],
                                version=h['version']+'-'+h['release'], vendor=h['vendor'])

        elif self.scan_mode=='apt':
            for (id, pkg) in self.user_pkgs.items():

                # pkg's config files
                confs = []
                fi = pkg.installed_files
                for file in fi:
                    if file in confs: continue

                    if self.add_conffile(file)==False: continue

                    tmpips = get_ips(file)
                    self.add_ips(tmpips)

                    confs.append(file)

                for pkgv in pkg.versions:
                    self.add_package(pkg.shortname, manager='apt', provider='apt', files=confs,
                        description=pkgv.summary, platform=pkgv.architecture, version=pkgv.version)

        elif self.scan_mode=='sub_rpm':
            for (tag, info) in self.user_pkgs.items():
                #'name':name, 'version':ver, 'release':rel, 'arch':arch, 'epoch':epo, 'summary':sum, 'vendor':ven, 'configs':confs

                confs = []
                # parse the ips
                for file in info['configs']:
                    if file in confs: continue

                    if self.add_conffile(file)==False: continue

                    tmpips = get_ips(file)
                    self.add_ips(tmpips)

                    confs.append(file)

                #epoch = info['epoch'] if info['epoch'] else 0
                # add the package
                self.add_package(info['name'], manager='rpm', provider='rpm', files=confs,
                                 description=info['summary'], platform=info['arch'], version=info['version']+'-'+info['release'])

        elif self.scan_mode=='sub_dpkg':
            for (tag, info) in self.user_pkgs.items():
                confs = []
                for file in info['configs']:
                    #{'name':name, 'version':ver, 'arch':arch, 'description':desc, 'configs':conffiles}
                    if file in confs: continue

                    if self.add_conffile(file)==False: continue

                    tmpips = get_ips(file)
                    self.add_ips(tmpips)

                    confs.append(file)

                self.add_package(info['name'], manager='dpkg', provider='dpkg',
                                 files=confs, version=info['version']) class ScannerSource(ScannerBase):

    def scan(self):
        """
        scan repository
        """
        logging.info('searching for repository')

        self.user_repos = {'git':[], 'svn':[], 'hg':[]}    # save all the scm repos

        self.search_scm('/')
        #self.search_scm('svn', '/')
        #self.search_scm('hg', '/')
        self.add_repos()

    def search_scm(self, dir):
        """
        search all scm repositories in local directory dirname
        """
        for dirpath, dirnames, filenames in os.walk(dir):
            for dirname in dirnames:
                if dirname == '.git':
                    scm = 'git'
                elif dirname == '.svn':
                    scm = 'svn'
                elif dirname == '.hg':
                    scm = 'hg'
                else:
                    continue

                if scm=='svn':  #if svn scm, need to check subdirctory
                    head, tail = os.path.split(dirpath)
                    while head and tail:
                        if head in self.user_repos['svn']:
                            break
                        head, tail = os.path.split(head)
                    else:
                        subsvn = [ scmdir for scmdir in self.user_repos['svn'] if dirpath in scmdir ]
                        if len(subsvn)>0:
                            for subdir in subsvn: self.user_repos['svn'].remove(subdir)

                        self.add_local_repo(scm, dirpath)
                else:
                    self.add_local_repo(scm, dirpath)

    def add_repos(self):
        """
        get the repository info and add repo
        """
        for (scm, dirs) in self.user_repos.items():
            for dirname in dirs:
                sources = []    # http/ssh
                branches = []

                try:
                    if scm=='git':
                        p = subprocess.Popen(['-c', 'git --git-dir=' + dirname + '/.git --work-tree=' + dirname+' remote -v'],
                            stdout=subprocess.PIPE, shell=True)
                        for line in p.stdout:
                            src = line.split('\t')[1].split(' ')[0]
                            if src is not None and src not in sources:
                                sources.append(src)

                        # branches
                        p = subprocess.Popen(['-c', 'git --git-dir=' + dirname + '/.git --work-tree=' + dirname + ' branch'],
                            stdout=subprocess.PIPE, shell=True)
                        for line in p.stdout:
                            if line is not None:
                                lst = line.strip().split()
                                for br in lst:
                                    if br is not None and br!='*':
                                        branches.append(br)

                    elif scm=='svn':
                        p = subprocess.Popen(['-c', 'svn info ' + dirname + ' | grep URL | awk \'{print $NF}\''],
                            shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)

                        for source in p.stdout:
                            if source is not None: sources.append(source.strip())

                        ### branches
                    elif scm=='hg':
                        # sources
                        p = subprocess.Popen(['-c', 'hg paths -R' + dirname], shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
                        for line in p.stdout:
                            source = line.split('=')[1].strip()
                            if source is not None and source not in sources: sources.append(source)
                        # branches
                        p = subprocess.Popen(['-c', 'hg branches -R' + dirname], shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
                        for line in p.stdout:
                            branch = line.strip().split(' ')[0]
                            if branch is not None and branch not in branches: branches.append(branch)
                    else:
                        continue

                except OSError:
                    continue

                # add the public info
                s = get_stat(dirname)

                if len(sources)<=0: continue

                self.add_source(sources[0], os.path.basename(dirname), dirname, s[2], s[0], s[1], scm, branches)

    @property
    def user_repos(self):
        """
        local repos
        """
        if user_repos not in self:
            self.user_repos = defaultdict(list)

        return self.user_repos

    def add_local_repo(self, scm, dirname):
        """
        add all repos to local repos
        """
        if scm not in self.user_repos:
            self.user_repos[scm] = []

        self.user_repos[scm].append(dirname) class ScannerRepo(ScannerBase):

    def scan(self):
        """
        scan for repository config files
        """
        logging.info('searching for repository config files')

        if os.path.exists('/etc/yum.repos.d'):
            self.scan_yum()
        elif os.path.exists('/etc/apt'):
            self.scan_apt()

    def scan_yum(self):
        """
        scan yum repo config files
        """

        for dirpath, dirnames, files in os.walk('/etc/yum.repos.d'):
            for file in files:
                root, ext = os.path.splitext(file)
                if ext!='.repo': continue

                self.addfile(os.path.join(dirpath, file), 'yum')

    def scan_apt(self):
        """
        scan apt repo config files
        """

        try:
            with open('/etc/apt/sources.list'): self.addfile('/etc/apt/sources.list', 'apt')
        except IOError:
            return

        if os.path.exists('/etc/apt/sources.list.d'):
            for dirpath, dirnames, files in os.walk('/etc/apt/sources.list.d'):
                for file in files:
                    root, ext = os.path.splitext(file)
                    if ext!='.list': continue

                    self.addfile(os.path.join(dirpath, file), 'apt')


    # add per config file
    def addfile(self, pathname, prov):
        # only plane text file
        if valid_txtfile(pathname) == False:
            return

        # get owner, group and mode
        s = get_stat(pathname)

        # read the config file's content
        c = get_content(pathname)

        # add the config file:
        # checksum, content, group, mode, owner, path, force=False, provider=None,
        # recurse=None, recurselimit=None, source=None
        self.add_repo('md5', c, s[0], s[1], s[2], pathname, provider=prov) class ScannerProcess(ScannerBase):

    def scan(self):
        """
        scan process
        """
        logging.info('searching for process')

        self.get_processes()

    def get_processes(self):
        """
        get all the process info
        """

        try:
            p = subprocess.Popen(['-c', 'ps -eo pid,fuser,s,pcpu,pmem,comm,ppid'], shell=True, stdout=subprocess.PIPE)

            first = True
            for line in p.stdout:
                if first:   # ignore the headline
                    first = False
                    continue

                lst = line.strip().split()

                # check data completeness
                nonlst = [info for info in lst if info is None]
                if len(nonlst)>0: continue

                self.add_proc(lst[0], lst[1], lst[2], lst[3], lst[4], lst[5], lst[6])

        except OSError:
            return

    def get_accounts(self):
        """
        parse the system accounts config file('/etc/passwd')
        """

        try:
            data = open('/etc/passwd').read()
            for line in data:
                att = line.strip().split(':')
                if att[0] is not None and att[2] is not None:
                    self.add_accout(att[2], att[0])
        except IOError:
            return False

        return True

    @property
    def accounts(self):
        if accounts not in self:
            self.accounts = {}

        return self.accounts

    def add_accout(self, uid, name):
        if uid not in self.accounts:
            self.accounts[uid] = name class ScannerBase():
    def __init__(self, packages, files, crons, groups, mounts, hosts, repos, services, sshkeys, users, ips, sources, proces):
        self.packages = packages
        self.files = files
        self.crons = crons
        self.groups = groups
        self.mounts = mounts
        self.hosts = hosts
        self.repos = repos
        self.services = services
        self.keys = sshkeys
        self.users = users
        self.ips = ips
        self.sources = sources
        self.proces = proces

    def scan(self):
        '''
        Implement in ScannerYum, ScannerApt...
        '''
        pass

    @GeneralException
    def subprocess(self, command):
        logging.info('ScannerBase.subprocess(): Command, %s' % str(command))
        try:
            # redirect the err to /dev/null
            devnull = open('/dev/null', 'w')
            p = subprocess.Popen(
                                 command,
                                 close_fds=True,
                                 stdout=subprocess.PIPE,
                                 stderr=devnull
                                 )
            return self.__generator(p.stdout)
        except OSError:
            logging.debug("ScannerBase.subprocess(): commnand failed, command: %s" % str(command))
            return

    @GeneralException
    def __generator(self, stdout):
        for line in stdout:
            yield line

    @GeneralException
    def add_package(self, *args, **kargs):
        _package = Package(*args, **kargs)

        if self.rules:
            # attribute filter
            if _package.attr_filter(self.rules['discard']): return
            # additional filter
            _package.add_filter(self.rules['addition'])

        self.packages[_package.primaryvalue] = _package.prase()

    @GeneralException
    def get_packages(self):
        return self.packages

    @GeneralException
    def add_file(self, *args, **kargs):
        _file = File(*args, **kargs)

        if self.rules:
            if _file.attr_filter(self.rules['discard']): return
            _file.add_filter(self.rules['addition'])

        self.files[_file.primaryvalue] = _file.prase()

    @GeneralException
    def get_files(self):
        return self.files

    @GeneralException
    def add_user(self, *args, **kargs):
        _user = User(*args, **kargs)

        if self.rules:
            if _user.attr_filter(self.rules['discard']): return
            _user.add_filter(self.rules['addition'])

        self.users[_user.primaryvalue] = _user.prase()

    @GeneralException
    def get_users(self):
        return self.users

    @GeneralException
    def add_service(self, *args, **kargs):
        _service = Service(*args, **kargs)

        if self.rules:
            if _service.attr_filter(self.rules['discard']): return
            _service.add_filter(self.rules['addition'])

        self.services[_service.primaryvalue] = _service.prase()

    @GeneralException
    def get_services(self):
        return self.services

    @GeneralException
    def add_repo(self, *args, **kargs):
        _repo = Repository(*args, **kargs)  

        if self.rules:     
            if _repo.attr_filter(self.rules['discard']): return
            _repo.add_filter(self.rules['addition'])

        self.repos[_repo.primaryvalue] = _repo.prase()

    @GeneralException
    def get_repos(self):
        return self.repos

    @GeneralException
    def add_group(self, *args, **kargs):
        _group = Group(*args, **kargs)

        if self.rules:
            if _group.attr_filter(self.rules['discard']): return
            _group.add_filter(self.rules['addition'])

        self.groups[_group.primaryvalue] = _group.prase()

    @GeneralException
    def get_groups(self):
        return self.groups

    @GeneralException
    def add_cron(self, *args, **kargs):
        _cron = Cron(*args, **kargs)

        if self.rules:
            if _cron.attr_filter(self.rules['discard']): return
            _cron.add_filter(self.rules['addition'])

        self.crons[_cron.primaryvalue] = _cron.prase()

    @GeneralException
    def get_crons(self):
        return self.crons

    @GeneralException
    def add_host(self, *args, **kargs):
        _host = Host(*args, **kargs)

        if self.rules:
            if _host.attr_filter(self.rules['discard']): return
            _host.add_filter(self.rules['addition'])

        self.hosts[_host.primaryvalue] = _host.prase()

    @GeneralException
    def get_hosts(self):
        return self.hosts

    @GeneralException
    def add_mount(self, *args, **kargs):
        _mount = Mount(*args, **kargs)

        if self.rules:
            if _mount.attr_filter(self.rules['discard']): return
            _mount.add_filter(self.rules['addition'])

        self.mounts[_mount.primaryvalue] = _mount.prase()

    @GeneralException
    def get_mounts(self):
        return self.mounts

    @GeneralException
    def add_key(self, *args, **kargs):
        _key = SSHKey(*args, **kargs)

        if self.rules:
            if _key.attr_filter(self.rules['discard']): return
            _key.add_filter(self.rules['addition'])
            
        self.keys[_key.primaryvalue] = _key.prase()

    @GeneralException
    def get_keys(self):
        return self.keys

    @GeneralException
    def add_ip(self, mip):
        self.ips.append(mip)

    @GeneralException
    def get_ips(self):
        return self.ips

    @GeneralException
    def add_source(self, *args, **kargs):
        _source = Source(*args, **kargs)

        if self.rules:
            if _source.attr_filter(self.rules['discard']): return
            _source.add_filter(self.rules['addition'])

        self.sources[_source.primaryvalue] = _source.prase()

    @GeneralException
    def get_sources(self):
        return self.sources

    @GeneralException
    def add_proc(self, *args, **kargs):
        _process = Process(*args, **kargs)

        if self.rules:
            if _process.attr_filter(self.rules['discard']): return
            _process.add_filter(self.rules['addition'])

        self.proces[_process.primaryvalue] = _process.prase()

    @GeneralException
    def get_proces(self):
        return self.proces

    @GeneralException
    def init_filter(self, rules=None):
        """
        init the filter rules
        """
        
        self.rules = rules

        if self.rules:  # init the discard and addition rules if not
            if 'discard' not in self.rules: self.rules['discard'] = {}
            if 'addition' not in self.rules: self.rules['addition'] = {}
import logging
import time
from pysa.scanner.actions.file import ScannerFile
from pysa.scanner.actions.gem import ScannerGem
from pysa.scanner.actions.npm import ScannerNpm
from pysa.scanner.actions.php import ScannerPhp
from pysa.scanner.actions.pypi import ScannerPypi
from pysa.scanner.actions.service import ScannerService
from pysa.scanner.actions.host import ScannerHost
from pysa.scanner.actions.mount import ScannerMount
from pysa.scanner.actions.cron import ScannerCron
from pysa.scanner.actions.sshkey import ScannerKey
from pysa.scanner.actions.user import ScannerUser
from pysa.scanner.actions.group import ScannerGroup
from pysa.scanner.actions.package import ScannerPackage
from pysa.scanner.actions.source import ScannerSource
from pysa.scanner.actions.repository import ScannerRepo
from pysa.scanner.actions.process import ScannerProcess
from pysa.scanner.actions.base import ScannerBase
'''
Created on 2013-3-28

    pysa - reverse a complete computer setup
    Copyright (C) 2013  MadeiraCloud Ltd.

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

@author: Ken
'''
#------------------------------------------------------------

#------------------------------------------------------------

class ScannerHandler():

    # stay aware of the order
    handler = {
                "file"      : ScannerFile,
                "gem"       : ScannerGem,
                "npm"       : ScannerNpm,
                "php"       : ScannerPhp,
                "pypi"      : ScannerPypi,
                "service"   : ScannerService,
                "host"      : ScannerHost,
                'user'      : ScannerUser,
                'group'     : ScannerGroup,

                'mount'     : ScannerMount,