original_docstring:  Remove redundant files and directories. Should be migrated to Pilot2 
docstring: Remove redundant files and directories. Should be migrated to Pilot2
docstring_tokens: Remove redundant files and directories . Should be migrated to Pilot2
code: def removeRedundantFiles(workdir, outputfiles=[]):
      """ Remove redundant files and directories. Should be migrated to Pilot2 """
  
      logger.info("Removing redundant files prior to log creation")
  
      workdir = os.path.abspath(workdir)
  
      dir_list = ["AtlasProduction*",
                  "AtlasPoint1",
                  "AtlasTier0",
                  "buildJob*",
                  "CDRelease*",
                  "csc*.log",
                  "DBRelease*",
                  "EvgenJobOptions",
                  "external",
                  "fort.*",
                  "geant4",
                  "geomDB",
                  "geomDB_sqlite",
                  "home",
                  "o..pacman..o",
                  "pacman-*",
                  "python",
                  "runAthena*",
                  "share",
                  "sources.*",
                  "sqlite*",
                  "sw",
                  "tcf_*",
                  "triggerDB",
                  "trusted.caches",
                  "workdir",
                  "*.data*",
                  "*.events",
                  "*.py",
                  "*.pyc",
                  "*.root*",
                  "JEM",
                  "tmp*",
                  "*.tmp",
                  "*.TMP",
                  "MC11JobOptions",
                  "scratch",
                  "jobState-*-test.pickle",
                  "*.writing",
                  "pwg*",
                  "pwhg*",
                  "*PROC*",
                  "madevent",
                  "HPC",
                  "objectstore*.json",
                  "saga",
                  "radical",
                  "ckpt*"]
  
      # remove core and pool.root files from AthenaMP sub directories
      try:
          cleanupAthenaMP(workdir, outputfiles)
      except Exception, e:
          print("Failed to execute cleanupAthenaMP(): %s" % (e))
  
      # explicitly remove any soft linked archives (.a files) since they will be dereferenced by the tar command (--dereference option)
      matches = []
      import fnmatch
      for root, dirnames, filenames in os.walk(workdir):
          for filename in fnmatch.filter(filenames, '*.a'):
              matches.append(os.path.join(root, filename))
      for root, dirnames, filenames in os.walk(os.path.dirname(workdir)):
          for filename in fnmatch.filter(filenames, 'EventService_premerge_*.tar'):
              matches.append(os.path.join(root, filename))
      if matches != []:
          for f in matches:
              remove(f)
      # else:
      #    print("Found no archive files")
  
      # note: these should be partitial file/dir names, not containing any wildcards
      exceptions_list = ["runargs", "runwrapper", "jobReport", "log."]
  
      to_delete = []
      for _dir in dir_list:
          files = glob(os.path.join(workdir, _dir))
          exclude = []
  
          if files:
              for exc in exceptions_list:
                  for f in files:
                      if exc in f:
                          exclude.append(os.path.abspath(f))
  
              _files = []
              for f in files:
                  if not f in exclude:
                      _files.append(os.path.abspath(f))
              to_delete += _files
  
      exclude_files = []
      for of in outputfiles:
          exclude_files.append(os.path.join(workdir, of))
      for f in to_delete:
          if not f in exclude_files:
              remove(f)
  
      # run a second pass to clean up any broken links
      broken = []
      for root, dirs, files in os.walk(workdir):
          for filename in files:
              path = os.path.join(root, filename)
              if os.path.islink(path):
                  target_path = os.readlink(path)
                  # Resolve relative symlinks
                  if not os.path.isabs(target_path):
                      target_path = os.path.join(os.path.dirname(path), target_path)
                  if not os.path.exists(target_path):
                      broken.append(path)
              else:
                  # If it's not a symlink we're not interested.
                  continue
  
      if broken:
          for p in broken:
              remove(p)
  
      return 0
code_tokens: def removeRedundantFiles ( workdir , outputfiles = [ ] ) : logger . info ( "Removing redundant files prior to log creation" ) workdir = os . path . abspath ( workdir ) dir_list = [ "AtlasProduction*" , "AtlasPoint1" , "AtlasTier0" , "buildJob*" , "CDRelease*" , "csc*.log" , "DBRelease*" , "EvgenJobOptions" , "external" , "fort.*" , "geant4" , "geomDB" , "geomDB_sqlite" , "home" , "o..pacman..o" , "pacman-*" , "python" , "runAthena*" , "share" , "sources.*" , "sqlite*" , "sw" , "tcf_*" , "triggerDB" , "trusted.caches" , "workdir" , "*.data*" , "*.events" , "*.py" , "*.pyc" , "*.root*" , "JEM" , "tmp*" , "*.tmp" , "*.TMP" , "MC11JobOptions" , "scratch" , "jobState-*-test.pickle" , "*.writing" , "pwg*" , "pwhg*" , "*PROC*" , "madevent" , "HPC" , "objectstore*.json" , "saga" , "radical" , "ckpt*" ] try : cleanupAthenaMP ( workdir , outputfiles ) except Exception , e : print ( "Failed to execute cleanupAthenaMP(): %s" % ( e ) ) matches = [ ] import fnmatch for root , dirnames , filenames in os . walk ( workdir ) : for filename in fnmatch . filter ( filenames , '*.a' ) : matches . append ( os . path . join ( root , filename ) ) for root , dirnames , filenames in os . walk ( os . path . dirname ( workdir ) ) : for filename in fnmatch . filter ( filenames , 'EventService_premerge_*.tar' ) : matches . append ( os . path . join ( root , filename ) ) if matches != [ ] : for f in matches : remove ( f ) exceptions_list = [ "runargs" , "runwrapper" , "jobReport" , "log." ] to_delete = [ ] for _dir in dir_list : files = glob ( os . path . join ( workdir , _dir ) ) exclude = [ ] if files : for exc in exceptions_list : for f in files : if exc in f : exclude . append ( os . path . abspath ( f ) ) _files = [ ] for f in files : if not f in exclude : _files . append ( os . path . abspath ( f ) ) to_delete += _files exclude_files = [ ] for of in outputfiles : exclude_files . append ( os . path . join ( workdir , of ) ) for f in to_delete : if not f in exclude_files : remove ( f ) broken = [ ] for root , dirs , files in os . walk ( workdir ) : for filename in files : path = os . path . join ( root , filename ) if os . path . islink ( path ) : target_path = os . readlink ( path ) if not os . path . isabs ( target_path ) : target_path = os . path . join ( os . path . dirname ( path ) , target_path ) if not os . path . exists ( target_path ) : broken . append ( path ) else : continue if broken : for p in broken : remove ( p ) return 0
short_docstring: Remove redundant files and directories.
short_docstring_tokens: Remove redundant files and directories .
comment: """ Remove redundant files and directories. Should be migrated to Pilot2 """$SEP$# remove core and pool.root files from AthenaMP sub directories$SEP$# explicitly remove any soft linked archives (.a files) since they will be dereferenced by the tar command (--dereference option)$SEP$# else:$SEP$#    print("Found no archive files")$SEP$# note: these should be partitial file/dir names, not containing any wildcards$SEP$# run a second pass to clean up any broken links$SEP$# Resolve relative symlinks$SEP$# If it's not a symlink we're not interested.