original_docstring: Run group_files on files in tree appropriately.
docstring: Run group_files on files in tree appropriately.
docstring_tokens: Run group_files on files in tree appropriately .
code: def group_tree(root, config):
      """Run group_files on files in tree appropriately."""
      files = []
      dirs = []
      if root == "/dev/null":
          return []
      for node in os.listdir(root):
          node_path = os.path.join(root, node)
          if node == "mdf.json":
              with open(node_path) as f:
                  try:
                      new_config = json.load(f)
                      logger.debug("Config updating: \n{}".format(new_config))
                  except Exception as e:
                      logger.warning("Error reading config file '{}': {}".format(node_path, str(e)))
                  else:
                      config = mdf_toolbox.dict_merge(new_config, config)
          elif os.path.isfile(node_path):
              files.append(node_path)
          elif os.path.isdir(node_path):
              dirs.append(node_path)
          else:
              logger.debug("Ignoring non-file, non-dir node '{}'".format(node_path))
  
      # Group the files
      # list "groups" is list of dict, each dict contains actual file list + extractor info/config
      groups = []
      # Group by dir overrides other grouping
      if config.get("group_by_dir"):
          groups.append({"files": files,
                         "extractors": [],
                         "params": {}})
      else:
          for format_rules in config.get("known_formats", {}).values():
              format_name_list = format_rules["files"]
              format_groups = {}
              # Check each file for rule matching
              # Match to appropriate group (with same pre/post pattern)
              #   eg a_[match]_b groups with a_[other match]_b but not c_[other match]_d
              for f in files:
                  fname = os.path.basename(f).lower().strip()
                  for format_name in format_name_list:
                      if format_name in fname:
                          pre_post_pattern = fname.replace(format_name, "")
                          if not format_groups.get(pre_post_pattern):
                              format_groups[pre_post_pattern] = []
                          format_groups[pre_post_pattern].append(f)
                          break
              # Remove grouped files from the file list and add groups to the group list
              for g in format_groups.values():
                  for f in g:
                      files.remove(f)
                  group_info = {
                      "files": g,
                      "extractors": format_rules["extractors"],
                      "params": format_rules["params"]
                  }
                  groups.append(group_info)
  
          # NOTE: Keep this grouping last!
          # Default grouping: Each file is a group
          groups.extend([{"files": [f],
                          "extractors": [],
                          "params": {}}
                         for f in files])
  
      [groups.extend(group_tree(d, config)) for d in dirs]
  
      return groups
code_tokens: def group_tree ( root , config ) : files = [ ] dirs = [ ] if root == "/dev/null" : return [ ] for node in os . listdir ( root ) : node_path = os . path . join ( root , node ) if node == "mdf.json" : with open ( node_path ) as f : try : new_config = json . load ( f ) logger . debug ( "Config updating: \n{}" . format ( new_config ) ) except Exception as e : logger . warning ( "Error reading config file '{}': {}" . format ( node_path , str ( e ) ) ) else : config = mdf_toolbox . dict_merge ( new_config , config ) elif os . path . isfile ( node_path ) : files . append ( node_path ) elif os . path . isdir ( node_path ) : dirs . append ( node_path ) else : logger . debug ( "Ignoring non-file, non-dir node '{}'" . format ( node_path ) ) groups = [ ] if config . get ( "group_by_dir" ) : groups . append ( { "files" : files , "extractors" : [ ] , "params" : { } } ) else : for format_rules in config . get ( "known_formats" , { } ) . values ( ) : format_name_list = format_rules [ "files" ] format_groups = { } for f in files : fname = os . path . basename ( f ) . lower ( ) . strip ( ) for format_name in format_name_list : if format_name in fname : pre_post_pattern = fname . replace ( format_name , "" ) if not format_groups . get ( pre_post_pattern ) : format_groups [ pre_post_pattern ] = [ ] format_groups [ pre_post_pattern ] . append ( f ) break for g in format_groups . values ( ) : for f in g : files . remove ( f ) group_info = { "files" : g , "extractors" : format_rules [ "extractors" ] , "params" : format_rules [ "params" ] } groups . append ( group_info ) groups . extend ( [ { "files" : [ f ] , "extractors" : [ ] , "params" : { } } for f in files ] ) [ groups . extend ( group_tree ( d , config ) ) for d in dirs ] return groups
short_docstring: Run group_files on files in tree appropriately.
short_docstring_tokens: Run group_files on files in tree appropriately .
comment: """Run group_files on files in tree appropriately."""$SEP$# Group the files$SEP$# list "groups" is list of dict, each dict contains actual file list + extractor info/config$SEP$# Group by dir overrides other grouping$SEP$# Check each file for rule matching$SEP$# Match to appropriate group (with same pre/post pattern)$SEP$#   eg a_[match]_b groups with a_[other match]_b but not c_[other match]_d$SEP$# Remove grouped files from the file list and add groups to the group list$SEP$# NOTE: Keep this grouping last!$SEP$# Default grouping: Each file is a group