Spaces:
Running
Running
File size: 1,210 Bytes
45311fe |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 |
import sys
import logging
import datasets
import transformers
def init_logger(log_file, log_level, dist_rank):
datasets.utils.logging.set_verbosity(log_level)
transformers.utils.logging.set_verbosity(log_level)
transformers.utils.logging.enable_default_handler()
transformers.utils.logging.enable_explicit_format()
datasets.utils.logging.disable_propagation()
# transformers.utils.logging.enable_propagation()
logger = logging.getLogger("")
log_format = logging.Formatter(fmt="[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
logger.setLevel(log_level)
console_handler = logging.StreamHandler(sys.stderr)
console_handler.setFormatter(log_format)
logger.addHandler(console_handler)
# transformer_logger = logging.getLogger("transformers")
# transformer_logger.handlers = []
# transformer_logger.propagate = True
if dist_rank in [-1, 0]:
file_handler = logging.FileHandler(log_file, mode="a")
file_handler.setLevel(log_level)
file_handler.setFormatter(log_format)
logger.addHandler(file_handler)
logging.getLogger("transformers").addHandler(file_handler)
|