from transformers import AutoConfig from transformers.models.auto.tokenization_auto import AutoTokenizer def is_model_on_hub( model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False ) -> tuple[bool, str]: """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses.""" try: config = AutoConfig.from_pretrained( model_name, revision=revision, trust_remote_code=trust_remote_code, token=token ) if test_tokenizer: try: tk = AutoTokenizer.from_pretrained( model_name, revision=revision, trust_remote_code=trust_remote_code, token=token ) except ValueError as e: return (False, f"uses a tokenizer which is not in a transformers release: {e}", None) except Exception as e: return ( False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None, ) return True, None, config except ValueError: return ( False, "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.", None, ) except Exception as e: return False, "was not found on hub!", None