add generation config, fixing warning output

ShaderEval.py

@@ -8,7 +8,7 @@ _CITATION = """\
 @InProceedings{huggingface:module,
 title = {A great new module},
 authors={huggingface, Inc.},
-year={2020}
+year={2023}
 }
 """
 
@@ -28,7 +28,7 @@ from evaluate.evaluation_suite import SubTask
 from datasets import Dataset
 from typing import Any, Callable, Dict, List, Optional, Union # used in .prepare_pipeline()
 import transformers
-from transformers import Pipeline, pipeline
+from transformers import Pipeline, pipeline, GenerationConfig #GenerationConfig to specify greedy and avoid error
 from datasets import load_dataset #used by Suite.run()
 
 # write a custom evaluator, inherent from: https://github.com/huggingface/evaluate/blob/v0.4.0/src/evaluate/evaluator/text_generation.py#L31
@@ -36,7 +36,12 @@ class ReturnGenerationEvaluator(evaluate.TextGenerationEvaluator):
     def __init__(self, task="text-generation", default_metric_name="exact_match", predictions_prefix: str = "generated"):
         super().__init__(task=task, default_metric_name=default_metric_name)
         self.predictions_prefix = predictions_prefix
-    PIPELINE_KWARGS = {"return_full_text":False, "do_sample":False} #these kwargs are for the pipeline call, not the pipeline init.
+
+    greedy_cfg = GenerationConfig(
+        do_sample = False, # default to ensure greedy
+        num_beams = 1, # same as above
+    )
+    PIPELINE_KWARGS = {"return_full_text": False, "generation_config":greedy_cfg} #these kwargs are for the pipeline call, not the pipeline init - but that seems to still work.
     
     # for the pipeline init we need to copy the whole function and add two lines. this still prints errors due to the pad_toke_id = eos_token_id change.
     # from: https://github.com/huggingface/evaluate/blob/v0.4.0/src/evaluate/evaluator/base.py#L375
@@ -98,12 +103,25 @@ class ReturnGenerationEvaluator(evaluate.TextGenerationEvaluator):
         # fixinging default for max_lenght 
         pipe.model.config.max_length = self._resolve_context_lenght(pipe=pipe)
 
-        # specify eos tokens to be all of those that include a ; so we can stop early.
-        self.PIPELINE_KWARGS.update({"eos_token_id": [v for k,v in pipe.tokenizer.vocab.items() if ";" in k]}) #didn't see that this was passed all the way already.
-        # solution found here: https://huggingface.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig.eos_token_id but does it actually work?
+        # update the generation config with information from the pipe
+        self._update_generation_config(pipe)
 
         return pipe
 
+    def _update_generation_config(self, pipe):
+        """
+        Update the generation config with information from the pipe. Sets eos_token_id and pad_token_id.
+        Args:
+            pipe (:class:`~transformers.Pipeline`): we need to access the tokenizer.vocab
+        returns:
+            None
+        """
+        semicolon_token_ids = [v for k,v in pipe.tokenizer.vocab.items() if ";" in k] # this requires the tokenizer, which we only have once a pipe is made.
+        # GenerationConfig.update also exists, but it does only replace, not add kwargs.
+        self.greedy_cfg.eos_token_id = semicolon_token_ids # eos_token_id can be a list, so we give them all possible tokens.
+        self.greedy_cfg.pad_token_id = semicolon_token_ids[0] # pad_token_id has to be an int, so we just take the first one.
+        return None # doesn't do anything?
+
     def _resolve_context_lenght(self, model_or_pipeline=None, pipe=None): #TODO should really copy the typing hints here.
         if isinstance(model_or_pipeline, transformers.GPT2Model): # you are comparing a string here -.-
             return model_or_pipeline.config.n_ctx # how GPT2 models might handle is, seen with 

app.py

@@ -38,17 +38,18 @@ text = """# Welcome to the ShaderEval Suite.
             - The results will be displayed in the **Output** box
             
             ## Todo (feel free to contribute in a Pull Request)
-            - leaderboard 
-            - supporting batches to speed up inference
-            - CER metric (via a custom metric perhaps?)
-            - removing the pad_token warning
-            - adding OpenVINO pipelines for inference, pending on OpenVINO release
+            - [ ] leaderboard 
+            - [ ] supporting batches to speed up inference
+            - [ ] CER metric (via a custom metric perhaps?)
+            - [x] removing the pad_token warning
+            - [ ] adding OpenVINO pipelines for inference, pending on OpenVINO release
             """
 
 
 def run_suite(model_cp, snippet):
     # print(model_cp, snippet)
     results = suite.run(model_cp, snippet)
+    print(results) # so they show up in the logs for me.
     return results[0]
 
 with gr.Blocks() as site: