messege

MAR-INF/MANIFEST.json

@@ -0,0 +1,11 @@
+{
+  "createdOn": "09/06/2022 17:16:36",
+  "runtime": "python",
+  "model": {
+    "modelName": "gpt-2-ko-small-finetune",
+    "serializedFile": "pytorch_model.bin",
+    "handler": "TextGenerationHandlerForString.py",
+    "modelVersion": "1.0"
+  },
+  "archiverVersion": "0.5.1"
+}

TextGenerationHandlerForString.py

@@ -0,0 +1,89 @@
+import gc
+import json
+
+import torch
+from ts.torch_handler.base_handler import BaseHandler
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class TextGenerationHandlerForString(BaseHandler):
+    def __init__(self):
+        super(TextGenerationHandlerForString, self).__init__()
+        self.model = None
+        self.tokenizer = None
+        self.device = None
+        self.task_config = None
+        self.initialized = False
+
+    def load_model(self, model_dir):
+        if self.device.type == "cuda":
+            self.model = AutoModelForCausalLM.from_pretrained(model_dir, torch_dtype="auto", low_cpu_mem_usage=True)
+            if self.model.dtype == torch.float32:
+                self.model = self.model.half()
+        else:
+            self.model = AutoModelForCausalLM.from_pretrained(model_dir, torch_dtype="auto")
+        self.tokenizer = AutoTokenizer.from_pretrained(model_dir)
+        try:
+            self.task_config = self.model.config.task_specific_params["text-generation"]
+        except Exception:
+            self.task_config = {}
+        # TODO: Need to compare performance
+        self.model.to(self.device, non_blocking=True)
+
+    def initialize(self, ctx):
+        self.manifest = ctx.manifest
+        properties = ctx.system_properties
+        model_dir = properties.get("model_dir")
+        self.device = torch.device(
+            "cuda:" + str(properties.get("gpu_id"))
+            if torch.cuda.is_available()
+            else "cpu"
+        )
+        self.load_model(model_dir)
+        self.model.eval()
+        self.initialized = True
+
+    def preprocess(self, requests):
+        input_batch = {}
+        for idx, data in enumerate(requests):
+            input_batch["input_text"] = data.get("body").get("text")
+            input_batch["num_samples"] = data.get("body").get("num_samples")
+            input_batch["length"] = data.get("body").get("length")
+        del requests
+        gc.collect()
+        return input_batch
+
+    def inference(self, input_batch):
+        input_text = input_batch["input_text"]
+        length = input_batch["length"]
+        num_samples = input_batch["num_samples"]
+        input_ids = self.tokenizer.encode(input_text, return_tensors="pt").to(
+            self.device
+        )
+        self.task_config["max_length"] = length
+        self.task_config["num_return_sequences"] = num_samples
+        inference_output = self.model.generate(input_ids, **self.task_config)
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        del input_batch
+        gc.collect()
+        return inference_output
+
+    def postprocess(self, inference_output):
+        output = self.tokenizer.batch_decode(
+            inference_output.tolist(), skip_special_tokens=True
+        )
+        del inference_output
+        gc.collect()
+        return [json.dumps(output, ensure_ascii=False)]
+
+    def handle(self, data, context):
+        self.context = context
+        data = self.preprocess(data)
+        data = self.inference(data)
+        data = self.postprocess(data)
+        return data

config.json

@@ -0,0 +1,51 @@
+{
+  "_name_or_path": "/model",
+  "_num_labels": 1,
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "author": "Heewon Jeon([email protected])",
+  "bos_token_id": 0,
+  "created_date": "2021-04-28",
+  "embd_pdrop": 0.1,
+  "eos_token_id": 1,
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_epsilon": 1e-05,
+  "license": "CC-BY-NC-SA 4.0",
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "pad_token_id": 3,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.13.0",
+  "use_cache": true,
+  "vocab_size": 51200
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f67056887c88024a8cc3de7c99e259097b8baa5249490c50830700b4891d4aa4
+size 513300713

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token": "<mask>"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "add_prefix_space": false, "pad_token": "<pad>", "mask_token": "<mask>", "special_tokens_map_file": null, "name_or_path": "/model", "tokenizer_class": "GPT2Tokenizer"}