git init

language/bert/squad_QSL.py

+# coding=utf-8
+# Copyright 2021 Arm Limited and affiliates.
+# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved.
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pickle
+import mlperf_loadgen as lg
+from create_squad_data import read_squad_examples, convert_examples_to_features
+from transformers import BertTokenizer
+import os
+import sys
+
+sys.path.insert(0, os.getcwd())
+
+
+# To support feature cache.
+
+max_seq_length = 384
+max_query_length = 64
+doc_stride = 128
+
+
+class SQuAD_v1_QSL:
+    def __init__(
+        self,
+        total_count_override=None,
+        perf_count_override=None,
+        cache_path="eval_features.pickle",
+    ):
+        print("Constructing QSL...")
+        eval_features = []
+        # Load features if cached, convert from examples otherwise.
+        if os.path.exists(cache_path):
+            print("Loading cached features from '%s'..." % cache_path)
+            with open(cache_path, "rb") as cache_file:
+                eval_features = pickle.load(cache_file)
+        else:
+            print(
+                "No cached features at '%s'... converting from examples..." % cache_path
+            )
+
+            print("Creating tokenizer...")
+            vocab_file = os.environ.get("VOCAB_FILE")
+            if not vocab_file:
+                vocab_file = "build/data/bert_tf_v1_1_large_fp32_384_v2/vocab.txt"
+            tokenizer = BertTokenizer(vocab_file)
+
+            print("Reading examples...")
+            dataset_file = os.environ.get("DATASET_FILE")
+            if not dataset_file:
+                dataset_file = "build/data/dev-v1.1.json"
+            eval_examples = read_squad_examples(
+                input_file=dataset_file,
+                is_training=False,
+                version_2_with_negative=False,
+            )
+
+            print("Converting examples to features...")
+
+            def append_feature(feature):
+                eval_features.append(feature)
+
+            convert_examples_to_features(
+                examples=eval_examples,
+                tokenizer=tokenizer,
+                max_seq_length=max_seq_length,
+                doc_stride=doc_stride,
+                max_query_length=max_query_length,
+                is_training=False,
+                output_fn=append_feature,
+                verbose_logging=False,
+            )
+
+            print("Caching features at '%s'..." % cache_path)
+            with open(cache_path, "wb") as cache_file:
+                pickle.dump(eval_features, cache_file)
+
+        self.eval_features = eval_features
+        self.count = total_count_override or len(self.eval_features)
+        self.perf_count = perf_count_override or self.count
+        self.qsl = lg.ConstructQSL(
+            self.count,
+            self.perf_count,
+            self.load_query_samples,
+            self.unload_query_samples,
+        )
+        print("Finished constructing QSL.")
+
+    def load_query_samples(self, sample_list):
+        pass
+
+    def unload_query_samples(self, sample_list):
+        pass
+
+    def get_features(self, sample_id):
+        return self.eval_features[sample_id]
+
+    def __del__(self):
+        print("Finished destroying QSL.")
+
+
+def get_squad_QSL(total_count_override=None, perf_count_override=None):
+    return SQuAD_v1_QSL(total_count_override, perf_count_override)

language/bert/tf_SUT.py

+# coding=utf-8
+# Copyright 2021 Arm Limited and affiliates.
+# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved.
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import array
+import os
+import sys
+
+sys.path.insert(
+    0,
+    os.path.join(
+        os.getcwd(), "DeepLearningExamples", "TensorFlow", "LanguageModeling", "BERT"
+    ),
+)
+sys.path.insert(0, os.getcwd())
+
+try:
+    from squad_QSL import get_squad_QSL
+    from tensorflow.python.platform import gfile
+    import tensorflow as tf
+    import numpy as np
+    import mlperf_loadgen as lg
+except ImportError:
+    raise Exception("Error importing local modules")
+
+
+class BERT_TF_SUT:
+    def __init__(self, args):
+        print("Loading TF model...")
+        infer_config = tf.compat.v1.ConfigProto()
+        infer_config.intra_op_parallelism_threads = (
+            int(os.environ["TF_INTRA_OP_PARALLELISM_THREADS"])
+            if "TF_INTRA_OP_PARALLELISM_THREADS" in os.environ
+            else os.cpu_count()
+        )
+        infer_config.inter_op_parallelism_threads = (
+            int(os.environ["TF_INTER_OP_PARALLELISM_THREADS"])
+            if "TF_INTER_OP_PARALLELISM_THREADS" in os.environ
+            else os.cpu_count()
+        )
+        infer_config.use_per_session_threads = 1
+        self.sess = tf.compat.v1.Session(config=infer_config)
+        model_file = os.environ.get(
+            "ML_MODEL_FILE_WITH_PATH",
+            "build/data/bert_tf_v1_1_large_fp32_384_v2/model.pb",
+        )
+        with gfile.FastGFile(model_file, "rb") as f:
+            graph_def = tf.compat.v1.GraphDef()
+            graph_def.ParseFromString(f.read())
+            self.sess.graph.as_default()
+            tf.import_graph_def(graph_def, name="")
+
+        print("Constructing SUT...")
+        self.sut = lg.ConstructSUT(self.issue_queries, self.flush_queries)
+        print("Finished constructing SUT.")
+
+        self.qsl = get_squad_QSL(args.max_examples)
+
+    def issue_queries(self, query_samples):
+        for i in range(len(query_samples)):
+            eval_features = self.qsl.get_features(query_samples[i].index)
+            input_ids = np.array([eval_features.input_ids])
+            input_mask = np.array([eval_features.input_mask])
+            segment_ids = np.array([eval_features.segment_ids])
+            feeds = {
+                "input_ids:0": input_ids,
+                "input_mask:0": input_mask,
+                "segment_ids:0": segment_ids,
+            }
+            result = self.sess.run(["logits:0"], feed_dict=feeds)
+
+            logits = [float(x) for x in result[0].flat]
+            response_array = array.array(
+                "B", np.array(logits).astype(np.float32).tobytes()
+            )
+            bi = response_array.buffer_info()
+            response = lg.QuerySampleResponse(
+                query_samples[i].id, bi[0], bi[1])
+            lg.QuerySamplesComplete([response])
+
+    def flush_queries(self):
+        pass
+
+    def __del__(self):
+        print("Finished destroying SUT.")
+
+
+def get_tf_sut(args):
+    return BERT_TF_SUT(args)

language/bert/tf_estimator_SUT.py

+# coding=utf-8
+# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved.
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import array
+import json
+import os
+import sys
+
+sys.path.insert(
+    0,
+    os.path.join(
+        os.getcwd(), "DeepLearningExamples", "TensorFlow", "LanguageModeling", "BERT"
+    ),
+)
+sys.path.insert(0, os.getcwd())
+
+try:
+    from squad_QSL import get_squad_QSL
+    import tensorflow as tf
+    import numpy as np
+    import modeling
+    import mlperf_loadgen as lg
+except ImportError:
+    raise Exception("Error importing local modules")
+
+
+# Allow TF to increase GPU memory usage dynamically to prevent cuBLAS init
+# problems.
+config = tf.compat.v1.ConfigProto()
+config.gpu_options.allow_growth = True
+session = tf.compat.v1.Session(config=config)
+
+
+class BERT_TF_ESTIMATOR_SUT:
+    def __init__(self, batch_size=8):
+        print("Loading TF model...")
+        bert_config = modeling.BertConfig.from_json_file("bert_config.json")
+
+        model_fn = self.model_fn_builder(
+            bert_config=bert_config,
+            init_checkpoint=os.environ.get(
+                "ML_MODEL_FILE_WITH_PATH",
+                "build/data/bert_tf_v1_1_large_fp32_384_v2/model.ckpt-5474",
+            ),
+        )
+
+        self.estimator = tf.estimator.Estimator(model_fn=model_fn)
+        self.batch_size = batch_size
+
+        print("Constructing SUT...")
+        self.sut = lg.ConstructSUT(self.issue_queries, self.flush_queries)
+        print("Finished constructing SUT.")
+
+        self.qsl = get_squad_QSL()
+
+    def issue_queries(self, query_samples):
+        input_ids = np.zeros((len(query_samples), 1, 384), dtype=np.int32)
+        input_mask = np.zeros((len(query_samples), 1, 384), dtype=np.int32)
+        segment_ids = np.zeros((len(query_samples), 1, 384), dtype=np.int32)
+        for sample_idx in range(len(query_samples)):
+            eval_features = self.qsl.get_features(
+                query_samples[sample_idx].index)
+            input_ids[sample_idx, ...] = np.array(eval_features.input_ids)
+            input_mask[sample_idx, ...] = np.array(eval_features.input_mask)
+            segment_ids[sample_idx, ...] = np.array(eval_features.segment_ids)
+
+        def input_fn():
+            inputs = {
+                "input_ids": input_ids,
+                "input_mask": input_mask,
+                "segment_ids": segment_ids,
+            }
+            return tf.data.Dataset.from_tensor_slices(inputs)
+
+        for i, result in enumerate(self.estimator.predict(input_fn)):
+            logits = [float(x) for x in result["logits"].flat]
+            response_array = array.array(
+                "B", np.array(logits).astype(np.float32).tobytes()
+            )
+            bi = response_array.buffer_info()
+            response = lg.QuerySampleResponse(
+                query_samples[i].id, bi[0], bi[1])
+            lg.QuerySamplesComplete([response])
+
+    def flush_queries(self):
+        pass
+
+    def __del__(self):
+        print("Finished destroying SUT.")
+
+    def create_model(
+        self,
+        bert_config,
+        is_training,
+        input_ids,
+        input_mask,
+        segment_ids,
+        use_one_hot_embeddings,
+    ):
+        """Creates a classification model."""
+        model = modeling.BertModel(
+            config=bert_config,
+            is_training=is_training,
+            input_ids=input_ids,
+            input_mask=input_mask,
+            token_type_ids=segment_ids,
+            use_one_hot_embeddings=use_one_hot_embeddings,
+            compute_type=tf.float32,
+        )
+
+        final_hidden = model.get_sequence_output()
+
+        final_hidden_shape = modeling.get_shape_list(
+            final_hidden, expected_rank=3)
+        batch_size = final_hidden_shape[0]
+        seq_length = final_hidden_shape[1]
+        hidden_size = final_hidden_shape[2]
+
+        output_weights = tf.get_variable(
+            "cls/squad/output_weights",
+            [2, hidden_size],
+            initializer=tf.truncated_normal_initializer(stddev=0.02),
+        )
+
+        output_bias = tf.get_variable(
+            "cls/squad/output_bias", [2], initializer=tf.zeros_initializer()
+        )
+
+        final_hidden_matrix = tf.reshape(
+            final_hidden, [batch_size * seq_length, hidden_size]
+        )
+        logits = tf.matmul(
+            final_hidden_matrix,
+            output_weights,
+            transpose_b=True)
+        logits = tf.nn.bias_add(logits, output_bias)
+
+        logits = tf.reshape(logits, [batch_size, seq_length, 2])
+        return logits
+
+        # logits = tf.transpose(logits, [2, 0, 1])
+
+        # unstacked_logits = tf.unstack(logits, axis=0, name='unstack')
+
+        # (start_logits, end_logits) = (unstacked_logits[0], unstacked_logits[1])
+
+        # return (start_logits, end_logits)
+
+    def model_fn_builder(
+        self, bert_config, init_checkpoint, use_one_hot_embeddings=False
+    ):
+        """Returns `model_fn` closure for Estimator."""
+
+        def model_fn(features, labels):  # pylint: disable=unused-argument
+            input_ids = features["input_ids"]
+            input_mask = features["input_mask"]
+            segment_ids = features["segment_ids"]
+
+            logits = self.create_model(
+                bert_config=bert_config,
+                is_training=False,
+                input_ids=input_ids,
+                input_mask=input_mask,
+                segment_ids=segment_ids,
+                use_one_hot_embeddings=use_one_hot_embeddings,
+            )
+
+            tvars = tf.compat.v1.trainable_variables()
+
+            initialized_variable_names = {}
+            (assignment_map, initialized_variable_names) = (
+                modeling.get_assignment_map_from_checkpoint(
+                    tvars, init_checkpoint)
+            )
+
+            tf.compat.v1.train.init_from_checkpoint(
+                init_checkpoint, assignment_map)
+
+            predictions = {"logits": logits}
+            output_spec = tf.estimator.EstimatorSpec(
+                mode=tf.estimator.ModeKeys.PREDICT, predictions=predictions
+            )
+
+            return output_spec
+
+        return model_fn
+
+
+def get_tf_estimator_sut():
+    return BERT_TF_ESTIMATOR_SUT()

language/bert/tf_freeze_bert.py

+# coding=utf-8
+# Copyright (c) 2020 NVIDIA CORPORATION. All rights reserved.
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tensorflow.tools.graph_transforms import TransformGraph
+from tensorflow.python.tools import optimize_for_inference_lib
+from tensorflow.python.framework import graph_util
+from tensorflow.python.framework import dtypes
+import tensorflow as tf
+import numpy as np
+import modeling
+import array
+import json
+import os
+import sys
+
+sys.path.insert(
+    0,
+    os.path.join(
+        os.getcwd(), "DeepLearningExamples", "TensorFlow", "LanguageModeling", "BERT"
+    ),
+)
+sys.path.insert(0, os.getcwd())
+
+
+def save_model(fname, sess, graph=None):
+    def save(fname, graph_def):
+        pass
+        with tf.Graph().as_default() as g:
+            tf.import_graph_def(graph_def, name="")
+            graph_def = g.as_graph_def(add_shapes=True)
+        tf.train.write_graph(graph_def, ".", fname, as_text=False)
+
+    if graph is None:
+        graph_def = sess.graph_def
+    else:
+        graph_def = graph.as_graph_def(add_shapes=True)
+
+    input_nodes = [
+        "IteratorGetNext:0",
+        "IteratorGetNext:1",
+        "IteratorGetNext:2"]
+    output_nodes = ["logits"]
+
+    graph_def = graph_util.convert_variables_to_constants(
+        sess=sess, input_graph_def=graph_def, output_node_names=output_nodes
+    )
+    graph_def = graph_util.remove_training_nodes(
+        graph_def, protected_nodes=output_nodes
+    )
+    graph_def = optimize_for_inference_lib.optimize_for_inference(
+        graph_def, [], output_nodes, dtypes.float32.as_datatype_enum
+    )
+
+    transforms = [
+        "remove_nodes(op=Identity, op=StopGradient)",
+        "fold_batch_norms",
+        "fold_old_batch_norms",
+    ]
+    graph_def = TransformGraph(
+        graph_def,
+        input_nodes,
+        output_nodes,
+        transforms)
+    save("build/data/bert_tf_v1_1_large_fp32_384_v2/model.pb", graph_def)
+
+
+def create_model(
+    bert_config, is_training, input_ids, input_mask, segment_ids, use_one_hot_embeddings
+):
+    """Creates a classification model."""
+    model = modeling.BertModel(
+        config=bert_config,
+        is_training=is_training,
+        input_ids=input_ids,
+        input_mask=input_mask,
+        token_type_ids=segment_ids,
+        use_one_hot_embeddings=use_one_hot_embeddings,
+        compute_type=tf.float32,
+    )
+
+    final_hidden = model.get_sequence_output()
+
+    final_hidden_shape = modeling.get_shape_list(final_hidden, expected_rank=3)
+    batch_size = final_hidden_shape[0]
+    seq_length = final_hidden_shape[1]
+    hidden_size = final_hidden_shape[2]
+
+    output_weights = tf.get_variable(
+        "cls/squad/output_weights",
+        [2, hidden_size],
+        initializer=tf.truncated_normal_initializer(stddev=0.02),
+    )
+
+    output_bias = tf.get_variable(
+        "cls/squad/output_bias", [2], initializer=tf.zeros_initializer()
+    )
+
+    final_hidden_matrix = tf.reshape(
+        final_hidden, [batch_size * seq_length, hidden_size]
+    )
+    logits = tf.matmul(final_hidden_matrix, output_weights, transpose_b=True)
+    logits = tf.nn.bias_add(logits, output_bias)
+
+    logits = tf.reshape(logits, [batch_size, seq_length, 2], name="logits")
+    return logits
+
+
+def main():
+    bert_config = modeling.BertConfig.from_json_file("bert_config.json")
+    init_checkpoint = "build/data/bert_tf_v1_1_large_fp32_384_v2/model.ckpt-5474"
+
+    input_ids = tf.placeholder(tf.int32, shape=(1, 384), name="input_ids")
+    input_mask = tf.placeholder(tf.int32, shape=(1, 384), name="input_mask")
+    segment_ids = tf.placeholder(tf.int32, shape=(1, 384), name="segment_ids")
+
+    logits = create_model(
+        bert_config=bert_config,
+        is_training=False,
+        input_ids=input_ids,
+        input_mask=input_mask,
+        segment_ids=segment_ids,
+        use_one_hot_embeddings=False,
+    )
+
+    tvars = tf.compat.v1.trainable_variables()
+
+    initialized_variable_names = {}
+    (assignment_map, initialized_variable_names) = (
+        modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint)
+    )
+
+    tf.compat.v1.train.init_from_checkpoint(init_checkpoint, assignment_map)
+
+    predictions = {"logits": logits}
+    output_spec = tf.estimator.EstimatorSpec(
+        mode=tf.estimator.ModeKeys.PREDICT, predictions=predictions
+    )
+
+    with tf.Session() as sess:
+        sess.run(tf.global_variables_initializer())
+        save_model("bert_large_nv.pb", sess)
+
+
+if __name__ == "__main__":
+    main()

language/bert/user.conf

+# Please set these fields depending on the performance of your system to
+# override default LoadGen settings.
+*.SingleStream.target_latency = 10
+*.MultiStream.target_latency = 80
+*.Server.target_qps = 1.0
+*.Offline.target_qps = 32.0
\ No newline at end of file

language/gpt-j/DATASETS_MODELS.md

+# Datasets
+
+This is a comprehensive list of public datasets and models used by this repository.
+
+| Name (Link/Source) | Framework | Use Case |
+|--------------------| --------- | -------- |
+| [cnn_dailymail (Hugging Face)](https://huggingface.co/datasets/cnn_dailymail) | PyTorch | Text Summarization |
+| [gpt-j-6b (Hugging Face)](https://huggingface.co/EleutherAI/gpt-j-6b) | PyTorch | Text Summarization |

language/gpt-j/GPTJ_QDL.py

+# For QDL
+import threading
+import requests
+from time import sleep
+import mlperf_loadgen as lg
+import os
+import numpy as np
+import array
+import time
+
+
+class GPTJ_QDL:
+    """QDL acting as a proxy to the SUT.
+    This QDL communicates with the SUT via HTTP.
+    It uses two endpoints to communicate with the SUT:
+    - /predict/ : Send a query to the SUT and get a response.
+    - /getname/ : Get the name of the SUT. Send a getname to the SUT and get a response.
+    """
+
+    def __init__(self, qsl, sut_server_addr: list, scenario: str):
+        self.scenario = scenario
+        self.sut_server_addr = sut_server_addr
+        self.num_nodes = len(sut_server_addr)
+        self.qsl = qsl
+
+        # Construct QDL from the python binding
+        self.qdl = lg.ConstructQDL(
+            self.issue_query, self.flush_queries, self.client_get_name
+        )
+        print("Finished constructing QDL!")
+
+        # For round robin between the SUTs:
+        self.next_sut_id = 0
+        self.lock = threading.Lock()
+
+    def issue_query(self, query_samples):
+        """Process the query to send to the SUT"""
+        threading.Thread(
+            target=self.process_query_async,
+            args=[query_samples]).start()
+
+    def flush_queries(self):
+        """Flush the queries. Dummy implementation."""
+        pass
+
+    def process_query_async(self, query_samples):
+        """
+        This function is called by the Loadgen in a separate thread.
+        It is responsible for
+            1. Creating a query for the SUT, by reading the features from the QSL.
+            2. Sending the query to the SUT.
+            3. Waiting for the response from the SUT.
+            4. Deserializing the response.
+            5. Calling mlperf_loadgen.QuerySamplesComplete(query_samples, response)
+        Args:
+            query_samples: A list of QuerySample objects.
+        """
+
+        max_num_threads = int(
+            os.environ.get(
+                "CM_MAX_NUM_THREADS",
+                os.cpu_count()))
+        if self.scenario == "Offline":
+            # Client sends multiple requests using threads
+            # Pause when active thread equals the set max number of threads
+            # Only sends next request after recieving respose from server to
+            # any of the currently active threads
+            print("Executing Offline scenario!")
+            for i in range(len(query_samples)):
+                index = query_samples[i].index
+                input_ids_tensor = self.qsl.data_object.source_encoded_input_ids[index]
+                input_masks_tensor = self.qsl.data_object.source_encoded_attn_masks[
+                    index
+                ]
+                text = self.qsl.data_object.sources[index]
+                query = {
+                    "input_text": text,
+                    "input_ids_tensor": input_ids_tensor.tolist(),
+                    "input_masks_tensor": input_masks_tensor.tolist(),
+                }
+                n = threading.active_count()
+                while n >= max_num_threads:
+                    sleep(0.0001)
+                    n = threading.active_count()
+                threading.Thread(
+                    target=self.client_predict_worker, args=[
+                        query, query_samples[i].id]
+                ).start()
+        if self.scenario == "Server":
+            # Client sends request to server
+            # Number of samples can vary based on Poisson distribution
+            index = query_samples[0].index
+            input_ids_tensor = self.qsl.data_object.source_encoded_input_ids[index]
+            input_masks_tensor = self.qsl.data_object.source_encoded_attn_masks[index]
+            text = self.qsl.data_object.sources[index]
+            query = {
+                "input_text": text,
+                "input_ids_tensor": input_ids_tensor.tolist(),
+                "input_masks_tensor": input_masks_tensor.tolist(),
+            }
+            self.client_predict_worker(query, query_samples[0].id)
+
+    def get_sut_id_round_robin(self):
+        """Get the SUT id in round robin."""
+        with self.lock:
+            res = self.next_sut_id
+            self.next_sut_id = (self.next_sut_id + 1) % self.num_nodes
+        return res
+
+    def client_predict_worker(self, query, query_id):
+        """Serialize the query, send it to the SUT in round robin, and return the deserialized response."""
+        url = "{}/predict/".format(
+            self.sut_server_addr[self.get_sut_id_round_robin()])
+        responses = []
+        # Start the timer
+        startTime = time.time()
+        # Sending the request to the server through POST method
+        # Upon recieving the response, it is stored in response variable
+        response = requests.post(url, json={"query": query})
+        # Measure the response time
+        endTime = time.time()
+        # calculate the latency
+        print(f"Latency = {endTime-startTime}")
+        output = response.json()["result"]
+        response_text = output["response_text"]
+        print(query["input_text"])
+        print(response_text)
+
+        output_batch = np.array(output["pred_output_batch"]).astype(np.int32)
+        response_array = array.array("B", output_batch.tobytes())
+        bi = response_array.buffer_info()
+
+        responses.append(lg.QuerySampleResponse(query_id, bi[0], bi[1]))
+        lg.QuerySamplesComplete(responses)
+
+    def client_get_name(self):
+        """Get the name of the SUT from ALL the SUTS."""
+        if len(self.sut_server_addr) == 1:
+            return requests.post(
+                f"{self.sut_server_addr[0]}/getname/").json()["name"]
+
+        sut_names = [
+            requests.post(f"{addr}/getname/").json()["name"]
+            for addr in self.sut_server_addr
+        ]
+        return "Multi-node SUT: " + ", ".join(sut_names)
+
+    def __del__(self):
+        lg.DestroyQDL(self.qdl)

language/gpt-j/GPTJ_QSL.py

+import mlperf_loadgen as lg
+from dataset import Dataset
+
+
+class GPTJ_QSL:
+    def __init__(self, dataset_path: str, max_examples: int):
+        self.dataset_path = dataset_path
+        self.max_examples = max_examples
+
+        # creating data object for QSL
+        self.data_object = Dataset(
+            self.dataset_path, total_count_override=self.max_examples
+        )
+
+        # construct QSL from python binding
+        self.qsl = lg.ConstructQSL(
+            self.data_object.count,
+            self.data_object.perf_count,
+            self.data_object.LoadSamplesToRam,
+            self.data_object.UnloadSamplesFromRam,
+        )
+
+        print("Finished constructing QSL.")
+
+
+def get_GPTJ_QSL(dataset_path: str, max_examples: int):
+    return GPTJ_QSL(dataset_path, max_examples)

language/gpt-j/LICENSE

+                                Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

language/gpt-j/README.md

+# GPT-J Reference Implementation
+
+Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks/language/gpt-j) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker.
+
+Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks/language/gpt-j/) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker.
+
+
+
+### Setup Instructions
+
+```bash
+WORK_DIR=$PWD
+# Create Environment (conda)
+conda create -n llm python=3.9 -y
+conda activate llm
+conda install mkl mkl-include -y
+conda install gperftools jemalloc==5.2.1 -c conda-forge -y
+
+# install pytorch
+# you can find other nightly version in https://download.pytorch.org/whl/nightly/
+pip install https://download.pytorch.org/whl/nightly/cpu-cxx11-abi/torch-2.0.0.dev20230228%2Bcpu.cxx11.abi-cp39-cp39-linux_x86_64.whl
+
+
+# installation
+pip install transformers datasets evaluate accelerate simplejson nltk rouge_score
+
+# Setup Environment Variables
+export KMP_BLOCKTIME=1
+export KMP_SETTINGS=1
+export KMP_AFFINITY=granularity=fine,compact,1,0
+# IOMP
+export LD_PRELOAD=${LD_PRELOAD}:${CONDA_PREFIX}/lib/libiomp5.so
+# Tcmalloc is a recommended malloc implementation that emphasizes fragmentation avoidance and scalable concurrency support.
+export LD_PRELOAD=${LD_PRELOAD}:${CONDA_PREFIX}/lib/libtcmalloc.so
+```
+### Build Loadgen
+```sh
+git clone --recurse-submodules https://github.com/mlcommons/inference.git mlperf_inference
+```
+
+Build:
+
+```sh
+cd mlperf_inference/loadgen
+CFLAGS="-std=c++14 -O3" python setup.py bdist_wheel
+cd ..; pip install --force-reinstall loadgen/dist/`ls -r loadgen/dist/ | head -n1` ; cd -
+cp ../mlperf.conf ../../
+cd ../..
+```
+### Clone 
+```sh
+git clone https://github.com/mlcommons/inference.git
+cd inference
+cd language/gpt-j/
+```
+
+
+### Download & Process Dataset
+Downloads the raw data, processes and saves it as json file inside data/
+```
+python download_cnndm.py
+```
+
+_To the extent that any public datasets are referenced by Intel or accessed using tools or code provided by Intel those datasets are provided by the third party indicated as the data source. Intel does not create the data, or datasets, and does not warrant their accuracy or quality. By accessing the public dataset(s) you agree to the terms associated with those datasets and that your use complies with the applicable license._
+
+_Intel expressly disclaims the accuracy, adequacy, or completeness of any public datasets, and is not liable for any errors, omissions, or defects in the data, or for any reliance on the data.  Intel is not liable for any liability or damages relating to your use of public datasets._
+
+### Calibration
+Downloads CNN-Daily Mail dataset and creates the calibration dataset (JSON) for post-training quantization
+```
+pip install datasets
+python prepare-calibration.py --calibration-list-file calibration-list.txt --output-dir </path/to/output-folder>
+```
+### Download GPT-J model
+Please download the fine-tuned GPT-J checkpoint using the instructions below. The download_gptj.py only downloads the default huggingface model which is not fine-tuned on CNN-Daily mail dataset. 
+
+#### CM method
+
+The following MLCommons CM commands can be used to programmatically download the model checkpoint. 
+
+```
+cm run script --tags=get,ml-model,gptj,_pytorch,_rclone ---outdirname =./model -P
+```
+
+#### Manual method
+
+The above command automatically runs a set of Rclone commands to download the data from a Cloudflare R2 bucket. However, if you'd like to run the Rclone commands manually, you can do so as follows:
+
+To run Rclone on Windows, you can download the executable [here](https://rclone.org/install/#windows).
+To install Rclone on Linux/macOS/BSD systems, run:
+```
+sudo -v ; curl https://rclone.org/install.sh | sudo bash
+```
+Once Rclone is installed, run the following command to authenticate with the bucket:
+```
+rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com
+```
+You can then navigate in the terminal to your desired download directory and run the following command to download the model checkpoint:
+
+```
+rclone copy mlc-inference:mlcommons-inference-wg-public/gpt-j ./model -P
+```
+
+
+### Running the Benchmark
+Replace the model and dataset path arguments with your corresponding paths. For evaluating the ROUGE score after the run, include --accuracy as shown below. For user specific target qps, please include user.conf.
+```
+python main.py --scenario=[Offline | Server | SingleStream] --model-path=./model/ --dataset-path=./data/cnn_eval.json [--accuracy] --max_examples=[Maximum number of examples to consider] [--gpu]
+```
+### Evaluate accuracy run 
+Evaluates the ROGUE scores from the accuracy logs. Only applicable when specifying [--accuracy] while running main.py
+```
+python evaluation.py --mlperf-accuracy-file ./build/logs/mlperf_log_accuracy.json --dataset-file ./data/cnn_eval.json
+```
+
+### Reference Model - ROUGE scores
+The following are the rouge scores obtained when evaluating the GPT-J fp32 model on the entire validation set (13368 samples) using beam search, beam_size=4
+
+ROUGE 1 - 42.9865
+
+ROUGE 2 - 20.1235
+
+ROUGE L - 29.9881
+
+### License:
+Apache License Version 2.0.
+
+### Datasets & Models:
+
+To the extent that any data, datasets or models are referenced by Intel or accessed using tools or code on this site such data, datasets and models are provided by the third party indicated as the source of such content. Intel does not create the data, datasets, or models, provide a license to any third-party data, datasets, or models referenced, and does not warrant their accuracy or quality.  By accessing such data, dataset(s) or model(s) you agree to the terms associated with that content and that your use complies with the applicable license. 
+
+This is a comprehensive list of public datasets and models used by this repository.
+
+| Name (Link/Source) | Framework | Use Case |
+|--------------------| --------- | -------- |
+| [cnn_dailymail (Hugging Face)](https://huggingface.co/datasets/cnn_dailymail) | PyTorch | Text Summarization |
+| [gpt-j-6b (Hugging Face)](https://huggingface.co/EleutherAI/gpt-j-6b) | PyTorch | Text Summarization |
+
+Intel expressly disclaims the accuracy, adequacy, or completeness of any data, datasets or models, and is not liable for any errors, omissions, or defects in such content, or for any reliance thereon. Intel also expressly disclaims any warranty of non-infringement with respect to such data, dataset(s), or model(s). Intel is not liable for any liability or damages relating to your use of such data, datasets or models.
+
+
+## Loadgen over the Network 
+
+```
+pip install cm4mlops
+```
+
+The below CM command will launch the SUT server
+
+```
+cm run script --tags=run-mlperf,inference,_performance-only --model=gptj-99  \
+--backend=pytorch   --device=cuda --beam_size=1 --precision=bfloat16 \
+--network=sut --rerun --quiet --adr.compiler.tags=gcc 
+```
+
+#### Note: 
+In our experimentation, we found out that in addition to memory occupied by the model, KV cache of size around 6xbeam_size GB occupies the memory.
+
+Once the SUT server is launched, the below command can be run on the loadgen node to do issue queries to the SUT nodes. In this command `-sut_servers` has just the localhost address - it can be changed to a comma-separated list of any hostname/IP in the network. 
+
+```
+cm run script --tags=run-mlperf,inference,_performance-only --model=gptj-99 \
+--backend=pytorch  --test_query_count=30  \
+--network=lon  --rerun --quiet --scenario=Offline \
+--sut_servers,=http://localhost:8000 --adr.compiler.tags=gcc
+```

language/gpt-j/backend_PyTorch.py

+import os
+import time
+import numpy as np
+import array
+import torch
+from torch.nn.functional import pad
+from torch.utils.data import DataLoader
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import mlperf_loadgen as lg
+from tqdm import tqdm
+from accelerate import disk_offload
+
+gen_kwargs = {
+    "early_stopping": True,
+    "max_new_tokens": 128,
+    "min_new_tokens": 30,
+    "num_beams": int(
+        os.environ.get("GPTJ_BEAM_SIZE", "4")
+    ),  # only beam_size 4 is allowed for official submission
+}
+
+
+class SUT_base:
+    def __init__(
+        self,
+        model_path,
+        dtype,
+        dataset_path,
+        scenario,
+        max_examples,
+        use_gpu=False,
+        network=None,
+        qsl=None,
+    ):
+        self.network = network
+        self.model_name = "EleutherAI/gpt-j-6B"
+        self.model_path = model_path
+        self.use_gpu = use_gpu
+        self.dataset_path = dataset_path
+        self.max_examples = max_examples
+        self.scenario = scenario
+        self.qsl = qsl
+        print("Loading PyTorch model...")
+
+        # dtype
+        if dtype == "bfloat16":
+            self.amp_enabled = True
+            self.amp_dtype = torch.bfloat16
+            print("BF16 autocast")
+        elif dtype == "float16":
+            self.amp_enabled = True
+            self.amp_dtype = torch.float16
+        else:
+            self.amp_enabled = False
+            self.amp_dtype = torch.float32
+        try:
+            self.model = AutoModelForCausalLM.from_pretrained(
+                self.model_path,
+                device_map="auto" if not self.use_gpu else None,
+                low_cpu_mem_usage=True if not self.use_gpu else False,
+                torch_dtype=self.amp_dtype,
+                offload_folder=(
+                    "offload" if not self.use_gpu else None
+                ),  # specify offload folder when using devices with less RAM
+                offload_state_dict=(
+                    True if not self.use_gpu else False
+                ),  # to have some shards of the model to be on the disk
+            )
+        except ValueError as e:
+            if "disk_offload" in str(e):
+                print("Offloading the whole model to disk...")
+                self.model = AutoModelForCausalLM.from_pretrained(
+                    self.model_path,
+                    low_cpu_mem_usage=True if not self.use_gpu else False,
+                    torch_dtype=self.amp_dtype,
+                    offload_state_dict=True if not self.use_gpu else False,
+                ).cpu()
+                disk_offload(model=self.model, offload_dir="offload")
+
+        # Cast the model to GPU if the flag is set.
+        if self.use_gpu:
+            print(f"Casting models to GPU...")
+            assert torch.cuda.is_available(), "torch gpu is not available, exiting..."
+            self.device = torch.device("cuda:0")
+            self.model.to(self.device)
+
+        self.model.eval()
+        try:  # for systems with low ram, the below command gives error as some part is offloaded to disk
+            self.model = self.model.to(memory_format=torch.channels_last)
+        except BaseException:
+            pass
+
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            self.model_name,
+            model_max_length=1919,
+            padding_side="left",
+            use_fast=False,
+        )
+        self.tokenizer.pad_token = self.tokenizer.eos_token
+
+        # calculate the memory size taken by the model
+        self.total_mem_size = 0
+        parameters = list(self.model.parameters())
+        for param in tqdm(parameters):
+            self.total_mem_size += param.numel() * param.element_size()
+        self.total_mem_size = self.total_mem_size / (1024**3)
+
+        # construct SUT
+        self.sut = lg.ConstructSUT(self.issue_queries, self.flush_queries)
+
+    def issue_queries(self, query_samples):
+        print("Number of Samples in query_samples : ", len(query_samples))
+
+        total_samples_done = 0
+        list_prompts_tokens = []
+        list_prompts_attn_masks = []
+
+        # Pass each query to inference_call function
+        # Activates only when scenario is Offline and network mode is None
+        for i in tqdm(range(len(query_samples))):
+            index = query_samples[i].index
+            input_ids_tensor = self.qsl.data_object.source_encoded_input_ids[index]
+            input_masks_tensor = self.qsl.data_object.source_encoded_attn_masks[index]
+            text = self.qsl.data_object.sources[index]
+            query = {
+                "input_text": text,
+                "input_ids_tensor": input_ids_tensor.tolist(),
+                "input_masks_tensor": input_masks_tensor.tolist(),
+            }
+            self.inference_call(query, query_samples[i].id)
+
+    def inference_call(self, query, query_id=None):
+        """Common for all scenarios"""
+        torch_device_type = "cuda" if self.use_gpu else "cpu"
+
+        input_ids_tensor = torch.tensor(query["input_ids_tensor"])
+        input_masks_tensor = torch.tensor(query["input_masks_tensor"])
+
+        # Moves the tensor to CPU or GPU as per argument passed by user
+        input_ids_tensor = input_ids_tensor.to(torch_device_type)
+        input_masks_tensor = input_masks_tensor.to(torch_device_type)
+
+        with torch.inference_mode(), torch.autocast(
+            device_type=torch_device_type,
+            enabled=self.amp_enabled,
+            dtype=self.amp_dtype if self.amp_enabled else None,
+        ):
+            input_batch = dict()
+            input_batch["input_ids"] = input_ids_tensor
+            input_batch["attention_mask"] = input_masks_tensor
+
+            output_batch = (
+                self.model.generate(  # takes Long and Int tensor datatype only
+                    **input_batch,
+                    **gen_kwargs,
+                    pad_token_id=self.tokenizer.eos_token_id,
+                )
+            )
+
+            input_batch_lengths = [x.shape[0]
+                                   for x in input_batch["input_ids"]]
+
+            output_batch_lengths = [x.shape[0] for x in output_batch]
+
+            output_batch_truncated = []
+            for data, source_len in zip(output_batch, input_batch_lengths):
+                output_batch_truncated.append(data[source_len:])
+
+            output_batch_truncated = torch.stack(output_batch_truncated)
+
+            # Loadgen monitors the reponse in corresponding functions
+            if (
+                self.scenario == "SingleStream" or self.scenario == "Server"
+            ) and self.network is None:
+                return output_batch_truncated
+
+            pred_output_batch = output_batch_truncated.cpu().numpy()
+
+            decoded_outputs = [
+                self.tokenizer.decode(output, skip_special_tokens=True)
+                for output in pred_output_batch
+            ]
+            response_text = decoded_outputs[0]
+
+            # Loadgen monitors the response in GPT_QDL
+            if self.network == "sut":
+                return {
+                    "pred_output_batch": pred_output_batch.tolist(),
+                    "response_text": response_text,
+                }
+
+            response_array = array.array("B", pred_output_batch[0].tobytes())
+            bi = response_array.buffer_info()
+            response = lg.QuerySampleResponse(query_id, bi[0], bi[1])
+            lg.QuerySamplesComplete([response])
+
+    def flush_queries(self):
+        pass
+
+    def __del__(self):
+        print("Finished destroying SUT.")
+
+
+class SUT_Offline(SUT_base):
+    def __init__(
+        self,
+        model_path,
+        dtype,
+        dataset_path,
+        scenario,
+        max_examples,
+        use_gpu,
+        network,
+        qsl,
+    ):
+        SUT_base.__init__(
+            self,
+            model_path,
+            dtype,
+            dataset_path,
+            scenario,
+            max_examples,
+            use_gpu,
+            network,
+            qsl,
+        )
+
+    """IssueQuery and inference methods implemented in Base class"""
+
+
+class SUT_Server(SUT_base):
+    def __init__(
+        self,
+        model_path,
+        dtype,
+        dataset_path,
+        scenario,
+        max_examples,
+        use_gpu,
+        network,
+        qsl,
+    ):
+
+        SUT_base.__init__(
+            self,
+            model_path,
+            dtype,
+            dataset_path,
+            scenario,
+            max_examples,
+            use_gpu,
+            network,
+            qsl,
+        )
+        self.sut = lg.ConstructSUT(self.issue_queries, self.flush_queries)
+        self.total_samples_done = 0
+
+    def issue_queries(self, query_samples):
+        # The issue queries function is called multiple times by the loadgen as
+        # per Poisson Distribution
+        index = query_samples[0].index
+        input_ids_tensor = self.qsl.data_object.source_encoded_input_ids[index]
+        input_masks_tensor = self.qsl.data_object.source_encoded_attn_masks[index]
+        text = self.qsl.data_object.sources[index]
+        query = {
+            "input_ids_tensor": input_ids_tensor.tolist(),
+            "input_masks_tensor": input_masks_tensor.tolist(),
+        }
+        pred_output_batch = (
+            self.inference_call(query, query_samples[0].id).cpu().numpy()
+        )
+        response_array = array.array("B", pred_output_batch.tobytes())
+        bi = response_array.buffer_info()
+        responses = [lg.QuerySampleResponse(query_samples[0].id, bi[0], bi[1])]
+        lg.QuerySamplesComplete(responses)
+
+        self.total_samples_done += 1
+        if self.total_samples_done % 5 == 0:
+            print("Completed : ", self.total_samples_done)
+
+
+class SUT_SingleStream(SUT_base):
+    def __init__(
+        self,
+        model_path,
+        dtype,
+        dataset_path,
+        scenario,
+        max_examples,
+        use_gpu,
+        network,
+        qsl,
+    ):
+        SUT_base.__init__(
+            self,
+            model_path,
+            dtype,
+            dataset_path,
+            scenario,
+            max_examples,
+            use_gpu,
+            network,
+            qsl,
+        )
+        self.sut = lg.ConstructSUT(self.issue_queries, self.flush_queries)
+        self.total_samples_done = 0
+
+    def issue_queries(self, query_samples):
+        # This function is called by the loadgen after completing the previous
+        # query
+        index = query_samples[0].index
+        input_ids_tensor = self.qsl.data_object.source_encoded_input_ids[index]
+        input_masks_tensor = self.qsl.data_object.source_encoded_attn_masks[index]
+        query = {
+            "input_ids_tensor": input_ids_tensor.tolist(),
+            "input_masks_tensor": input_masks_tensor.tolist(),
+        }
+
+        pred_output_batch = (
+            self.inference_call(query, query_samples[0].id).cpu().numpy()
+        )
+
+        response_array = array.array("B", pred_output_batch.tobytes())
+        bi = response_array.buffer_info()
+        responses = [lg.QuerySampleResponse(query_samples[0].id, bi[0], bi[1])]
+        lg.QuerySamplesComplete(responses)
+
+        self.total_samples_done += 1
+        if self.total_samples_done % 5 == 0:
+            print("Completed : ", self.total_samples_done)
+
+
+def get_SUT(
+    model_path,
+    scenario,
+    dtype,
+    dataset_path,
+    max_examples,
+    use_gpu=False,
+    network=None,
+    qsl=None,
+):
+    if scenario == "Offline":
+        return SUT_Offline(
+            model_path,
+            dtype,
+            dataset_path,
+            scenario,
+            max_examples,
+            use_gpu,
+            network,
+            qsl,
+        )
+    elif scenario == "Server":
+        return SUT_Server(
+            model_path,
+            dtype,
+            dataset_path,
+            scenario,
+            max_examples,
+            use_gpu,
+            network,
+            qsl,
+        )
+    elif scenario == "SingleStream":
+        return SUT_SingleStream(
+            model_path,
+            dtype,
+            dataset_path,
+            scenario,
+            max_examples,
+            use_gpu,
+            network,
+            qsl,
+        )

language/gpt-j/calibration-list.txt

+eceaa658027ad9625f832368198e11bd2fa38977
+70052e55c12c97a9bf6796a25b6ced8f3ec4be06
+9767fdf0a53da6ee9e2f75582cac5964d80e1b5d
+1f8c736647d06c42beb553b25a02e44ca15ca0fb
+d3ce7d615ecc15f094d8130654812ad77cd604a3
+55086c3f69cb41b991d3db0c6b10b0aa374788b4
+2745f93afca3edf25dd9ccfd094eef06298f62cb
+343644770a597a2dfa7548ba165fa9c6bdc88245
+e2cecb8734918ac6a2d9cc8afcfafb16b1781ae2
+feba32aa9aa3b51fb451bc48a54e78d02efe977f
+9c2e4d2f6085ef9f237e6fe1baf83000a264cf93
+d85158494b7041560466f153c4d050362f90a7e6
+1e14852c49e84434ca249951e0fe603610eb68f6
+369d721d1102f0cad726ad3426d79c965a224b28
+b9898d6014353a7411c0cec222996431c832c35f
+7cbe104b3203061bb544267879fa316436a1ab5f
+f48a6b4fa0827b4c6324bd47dc2e8954141b1a6a
+acb5ce76c230bc66633414678bf254387c3d6c85
+d70d5115ec3adc5fb3aee8b1e29c7f0f2db083be
+ffbe89f592457d39ab9c28de4fd89fbac2150f81
+d841808ba87a4aabbfe4427c53eb0e2e8a74995c
+2d4125c6162f9b4924a262a55bd8fe3faad0b3c7
+95fbe3b3a7e5fb6fa48289885df25d9a6e911d2d
+f6ffa98e7d46000bee325e5284a2ac897ba4149d
+31e424f7a6fe1f5ec61486eec17a85c78ca2ce8c
+2165fd490b9d14ce0cd3784beb2f4a1d10028a1d
+4d6b1a85d264397e956c0a03f70de123ed4fff5f
+4d20111e71a06547613e161292a51cc44eb74da0
+b90b35dfde9fc08fe1651e36bc02a3f1503e3b6e
+2d3b2eb21a6691c764aaa1f27030453fc44331ab
+dbf02196bae40e0adbcd1790294748a560b1e45c
+0ef5c715acd7a70f51a9d800c9e01bfe69657bed
+f0f65f40fc56b69bbfab5d88a157dc52ad967484
+db3575fd124f65a7aeee7b4512b5e0fbebf2c8ea
+1234fafb7b6ecc9224d6d979536115771467f4ae
+c31f79babaf4d93609ef3ee2966e25428a4fc130
+600b619001a9d840f5bb5ee3362787ee38df26fd
+5e68842017bc6f5806d962ae5ddea8490d763b82
+fa87334256af163f8f56d8035d1be4c7909147e9
+826f2414623f8e444725f1c9af593f27b440ebdc
+3603896c0fbb49d6231ced092ff7d057db2c43f1
+4b8505e0219b78f9645fb12177d97b8e29821ee5
+3332226f8b4f6c46ed3c66ad0765c3720475254f
+97223b7119e264a598afe67e0ba82fbb97fedd2b
+87fd2fd13729ba13920591bcc96a83ddf61625e0
+2160c5d812611becf442b5079a7908c2f48f6de7
+559d3b10273acbd4b98ff5557aee92f33e50129d
+273c1d2936592874fb599f39dce8d6a0813a49b3
+3e6affd8cc6ead338996574fe1d0cb03ca983ea2
+3b733db6e80a89bb4062c1e661b9f9d4690ea0c8
+4a0f219f4b67d9cda499b71e3ee54bff5736f8c1
+064396600b73dc4418ef783fc82f4fe1ff038f6d
+eee1cbcef13cd16096734414da992aa1a7048bee
+e190b373844d5d3a59b9abb867de8f0fdffddeda
+8700aeab383d481d613759f19717ea4381df1433
+087b05a98112135d9fb1c7628fffb72ae2456e9e
+f69c5a3c9ef4bfb195ad7ce2590021121a7afced
+82958d258a7fe963f8c9a001010365437bf15fc2
+b6b37b9bc60519fd214d6e980fcbb16da066eb68
+a49ac163d47c1e690f5d3237433194a9d0ab558a
+aa35fa6f613b29bf80806552b1bf260f04bbedc2
+c248fc3e54b451a4117f23f5adc88cb8067be3aa
+f21eae7e796721088234b885bc6eae0daef05738
+b5c4d6f671adfb997abb53a5f2f54519180df7b5
+457b2ab2b4edb94c4b67c1219451be80dc369e8b
+e80b0028e44685e39581ced42c1e1ed9cf44f74e
+c2d90734f9228cf3163187ad72405c90bb09d13b
+a999f5732f9bbe0991e6e895f9bfd42bdda75bf1
+cea6ac133923e62b186aeb17db37be6640a81200
+7facc85e37ababb8c029257f246fe0934f84a808
+21dcf444d4bec9d4b2a6ffb54112c4cbb797025f
+f880779ad1b262aac438a8cf3a6df9c0ecebdada
+7313410020b93dea1f223d2ffc0b997385d7886c
+b0b2948eac6b4e082bbd420da8dff3de6a187377
+360b51c738cde3fa09cef18c3d7672a1d20d3379
+cdaa77a96e1d96a672548b6dc0bd83bffe6f1619
+cd2cb113b1cd90e2ad235466df3a64dfc956877a
+7140dd21ed480a3f47a59c647c1f4e690939caf0
+f2c9d3d8f0622e67574f386197b92570730fa61c
+010cd75a5b587285f7697cdb6db6526bcc0320b2
+86983b2cddcc91369ad7d4ff61c9e6d258c78b71
+bad5a939cc0d695a97e7f3fedac53c93f04c3253
+0b8cb37a8d54e1761b3d99b8a6e6f921f07e00ae
+613111ac2a3831a7291656bee2def306453552d4
+0e29e9cf2a08c4b35ba840bde03b0537e3821a74
+2f425306562bf94bf5f4567b8c63c5b204a2c414
+8bdc6411954d7163b137c4970f6d6431aeeb9ee6
+be9ef18a4a0a08f94a0340b2df0d0f83144299b0
+cafdb2911d9af6038659354058586d6bd5174338
+0c14f1a16ad395dd9aff4437452cf555ae8858d2
+c3db5ce828128fc91c5bbf59d144e825f49ad271
+273221df42c8dd9e0296cf3ec203c59fa205ecd2
+5abe61623d56bbb0d7bbc79f5ffa96732b3a1d97
+98d6f13e4f3ed36591298430dcb49bddf89003b5
+18686c211cc5f466c48fb8dfaf5eaa00cf3ef0ee
+d85b6f625e9e57efaa1e3c21e626dd25e3414758
+69e2ec5703156e86990ba690b2a9d4dd0f5733b0
+cc665a9316a9b22400183776d396940ffeea2fde
+319a27f92609641ee0f0fe1f6a88a9aab739b98a
+94817c53b53b3979c3d32d197ec26dc489921e6b
+c803d0b957abefb26e1fe0ad87aea5fc80180a20
+6c5b9c68e7890f4b683e148ac62dace194b45b59
+b377a0f9dac858e2bf7271bac531591140a56e33
+6604ac30dd3d9a29faf40def3e4549feab4d9d02
+b7fc4ea36690353003cd172e597c456534bb2811
+b42286dd9b577bcc261e838045e51133266f7fce
+73f9e1772ab372f36898dc95ef18711434827717
+838a481841bf69bbe3ac8a3b53da54d692ffd084
+0a1c17d84f846eecce39356447e4d556ffdd07fa
+98f8a09b98daa05cd412e83d724e4b914e8d921f
+f859c4b5c39653753b21c575a98d525fc47c16bf
+2abf786570b9f9de945c70d9678cb67dd2a2e57e
+f4a4d960ecfb87fe06e38f56310a14d23caddb42
+d96bcf6baf6cf8c61d54ad9c9abd997bab870077
+c6eaf9d97b059f3e824a1ab4ffdfe45494e5f8a1
+8bbbd3d05e22fc3372abdbb471799bd5f5380a75
+09dba9542ee64697d789d47ecd3bd53bc8b0d953
+7d42400c49f9cb313345f7d9ea74c51d0147a4f8
+1f58239bc2fa91c3787bda0c2b9ca5cffaa7510d
+a0e187806f8ec8f02e0d329ac114ac44fe69a4b4
+ad2e90052e15364c93011386322f6a5007314348
+adba3dadc7c6a3177346ba9119a8df8f5b81ae0c
+af2d712599be471d1ba0b91fa18c347220ca595d
+77c610007679f13e1f5d314ddcf4b14c7e57876b
+0b1d14b9f6619ced37003ff77f22dbd122fca645
+310fe57a8497cb5215c1d70d9f9a22ab91d5c054
+7056abbadea0a2eec779e157890219171bc98938
+3863aca32c99daf6ef8a0de6f60471bd9c54b885
+b32151c4d36a4b42e9b832e14f539396627f8eae
+f72c81d6c240a6dc8395c6ef33b64edc423b1fae
+4409a8f066166ba9aad02611f6979b44fc91afae
+75474ebca7f3413b4e814794d6ffc13663120bae
+55e430f0c68f6f0c4cc996fcb87c2e53233e2738
+5dc954a346ff3dba1629e03f3e6485235d6d4742
+95a92435cdb8f2b51ede4ce6220c5613c8dbfc2c
+a39a529543dcbd6c0088cc301bd82173feb5f18d
+2b8179710437421d5a1f0d281515725ccebff3f3
+087f4bd441c0032caec0a1f65a139d336a09d133
+3636a522fd19a2ae4ff514319d5c1fc012c4bcb6
+c5f38edba57d815658097ba5ebe37532ca160d7f
+b0f17178ff8d37e5343119bd8917e262f386203d
+dedac076c1649b0edf55353e9bd374c0cd4ad956
+9a3e0978753d5354eaebdcec8550641314c71b83
+0c9881539c0c5249e911dd70b37cf7f74327b97d
+80e02450839ab9c1a08082e404c6f0398ae2e92f
+6e85c4357ae3dccaae1b354641d22a359a100d47
+b05f9fa99ca30d7ce2611a6deb139f2274d1ad3b
+306a3efeb8ab8919079525b9aa747093bdcab563
+98c31464a9052c1ccddc9cbb71c2529f3fba6f4d
+35ffb93ea7e6cb006d5019185815f03c67b94d77
+2f3f088adba0256b27e6da9efce4293106e34291
+02d287e484a9da84424d10eeb0c8f3ae52cbc70c
+33dedcbe9423f6031122f8be1f7c2c69ea4ad4ad
+03765604d9073697904c2dc4cf29e90b924f36f0
+d2845ecaded68fdc5b372d10c3663441ec8b358d
+4cd67a6ead5211ad92e1faaaf71ef28f7da2f593
+806333db217efd7e2a4562bb73e695bad88e712c
+1307f117d423d143e6083faa99255c2bf2a2f3fd
+c92b96b8114ec521af30fc090cef40c07f9544d6
+04b3d6adf722b3b33524c8d288c74e7db2632a2a
+ee5db06cf8ca3010774965c3674c273c680c1611
+709e0af32c2463474cb8ffb85d2dbc07960037c5
+d503ee11d8f7d43c67841ba1b6bd863a6180a223
+709555f0f163e09098b58d03898a9e0d6e7ca0b2
+010f73477cd20c14cab78ad9cef350ac8c0f55b3
+9e95f4744d105fbcbc32a12db7287cb64254325c
+6e47b7f6a76cc2728e61f4bdd30bef697d6490d2
+5620139a78269335505edf23a902bb0c9c264e3e
+70908608fd62696f99cb3f7a185b226fe32e475d
+0e907ac0dd02b47f4a0726790d01f0c57037ec2f
+bab2d77edbb5ffb3a7938f16fabd7ad3cc83fae9
+f5d5c855f2e708067e3532980aef101d20c40cab
+c50895475b8a401824cc9f1bfaaa8fd7797e172c
+98690365a6ab1e82c25ca08c26db63a834c21fb9
+3368d60307efa2661820d3240854967fbbf6fbc9
+1f7b9e38af5bfddef1649c83b119e45063bbee34
+3fc0feedd683b49702d0da9d7d3c36b7be02ca09
+2014aa562c1d05dbbba727db120d9e163fb8f43a
+e22249a8da0886b4c1338dbf2e54b766b13f4db6
+8344960457778ea0a4fdbba33e7eebb69aa979bf
+2c7fb7b897db7f304961e919cd5ef1a5a93877a4
+e2b409c9d97825acad579abe22e0a37b685d6ca9
+b2fadf8584ecefe2a32cc2eba6590d10dc8a9d26
+5f201c66bf26298986c3dd2aa84818a312a596e3
+8dffb657ffd1e331b99cc00cebf18645e219da12
+3236d66fb8a63916b6fd00c2f2ec417b5cde01b6
+95cf024d3f1e40344f16cf4faab052d6fb1e60cb
+9faac843955464da41331af273942e38561c9a8f
+d383fbcb1a69ef97a660318a2b36486e5fdd6a44
+9939959cf9cb1a14497e63aec0b88a08ad3e451c
+3c0c3ce2681718b816289eeeb3ac343ddc037fc4
+259d8cccfcb9b9edc00d757ec6efecde6fc06110
+9057ca8e09723c9959f923a412e409ee793d0062
+79f5ac0831ef03c2ebb40d325758350937a55313
+3f36f6f4d3317275130051db2405459021f56b8b
+e35c3d07dd54243acf4298ee0ce6ea7e4621e90f
+115a59d5c4916cb14b4c408bec36bbc6116043cc
+5d563efeae0cab135ec70ae4456a4e55bf598aff
+2d8d73eade954a63f892414accaf2db229ff3312
+d42bd8a35e147633d3d750266939c6539aecece9
+27155662fda1f5febdbb42e6572dda8d9e31588a
+a210a653a08fd0460b52c7eb68bdbde0c40ea63b
+fb4be2d8538e5e4418042eb7d81491dc7e94dcc5
+53940e5d960d1b63e5ec84fae802fcd599b20f01
+213dc667b6c665a4257c4afef5e5fd39d42eb01a
+08a16f7ffb9968774fe4562acfb79aa6a1a59a2b
+d1caa4726d8ac1d9ad611708038db896828f06f7
+67e3d20bbadc184c57efe184ce8ccc402de23bed
+ae05bddb7e816fd0e14e95cc525e06caf9392918
+b2bec4804d38db4d01520c4b65f410acb20e4d2a
+78a8c13605a8eda09a0ac0f04910b414eed6b765
+ebdd1d2b3891d6f0de29ffa1eeed3f03bbef7912
+d54da603155d9e507b81d7188e1baee2f984a99a
+18823ffe4e7d30056229c6b0c3b71f9c72c1d2e1
+86b8d10094b19ab1059b5dd7983f26fc2bb133ca
+a16c62ef8dfc132a0a5c406e429a08e1d40b8756
+64e19e8802e2f598c5a84858a4b2c0c43b99877b
+3fe31bfa86777b3f4a1bcbb46650f683fa477935
+d6e9929980eb730124e8cf6561991d43f19241e8
+20ffd27dc3be9eb895fe8a5ae3cffcd795ad100f
+3960b049e19e3217968723430f3595fb1d4e1dab
+846e738db5d5df03f621e5cca067016e84327f16
+2c20a17cc4846b8dc437fd00f84d08cd15d0c8d4
+16cbd9a93ee9067271748479378a31d24390e048
+77677862965b241d7b9c4ea61836ccf09b3e37a4
+3ec8db9d06345bf26aad0ccfd05408880946f4a7
+3747faa432e732538f1636c9aca56f068ec44a4d
+f1959058e2074a54c0bddf7afc60131df132415a
+216558f2fb3e918840acc2fca7c81f27c7a80e3f
+1c7e55eaf41d1e43121755c1cd667d210e45a000
+ce5c91d45d83f9f114814c8db9a1230b2d79eb02
+e735d473af54e1ff29a66b379fff9e88ccd8a164
+7809602d8d9398f05b032bc399a922af1567c56a
+e85a828dc7853ddcce5d7d919b07370236fd089d
+0f81b75410062d52138ab8a67ae49d03321e991f
+c9bfa4787bab601fe2e0749b4fb1e44d3f168373
+c03e79ef13869270df1be0c63ae86dddb7c21bf9
+91e1856c8de122ef09c10589afb9b3728bba9296
+0a661af9686af6c8f298c8309e8e1a96ef0cc08f
+08102a4509565732289f843007d08cfa72ea5456
+9b9b7248f513f621089a6cdb956828a3fa6da09b
+7e4ec1b8fa3a477f43f00075da2ed26a31db45f2
+4c5589e14718f8d6ef4027baa22b680f556d9ce9
+f63c67c039b3bfb83b3d46f4250e3509c2e9394a
+8500584842b1a7abfce6a2453fb9f76c5b39d26d
+5f43c3db85393d73b57174a6e3c72884cf1402e6
+1401f556e033d9f10dbdf83e9b5bfcf6a84823d1
+ff7e6d2a2c5fca5f33db717bd68228538fa09f37
+9d6addd57bfee73721c64830eb2d0fd27e8fb9bb
+a55a50b6cd898fc79bf4657fef0f0ad44de6a5fc
+6af08a408468481f5847013cd8b7f9c0ec7296ec
+7d54beb04ea368c6386dc8174ffa1915b3414bb9
+93456d2e7f067d518838df8cd7f32ee85289f4bd
+d603f66ebc365627756eab740140ed43f0e5f40d
+e9217085cfc52f0fc47d91f2feb681a33f88fb59
+d0912f63112be8069398b3f6c926c727469f1191
+2d1edae390d9f079095606c8bed0a83f5bd5d767
+b6136dd5f245f26dece12bf294d524bf584bed69
+c4a4d6e24e7753e098e09324e903c3fc2cb45f74
+8f49dee3dcf6b505e43475e3b7c15a5e25f0d85c
+edc1c91f5eb0547c18877e123cf3ec248ac734d7
+f9f269f3df343d14b11c40286b22f2c54d74d8af
+99c98449dd5a99222dae7cfb14bc060852f220e4
+017d27d00eb43678c15cb4a8dd4723a035323219
+61e137b37db0b3157c04fab0a5f4314fe4b03931
+43f54e39221310d45fed028b202e0e26490846be
+ff5b1552320e183941d8d58f726f589324035284
+f140814244c9e54cf4ff2085d7d52b2dd87d2737
+0264d85da73237f1967bcab20b2f99313a00250e
+7d2215881b5399038a625726794c523be20e567a
+77810c807b3c7452a00968927dc8b3b76c2aaf63
+361864a24f139d975cb02736e81d106b6b50de37
+5b08a44dcfaa7da30b066b62e688177ae4c27bc6
+f0bb651f7498ac35c750d4216b3fbdc1c6e83508
+0865448cb045a8b9568e679dbdb5b752ba0e38fc
+afbe85965b4aae74bc86d5c56c75fe55e782c7a0
+c4e68babe61c2389be350f11dfc8e2c5ddc9f032
+7de82ac3cca30893284f93cc133d87276f39f8df
+0e983ee75b47509844fddf43d06a989b3448376f
+645ec5713498f91b494d39bbe8ac6619a20d45e8
+2a853caa0177515501abb206103e15fed7bf2315
+6b675d840afe29591d304e7b52a1edb442decf2e
+aa4246332705bc11ed706555620cf99aecace692
+461aa6c463d8ed8a3485519f8347d3e8fd30d5f7
+0963c147bc9d5370ae2062863e776853744c64a7
+b6bc7591f950b6647f2d5cbf11bcfaccd8da0ec8
+aa3cbdb196eb266ebeb48c1be941df20ecc1bb90
+945cbe99df1af1b5db99d8dfcec142e5d0452065
+9c2b9de4b8928f63bfbaecc97bddee210e2cd38a
+6ae4c366fec9f8ffb28f74e03fee29f300e4b0c8
+e9797953e895ec7596bb0c80d6c3e13a6170ba32
+4d63952d88ef8b61c631d92744b8b88d5900ba82
+6c668fb743f9af4bb080654040e6416f7e9b5605
+31d2a88aa62215e0046d4db0c0cfcb7390e16762
+941f9ba5091a41a41338a0b5c06ef998ab76bf92
+4f31114f7ead2ec76449bdfba502b576c8cbdc51
+636ed1de3d915dd13e94ea6f83ed418139898672
+e1b8a490189840089a0e42f357d7e18aa04d695f
+e92c4914629728b8c18cf61320cf4a34baa77300
+9afcefd2944149fff4d5b74f5b26a39288b7cd59
+f802525632b1c8fa85b43911f07d8129694621c4
+c7b0320fd85f3ef25cee88621de6eb541b399c36
+ee5ee7b755e26ac0eabf5191e7747f6d72ddc84b
+65a03a7e863b3a5b97576bb3fdda2d8c4380c706
+cefa54e79f57eec0b1273f69ff7149dcd90c7ee0
+c2add6ebc7d17385f7e0d0d9fca5fc98115c68fc
+169f5f6ab3818fc14b9f2471ee0d8dbd61d5e566
+9862b8aab2db9c82fd1012792783a90ec79f7269
+0e1e33051f7d782d2643d645eff67157c37370a7
+8ab2735a3a614a5e95b2f53fabcc04cc482a0abd
+1d400e7242d8570c79f9f34c392ce02e217e01b8
+c56d3923764328f6767dec2e5617f562cc88e791
+f9689fb9656132e1c6d186851563f2b968643791
+08d845c78055627eb898cb74bc38274794351b17
+ed3449c7f2b4a2f4f1548af509dc9ab1960e9fa9
+0892fc2908f83d76b147c3ba1847af0056a47e9f
+eeb9ec2b66bfce439d6ad3f25e364d3b1d826bc4
+55dc9832dc56cae9f0bf180d2103a1d20c1b1ee8
+b05f9fa99ca30d7ce2611a6deb139f2274d1ad3b
+3152602658285f9edadaa1d9cb7cc4948ab8fa54
+ba620c801834cdcd41547b08712734e30e84ae52
+c02f067640c67b1aa5013207c2c7782ac6b97399
+3349e092bb3bf21585d52e72e2c782692932b139
+26e816229351dfe7578c758ba07c4d2d2a891b2b
+064f086f49fa410b664d59a0494367c421ed2f8a
+1241b04b4380b1a796390d32183e3e738d7b82ff
+79879bbee2c8f0b46fe44c80949e24b3c11ff7fb
+2f4e7d5a0130b48ba687536a3bd5623fa906f9a8
+a72c848bc3bffda7aed21ece2b07327153fc11f4
+b10cfb970a746327ce47764050473ea27b15f649
+b30975204e2d948c1ca8d33a9f6e755f86d8e200
+e54abc8237ffb5e2172f192200fbde85a100cdcf
+0eb8e5740eecc2098cd862cb5d1ff41f9aa97eb5
+0807b672dd1a7ee6f8038649f70a66cfa3ba4fed
+e22b4d2a35411b0b2270871f83c19e9f6efbfa67
+6bb9d73ac47b68b90872d97b9ac1e1aa34ae72fa
+c83faf99c08fd4d44d9ee38d1c3ef84c273909f2
+cf91d2b46870970ec013ea2ef0567f695ca80261
+151ea6f1dc4a40cf854a8b2d9fed22ea457d2afa
+cd29f730499023601901dc9ca801c279637c5a81
+4fbaf01100e4d6ee1823f1b25ba309fe73ffb6d9
+49654512a36b27837b069fef447ebcc460b0c911
+09df70a379653872798f1284efe95240944f6af6
+43e9d988417d90e85868aa09b5c53e2ddf0364bf
+5730ccc0f1a125be76253006f14a6d3a39fec5ae
+65012f2f3ec9d16629eb8577d149de30257127dd
+ae05bddb7e816fd0e14e95cc525e06caf9392918
+a65f3d75c5cbf99deccb00c9b94f91b5ad52a050
+bf6d04b98e0af89f073f4b71c5125017c9aa079b
+6f1f25365bd131c0caf19acf0f4fd02a3535f538
+23a87dcd1007f73c4a6278d230aacb6411c71266
+a6d88d33454805c4c3b9f3c50b1b2482048c32a2
+d6392fc14b8c5e61bb6342dfada3b5085dfa691a
+ad61eb84269497ea2e8d9e6f3b1a504d9bf82d7f
+b90e7d7139f69d50d53d5bab66a560785596bcb4
+237f4b3fafb5bca89627701cabdb01a61ddef306
+8df29408cf5fbf40bcdb5a73d9eff3e30b928638
+02f0cb84d4e8f2c78189f3008c327db6a7dde4b6
+d7449a49a1e808c3d2f2c87f6b6b26dc8cbfe638
+096fb4a6df33a35a8a4c28cf6707d6093b8fb483
+54ce333c923bc3d8107ed1b803575c249d92a7ca
+390a82f6f49cad470b3278465d07a9320c163fd3
+b44d920cfa42cb0ea8e279c4401f565577217323
+3c2a82b4460be3eb08988c038156f24e690ce149
+fbfdc61792dcee3d0102859ed2681489b037339b
+5d788ec362a874cc113c2204b06fae82d1d70ea7
+e4a186c4590ba156eb3e45862c2a5b4181e2fab3
+5f7c94ff1e4c755c47343046fa0ed6823124b85f
+aae35772a13f84876be5fadd919f1265159acae6
+22680dc843e4692474815b0c9ca78b9f4f1a116a
+1c23f12e590b2cb4a89314c0b933f12b7193a37b
+674cc0ecfb854619f3e50df0e4baecc67c73724a
+fe41f09089b134bad7f40be0ea4a6fa7a691655b
+12327d1afb02007b3b736570856176234edfa8bf
+26fe548ac5f3ff1e700b2cc6890c2d5b152234b4
+861bbb99177d314267023bf3699ec069f3bda6d6
+cdaeefc0fc597b0b591b76d20f979c1207e98880
+2a370c32d2c1464da03bc2440bc96ad23059e428
+6bba3c06659ebeaeca823bb7517baac4425faae5
+e7ddfb8e15144c1a1e48d8b98ce1a44b666c18ae
+62dd1f31cd4e2c8250c587b557f4c2be67d5e495
+3ab9f2c8f9512bf98dcd467971b3ffc6d612d308
+00120f91cfcab17bac165f7a4719019a628a9db3
+3ee94d218979b459196743ab0a3d2957f72422c3
+6970a0ff24ea28a7500763ec1b72a671bae412aa
+ab356240d60a6e7d6efce1a9638415f13bcf6591
+39d0bcbfe75fb7bbbee7d4bb72a77cebf03e39cf
+ae202f1474cd1ea41a5172230fb083e1bf932d17
+7dc9c7399c2e313bbc6264072f6a592b6915b082
+fa79ac193bddc262fb35a468c8bdfaae536bbd7d
+412630c97420afb50e5278d3406e0cf4b08d0b77
+eb1cbbbc594b324145c3155bae5614a2553a17c9
+c948920cd3dde6dda8767bafc8173c0c37127430
+43ecf30e43686b2a11f6b329f5046a68082b7272
+49b2330923275c10d5faf66681fea724f9938893
+ea6c2b9b8479325e3c081252b59c61047988736c
+d28da9e2bb92814351486125aa35b16d112f3a76
+393a89e4d5b67ce3e29678656d73a0dbc2ee930f
+daed716140202e583ad4cdc98fcd8b4b3aa5ae35
+51ce09bfb256ad9ac38a8e071f36b0097f6fa68c
+36beb23a74208850b1bec50c966e985aef7e4075
+52894d78609a1022f6dbc4ec4fe32fcc31fa9366
+3988b5a02009b7589973eeb2cb929f2d37c4f409
+cbf788987b75c11aa5c49518c5b3cb45e2c177df
+ba1d997f0df5b17a0899bb643a467e95332cf0d8
+4a0453760d334ccbc84d00ff67a87865fabae97b
+d39cf4acbfe4fbc26e001266243644ae35beb712
+d05b5e0e3619b87fc46e731ced5111f47ccda50d
+bb98ef0258299046aecb9fe0070ec309d1cac401
+080b5e433c62201fde1221066f4d723e3ef427dd
+92a6b2e9d9e7da09ab27eee906a8c38c0219f390
+04d7b42001e7249bef064a175aa1236be8211e4c
+75855c9acc21043af85c9f161fdb0f68af165771
+2fc69a299b1a7ce0a9dd2552a910f319773f3eb5
+98ebcb90dfca1bfcc7bd2cecf53cc12f7dde6970
+db8025d7b55de50ac56606023c838bb9975dceee
+f1530622d659a31a36b4cc5b79d3ad9302ecb384
+a9e946a8ee4153ae7e45143941da7f61fd04321a
+c6f17e5ec10ffaed02b111f02af4afa86d347d3e
+a7ce2524995e668268028e9f7237dbfbae3cadd1
+407f1d56cdeccb0e313c15ddaac53b186acdbf0c
+a8c0086c0ce76f960aedff7a7d28d9009751cc34
+e28316232274fd9444562d8f7b5b6949072d0a2c
+ce5ba5110238d225394df833987db22b197dc93d
+32f66b696f66911ca0c73e36ee32708d59124f32
+242c50a749fe607d1a652db0e06f453d5d7f80f4
+4375cd51b7ac544735b9c89df29db30369b0eb9b
+1458f8a2a10e49686cbc0b5e14a97acc3aa78a30
+c6ce5c4febbfb715453373d447c4b5572f5fcada
+0c145a6bf87af0340fe06024ad8ccea391c9134f
+b5d3ce4d0549d7802002f2d5e90f05ac35f5db5a
+7944055eb6cd49f12b5d42ca0b971eaa6dd51e07
+63c0d33115ed9eedd51c7f34177a113f6b40156b
+f2ab65bb852cc93aaeb521f0f65fc2ffd14e996c
+c7147ba3242871a59b80860245ab60c3d04c5ecb
+d9accb542e2321181468e8f7e490114b30c1cf53
+8ae25fe05b21fc819243746ad7caf4555e11df0f
+a3ddfaae625902b0394f854d6b341b21684638c4
+1a71c481191a57d4ae387450f040d1da83c10eb1
+7045e190e940ad597893d85b3336afa77cabe20c
+714db9c2b78eb2e4b26fb94c3927bf372a993900
+85ea5ac1300194927b58530756575dbc84dd46af
+18de5b9eb32fade90cb550ca65052bc1e0095a99
+667ae97a088538b0b321579c5b5bd12fa101e04a
+da9609278b099c165aa343793bc2e03c2ed17752
+27bb197b70f0475abd00cc0db2ffa53de84c9e75
+debde12bfd41f1960cacadf1239f1b50db2624d9
+c7ed3ba86b0e3978955714855a42b4a7d8c67233
+e5572a346b97cbaadbe68f0ec35a09d923a66383
+a729d22dafc85162347b87dd530c05caf64ae2f3
+c12d47630071fbbaa5b10507aa97f02c58aa37c7
+6d9afc0ab84073e890da12d0332a5987ab659d68
+240b251022182eb14ad96aa9f558150f8cb4c543
+000e009f6b1d954d827c9a550f3f24a5474ee82b
+a8fd170d0ff3f6178900977ce422ddcded7c6c43
+8c8d04a5274fd92716fcf0926aea0c06e83e7987
+61e06a233a45c987979139488084ccd0012d466c
+6d5f7fa2062d3e5ca89760dc09b13a16199d1359
+7e84933971ea1853295b9d73e4b75f3478498c72
+e10c7f72bc3de187cb7adfa31a1f098d0f47bbf9
+f450b642e3e32641ef9878aed22f732d314a8c4a
+ef17977a93067d945566356f538640febe56157b
+c2d40cafa9bdfbf9d04d096d09a6aab9584c3ef0
+87b1981c3d51bf560e628fec4e65e4bc8f54566a
+f9f7fc8e9da723e776abddb6f7e836fe72136eaf
+c2e3a01f8e6f8a021a1551f72529f92c9a7703e4
+d7dcb4acee7cca3e54b11ff196c0c26528e665ae
+3c2452d9487b0b6b5426fc7c502b4d8115236051
+e430e7bb4e1316f3652bcf9ac93ecda4aae3729b
+e21cec4d51eaca6fef39f717a12355853c8e25a8
+0a5c494f13f21e009f531c2a56543d274a8c5932
+3f68dbe78481050ae64297153361f374956140ce
+a54ee0a7552c6d6a5fd5ff6e0b67ecf511a8777a
+a1d08c47cbfe06de1206493bee12f301386725e2
+a91d9dc6a9ad7fa6aca2c9ca4d9c7aefd1503585
+991ee479aee8194c495fc11e06f91f7b33809161
+39405d7bb73434ad12a0106c15cc194689eb4de5
+89275be2a434addb83a29a275b63113f4500e328
+24a1a847d4d5f74b57a02c5898af9364aa83debf
+6ef88240d12dc57a102450dd26ee7a0510a848df
+3299e38fb5351c11d9beba7400722773f3b74e6c
+99f61cef6386573c8cce688a30fa2cc82a1dc05d
+a49f8c20eea4af67ec54408cca737bff98628769
+7cdffadd7c11226ad6973a1707e404fef96dd541
+d853339b5945d07de64c5d8738e89259dbd40401
+137797fd34ea672ca506d1c8848d0da355baf7d9
+9b00d0ce1008b4c3765bf57817dd5241bdeb5c8b
+81984aa7d079c46380e4baaf49d078c86466edc7
+78e6ca8bc83669866fdf9fc5ecc19797f4011261
+70172cb5f244c48d2cb41621c35858ccfdf31997
+a509b26bb05b83f9fbdff3465a2acfef5b35ae30
+694c620e98472213a53e932214054137e278a073
+67518afdc981945e4cbf620ff05c773934607a44
+2d3dd578113df1602cf753d2b11a4e802f616990
+4abddb8c6e0df6689eee21bfe27aa231d0ae8dc9
+ee3ed04d53a5c7d1f60a5fc4d7c6832a7f32d3bf
+f76269f3e3c431fccf5d9991a8a5da27977646fd
+ba30d4f9339aca62283d1df7756b42158f637931
+31dc8c2da848a7eb21c8287e23990cb3ee8b6307
+68ac8fb1f847b3307be2a6a9a0fe66235a5e8c4f
+8ce0548b386ac1e48150a945cf36dbb6a0bd0ae7
+2b4439bdb73d8cdb6637d275f426f13135d415fd
+3b0b095afa3ef1b73a2bae29a5a131bd02c0e714
+9170bc3cbca5d8f82b02ae1e33128c62fa2a00a1
+4ecac34a02791711bc456edca64c086b9aef357b
+d70512d3069e6532b7069fd0c8fda28d75324293
+9e39cf719ab85cc10326ed1d9df2273e75b67b89
+65c44ce096871da2588e1c140ac91ef771fbae97
+a8949852e1a6258f3e7146d5a0b073861d12dd56
+4baf74306ea4d6d60d89c2575484dbb111cbac83
+4a410aafb82e10f1cfee2062b5cf2e038a3d12af
+7c7cf4f235cd2c455b2826e96803b1a6a47ba4f8
+d42801a7b4d67e49ed3d417db4efc7dc6d4b5ce0
+9964dc1ba45079060e594be6429829042854b4ff
+db275395dd0a2455ae378265850a90a3025fac09
+65f5e35193414c2998a1b5de2f959cc785f1fd6c
+19c0f1cd0bf5780a7c2a8abd5d5d8dcdbcf2fb86
+279e575b7c82e95beab30d37836e1e56176d7ee3
+3945e22c503109659c8d463d4674d153a6f5e8ec
+e553763df924e731b9aecc68342af73ccf47ad2e
+128c6f4780cc59cfb76bde414ce42bbd544efacf
+ab3ae67419bec16ebce20cdbcf76f2e8508b35fd
+42986dc0132012b6150eb6066fdc1047d57fbf29
+7ca2a2cf7fcf4afe8dc3ebb4e7f8f9a599d5748e
+7c478b9c7f099c8149ecaf11b917d41b5cb36011
+6cdb121b56dcc9cced06a26d1b11bdc907e4aaa6
+2d60a7c9131bb6044c9636d42f6888295a519dcc
+883b0ca02ebd243bb393bfc6144974539735d64a
+d4c710c356bbc78529b427336e4bf7163a904239
+3952896b9156a43e6e2193cb3ae8a71a0cff6923
+79b92f4b2e09bb0bab7af1b036c03bcf075e2682
+6e0f11b530638be7f478b43a715dd3bcd6b17d04
+e7d4a4d0b37b35569ce85c261d6c9ce9b57558d2
+bad9a32c0b3f74e4f54de56f37f24a265f45fce9
+4ec3467fa91e3889a3ba2d695c863b5207ebe9c4
+d8a1ee24cafd2ad6a648b4b62be7b06f446b1a89
+d61dcadd8afce804e85d4b40ec5eacdf37f04fed
+1a15d2dc834fbfb276c67a2cd73d7e8cf650bfb2
+08d5f0581bd3f7196c90036e40f615c43c97eec5
+020a823e09ab8f8c2f13f78aba48cd5549848cb7
+fd091455b9ed6ec71dc4eabf4b59ffb5650ea2fc
+c0d25282a77168ec25c503cdba87a0b16f73e759
+19094aa75ff7ab5a9331eeefd36c15a201b0ab62
+2e74bfb1f3a9e72a00f6727067bb42cc6d8c4db1
+e3cfe5af31c4c3e4cb9bc30ddae635241b476b19
+d030d0a5f16a4087cc56137190fa0e7ddf19dacf
+fb15d9bff157666a98e09d0e75cb0f05d9998e51
+64cbcdde35d49cb5220009a855561f6a440c91c9
+faf5e36c67de12654252e4890b40297de6f0f18e
+f7ae25006ab8f8e2beeb218acbc5d273376d54ec
+0b95bd0ff7b9d14421fba10a50634c26f3bb0692
+9db0f5d741239f0adf441c69f7037f1143c99fa1
+1a528a69a27510d5b3036ecef3f8ce416cc8a9b0
+3f0ba68515f730c5edcd6b5f7a2487672238b381
+28c9b5d17fd3e52e27b5b8d6d5338f823f8abe96
+52a4638c81b3feb5cbd2b66987b1c7fc1ca7ae59
+10af2962663aabf4b56357038b430adb7b2d0986
+65961456d11269a4191a41b0f0a0f2d92fcb6907
+46e7cc2ba0bf218cb004f58ebc249e5e72b8c29f
+b616399a316a7816941a498c09de81c3ecdf0f03
+878cbf5db93ed95a2ddac0927543addd0d6105f6
+6d48e2be404813f7d346516d519369ede95d7226
+c968e9268088153bcf51f3555b80f69e7f162db3
+405f21274bd606e89a0366cd8aa82e6dbaf8050b
+2b9872f1248cda295127c4374dbe49850b81d95b
+c94ea8bfd1a74b0d93a2a207a1234b0ef1f73d0a
+b0f4c1bd78e59b33cd73b510dac2b45e3cd735ef
+158c5fc595eb5ebdd337f44438d98d5581a87756
+a210729626a48d3c75bf2adef15d856d0a9e5918
+248a68920a184395f2fb66fe69f7a2b1276e0f95
+8ceae55b2d091350328e94bb7e3ad1b2048efd6d
+26c43e258d65949742057d164454efac73bbb63b
+cd76aa45608dd3370639d3ae4d2e774ea7c3e5a9
+b645b284600692840ecb34473db3394bc354472f
+9caea797113b583b5ab74990ea22db63d14c2f99
+9e9394a307c29b74289f20464554131438b34216
+90519a813017e881d6d95e4df8952a393d1a7726
+7dc1bba5f4ccf529d19ed517880a10491df307a0
+653abdecd41eb6b1cc3315bd4a6e5819d1831df8
+a644add72093f735a99ce94b304e91703f250b94
+ca1385936bd95b3005b923bff4ff0077816e3d68
+e85217770edae3f88d5114fec35166bf7a80e4d4
+47bbe920c329ac749fa3dcfb10570fdeacc6fc3f
+dc36563ebbfbccc065d91fa24fe84f9b0402ed68
+1274f8b5947b2a5f87801d40503ef5b8c883771e
+694ec8ba0a9dc85fe62b5cb5041b71198936be89
+c2fa24208e4bf7d91592ed094f88713be35fa708
+1ee394eff8bbe8488411ecb68712b0a6f08280f5
+2849071dfbc3e18241f7a5243d4ca06e4418174e
+c0880199e5c76be3640005137c2c383f0c84b57c
+f0814435ad279f8e908c65049775a8676ce15f94
+8669e9660c67b2489c0e4308eefe20b8fb3d2cf7
+1c61acd00fa431d425fc79b0c90ecefafecb3ace
+999e5dd9f1857d3f65650882fa2cf6d19ae3b9ee
+04e19a432042f4044bf0d51e3657f890f10cfaca
+91a2a3da8d7ab82cb4034056381a44c4848ff19e
+1c0a7f35b6eef0226ed7af5ad8ac87ce07fff38e
+3458f834c56fbafded76527b01578e5ef34b9b42
+80bb4f72ab3dc526c23a3dff758e7777cf1b3c09
+6c4bccb7aeb5aac67d498b8da720199c63e277c1
+9061d2012210b95c86401af9dacb0e63ac871657
+fbeb39fdb55d24b827bdd578cd6a471a0a1063b4
+748b0badc59cdacb0717ac7a55a490f7e0ab4d71
+70952d29c9b7db955fbabde8800a629665a0d24e
+a576c35a831b1e889631b757ed86916341fc7202
+a5858355506446cb36f949d98bfa811e7d37e76b
+a5f97dba2ae0b7949bea49a0e7068a1c6ac42ae5
+1c16e278550ac208d9aa1a65d0a9795f4132bd5e
+5a2cf867e368a77f135a855cd1de59ba5fde99a1
+132d7c4172cc25eb59c7745e6d74cc4a4dd88dd6
+061dc0ca6fa3ee55aa7e688910169e4e6c74257b
+e2ee40eb4145cb1450572a7837ef544802b99866
+b2b0c6a0f14f3b76df69046861fd04972ac9f3ee
+dfc83bc2dc59d24775e3e8228beddf9e654167d1
+8ea9a921345fa2ea894bd9b953081f15713224cf
+c6b2ae26499e736ac081af57a7b41c39a7b97fda
+abac6c071e35ac30cfe3317089061124ac301495
+5e0f057dcb8b6c21806b379cd349d85598f5bf39
+b4623e58d28e1d790a508d26b754a752f70c288b
+621f194999cbead9449bbd7222c6e8852c5043c0
+5d952c025f36694c06917bb1a5395fa13ccb84d1
+a0fffbb1fe7c929f520de855ce045b840272cca6
+e2e8fe4405767b62766efa00f95dc7b501e9eff3
+d24495608f98c48e6f3030d4af691b009d09cf41
+826c9569ab9e52eb031dd692baf84337eb217cd6
+470ec40f14d3d077afc6702a1c1c0bb4baaeec57
+281237526c3d4125250aa204bd6798e16cca4bc3
+2fe0fa9e25453b1797f4cf786c40eddd64483d3f
+d231ee5ec82309024acd028a83ae876d9ffcce94
+71732a91bd25ea50aec127f95b7f8b8609db3da3
+793ec3ff30c242c570c9a9e8c95d78b05c7489ed
+4b5cb898edd34436e4065c5d3de05c2ec7d95153
+23d0930474aee4957dac9571e06d40757b5535aa
+992a3b15640c2613b5481fbe2cf022178e5f3ff8
+e8c0ce0985596758a82b71bdb6759c72af43d06d
+846bb87419fc959197879e04dc9c15f3723555d7
+8c6907fb70ecd74ffba960283bf596155a7fb273
+1cc3cd345edba8eedbcf183afd6e746b5b29a422
+f3e48a4d193edb98933989cf54dfb46310ffdd9f
+30a4a422254a4026dfb77d2660467994b18b1eb3
+5f5ea4d8846ad79c33bb149e6acb853ac78b4247
+c6d0ae6864fc9ac5307e23d283b1ca4b291b21e7
+2de834aa1cf63d1e6b7098c5528e4d021f131f00
+55e79f31165cd20502922ceda572d3b7db9cb41c
+6ee19b994fea7c9447b05e9dab49350e2f8c1377
+de44cbbe8fd64ea13caaddab77560a48806c2180
+a2619695901d714b44c3941aad3689a40abcf363
+485385e26c8fae0a7efd34ee11ba645662074a13
+8035d023b91b92978788fcdcaa6062c38883f4ae
+38a7dee7cd042726d64a95d3c5c3d341d656d68e
+0a280b46be0fde5d87ed47fc7e970e3ed494cad6
+aafdf3a5bd71126c9ad07d93285966dc04d40c85
+93cb2f4e32053398d3602e0cbdaa12e8bca062ef
+4ab039bae14499bc4f432f9f20a2509fe9310fb0
+29c411d098ce2a631503bc168ebb0ee6f65df497
+88ce7a6fa029b6e8f51c92f1666b02a404b827e6
+4f98a1294468c67a563b48d1ab6b4766a6d899ec
+ef0e221c7edde75caf0d5bedc0d93745890a854f
+94a1929f8845d841bb1f47667c1e489ab21bbc56
+4ab4b43b33178d53a5a445283d6be39ac57106de
+5ae9d4c70fa9dd4cc70037f4920ef15d4fc63d2f
+08af6504291f39451465dd1f1df6466e61c4595b
+23be51cd2399b9825facbac2a88475450c5927b9
+750af4d960982c1655e9edc08470aada3c72a9a8
+47f51c462114ca3590df18c3c96da04a217b79a1
+e9e46f551280ffcc98b45c3c9b18085ed14f38ba
+cb0ce99d40bb9d942aeaf08cbb83b075927a96d0
+62045a56ba933fab5e2bc61be05eb5bfb81a8527
+9112ffd4afcd36702f2e6ff7aafd653edf2557f6
+b1fadf90fed9739d72e98b56727e471e070b85b3
+75fba14591fb8de7567bd2378b5c5c114bad77b1
+7e81e309cfbc2385f04c4d377c4562efac6ca238
+0a38f333c2a4adc64c5f8d074508632418074755
+f4aa472201d2337eef2115dae23439e0a6dd9663
+d1adc7acf92d644b3e1a821668a05b024974c350
+699fc0441179a3cf82b303cbe25bd5a3be551ea0
+697a6cba6628b2f233f2a1cf317fe8127e4d05eb
+397e0ec274130aed3bc1bdc461bad41c485f629c
+02aba4aeba128defbfa587e3f07efdef724666b7
+5e31470b18e9dd499b9f8787056cf0e68d52e055
+9507c23d0741682f71cde608ae517c0c1ad2a4f4
+16c115ba95f7b71292bf5c00a1d425a8586c551c
+42ef162afc768b86b881bd4c59beb8839149d76a
+a7714cc78a381995633711d95465e883b613ddd9
+91519a0367b9eaa66b5ffe27964ca4c913093aed
+e90d736f7f5e32ff845a898036c529518cce0c6b
+82b0860745881e030c57a3d1bbfea46bc404bce2
+26c3e814a69b6335ab65aeb2c4a1e97015595206
+7990c00e8ae3117587f54a880d7d20d0578d4646
+260d91a2ece614587559ea3bf37f76e4d5a48beb
+bc19bf14032da3bd5d3e6b86fdddd47f80152747
+1badd9612877ba84a92b025096fca1e0a36f07e6
+4f339c24142d0442f20301c1992d523946d1c6d3
+fc89b29738b18fe4c0ece96bf00f2cbe687e45db
+ae3e7378f86cac99783c3de50d0c073e79a92759
+a1e6044b7e31b86d42b6dfb7ddaa1eb6bac2070a
+948ccfbc7690989a96170839cc5d622e12e0b044
+9ec16c5ec0fd561efdf57572fe22e3e768ecfdc4
+c1849b79963362d71d09ff4cea2c46f9b3a03d89
+e82a29a4c2fcb1ecaed942c6fb550a14b916345a
+b8412b898cf77763bdb3da689bb1bc9d10447116
+147a073799722bed54c3606c8833cdd58b1aa1dd
+79f4216bdb44dc618f168d2a5061481350c9a38b
+c4352c74759634af80f1f6acc69c55261dd12acb
+b6a68adc1d771af97938d64d3c21ef4fcc99cfb8
+b5f8a569e73948a0930d18622740f52ab91c1a42
+fb784ea50559ccf087521510e7760473038cef2e
+b7aadd7612b6a1970dedfdc175ec4780a8732703
+0146da64eabaab5d7f53e1ce58aeb9e74dea18f8
+7465d748cb38e50921b446953ef27b0c0fb6abe1
+4cc4e5eb162c622f786e1c98a9e00237f5687ee6
+a95f85c1672b2ff74f860a5980d83b440715deae
+afb99fceb6c0532a769b61a81e8dfeb7cb70a86a
+8a7b29224fce56d21e0b4d8b83cb42c32a4a2e29
+2a4c040c4c53d763d1263d8cf797e0b672c154fc
+a81140222a3c6e2323cb290f353d595686473491
+642db49c7519de4227b0dca5b23144945bbf54ad
+40df1682f34e463ba031f077e211a1e8eb1b7e0c
+cb778e62e3d6b15a836e50d65a18a269a8a82577
+3aed3c2855ec616b87c4cf79a69298ba45c427a0
+f18f929339d5ab26ce8e26e716da4ad095474768
+8e03a13f5223f71d02f875b4fe4e48cdc1ea3738
+4443f53a766617ae7c30c48d8cb55d6fdc3ceb30
+f38b2e2284f48100513689571ce9d41cff63bd4c
+a5182de1c12d3ff131a5dedc6130e02b43c3b267
+9054ed7b6b3eca10003d19098e5d3c51a8dc071e
+2f0833d2eed57049454b3e0f41dc02eb7587bcb9
+a9883044091acb92e2edc709d5136af372d06ebf
+1360b846750bd2c7e31cfe015c77c5968a9b541c
+156253c33dee7a50df8e9e5e78adcef72705f3c9
+42580982bcfa232a30d39a26fbcd605cb041e092
+5ea3d3524c6c824950289855e33037576e741d30
+87c21fd886502206ada74a652082ec8dbb0fe7c7
+958b77776ac602ff78ddee5ecc758ee170cb5fda
+0baa5e5fe65401be934349bd1f067b31a4a0f0d0
+0f18a06bd539d1de1e3abf38469e1d14030ed41b
+cc06d014fb7fc7f5d0fae8c3576c134a281ca14c
+cae072ed5710b9cea48c8cb0b011dc3a9cdceacf
+cc7928b6593cb03aa125a5865684da7fc0405d74
+9f71dfac529fa72a12235f016cd481b02192c3bc
+2f77ee6c43941f768b9771bca5a02332d89bf80b
+cb6bd932f7a9e1c4845bd0d974f8983f2d5d6968
+a3b808778e2f90b6605b09b68cd7f3eb4659477c
+7e7f7e823c05955c57123af1b61dabeaa5221825
+784321dfdb51fa207c790a3f670f0022fd575775
+69891ae41f6320ec437455913c5dd6d76a0241d0
+a243d306ec4c022a4199b6e160bdeba677415fc8
+1990eae2d51f66f9b5dd3b2d2beaa17f2b95599f
+4d55ea7163e490d3f37218482269961898c62a87
+4d33676bdf0c6738fa3088d5e972ceeadd3730e9
+54aefea5f3c14105ae08f09aa60ba5f6917b1b88
+d0fb72f2df7282c349193c0ea47af281034a2c32
+284f81ec4d1297d3949ed95a114f4c10011abf40
+83fec7b4d265b21ae38e07c6e1046416b7758993
+f78bc049eba41b15d9e2ea28bff38e508b0e71bc
+bb940fa349ab09d69edcb5f3a8fe96e55cdb69b8
+4dce40768f628700555244e91a69c5775d6caf6c
+d176f57c12f30fae319ccd5b50b3096837767ed5
+1bd87d9bd116cca4f00aa031cab25897d35418bf
+684d8d97b04fdbce1a08fffb59e1e280318cdfb7
+99b1b03fc906723790db2ebd04ecc51b8ed52052
+ce7ff18c9588042aaf62c8c71c69f769a16c4a7a
+ee14d42f2b34f4bb5bf90d8c813934aa5d6b5e01
+d7f730f7658fb4af7d492e848fb759d031726e34
+caefbcb40174cb97c8361dfaa7899beb20202509
+fbc427e1bc2cf82ac3756c8c7de4249b52e56505
+a2e4de47a027a36757d181f61e2d3fa6dde7274a
+85b17ae766f1da36b8ed0556a932d63bec08c785
+36ebbab9aeba7a8a04ceb800b2e445a85e4b2c0a
+5826b9a1cce4a960cbf4516004b194c988312730
+a06ecf2bf25af0a6b32be1d6a82ba618d9ecbb33
+8f03971de78085457c1440e3ca545ae5cbb5230a
+06588a8ab74f068ec61b89de9ca03a28f5ebd6f4
+72bd7e434c944937912039c7cf79c07bd40241f4
+14b5d1ee3b508505b96a3f403f1b6685e110c3f5
+6ce0eec1ba71291ba928d4a825e582c919a2457a
+c25eaa87d7ff1d1fd503bfb7049a41bbf282e916
+0ca9829040ed3d37f3df6341e28becc8df839409
+4e516c3549d4aa6a057dadc9f9f6f9aeabfe35db
+8a63d4ed82617bb5f3da2ab351138b4690c9e03b
+0956df18b019953eac5eeaf6eca49674af37e52a
+a8dedb9efe2e9bfe658503702a0602fbefcc3316
+c0787c279f755fe76464ca4fbc94e24add71e3ac
+b11cae312129d1e47a4102f87ad8e1f0781d34c7
+248e4202dbe0d45e76e930b614578206b3dbc383
+a0dc9ceccd24357326241c97c07df17c93e77420
+5468413e75a18f8d7acb2d26c2b80bddfc9adb99
+7c81d09ca7a80c686ba8530986cb53e555eb60a9
+8df13df883dbd7e8944d8098b74ebf3aeb4b735f
+e2ee2a92b8f493b2960c4e1ba2abf4f2a54c6758
+c470e5a7568645a10488f402443f3701f69403a4
+23e3897002ff686867b2372767d5d8f121cc9b4a
+f1f670d0fe617fb374b15bcc20110b89b6082aa4
+b17028b6a57a1301be1bb2021cf51d6fe4bdd354
+d3c8813d44913745f4ec4253e048af17d4cb159f
+b1b3a8940587229a063dc836cb0422065ac0d292
+ff7672f15b344e93c02d0d3b9676b8070a735e93
+331073170c761735eab3c9a516903016c2aad8dd
+07605b39c50fbc320453c583ee749ae4f97126d3
+e710845e4cc7eb6a1d99073dfbf6f9278c24bfa8
+0cce9dd80952ff900e8704e6115f9c1bacae894a
+2ffdb3f488210d4ebbe41759618bd8c6d15878bc
+16d1cc466220c90c009bad3f09c2a085bdd47d5a
+1a71e84c0c599408ec18a189dbd779d5e20d4e21
+51de15d048a6f3b0330e8da198b2d17260ce8c85
+36ac969d234f196366b404c9c714c3b8d30ddf6a
+244d6af0cf929f993a2ed2de0ede4f57d501eade
+a834e49430e3b3b1cd596dc1338a028e7166643e
+2e489b53225f71cc5b73f9aaef5c692737c0f6bf
+1d52cef8af071ed110d3ef8feb3e4b275dfddd01
+ea0e27967a6c62875355c5f423e4962835c5921a
+02535aacbfed4c3ab00a0945d59933dab54f6fd1
+0cf6cbdeca67c729a260b7c1f5710b7a1e0aefa5
+ca42191efe091ea06d25dceef9ebd84df8ce75e4
+c5001aeae4fe17f8b7ccc1d6c604727ae63c35e7
+730b9668fda289b194a3b66a53fd3745ef42ca32
+3207c07bc7ace3a01ad233641f1df91ab37a505e
+de657634cef20a388d43127a184619105d110a27
+dfe850a4b3c6c002dbee134a112f16f8e1b974c5
+1b0e7715e01a62130ac573c38834b09274a7a866
+99c6160a2f6e22b5040bb47a279f81b4224fb222
+ceadf8419c256716569dce2c60d98dd703bf2cb1
+d8fb13c8444f71e7f309d7ccdd7ef329a47a4df3
+75805d5f1f22bc6fcdff850c88a4fcce7dc3e17f
+e6bb6b913b34b30af0e19a93bca4f55b39579f88
+c29d27094de54106cc903c2e0dfeb89cdcf9ae02
+41ae19f40a339b6b47fceee00f512d849df292be
+04090c2dadbd1d446a8364df894344687131f841
+ba47d6e2e838b11290d702d1fc03261d27ba59d5
+ebd463e2b4b89a626e16b43071b06f3145cfb661
+78141478f00ac19912fa2b283e8c91e30eb3a7c6
+d8a4f3fba1b67bb6848489e45a92e9c1229ff7d2
+929b032a966f563e8401285e4d96850b17f640da
+82a3799090db99bdf611599094170b85bd4eee4e
+620fdb835eb7e095e9a34f8a165843f81fe50328
+66f89413b6f050fb903d58b36ec961461145af82
+fb824fa4ce932e860604ac21db4b555c6ad1114e
+d8c925f283216521073497659088f4ba707311c9
+a28c2815223f89026b6a198415a1291cd67eca0a
+8201b77f669191dd01caacbea1e3b5ffbab92962
+dbfd44e667bdeeb17295ab40d123ddae70d3daff
+f324f1736d24f14c7685df0f2a2cc4bb20999fa4
+296f977687e8ce959a2e38129ce1c0d31e755d8d
+e4888dafd50eaf43e1476701bd26bf940865d973
+17ca1470986faac5115d246d3f9b78244b7215eb
+0904469f246fecf43062b2863bc81f730a96b20e
+09b7e506802fe6fa4a12154e322dddbc34553f9c
+b526d3ce8d4649e96446e1e8947b674001fe16a5
+46665024a071b4916afcae4b9ed3cec0aaeabc7b
+36fee230f41e1fc89a26b1b7bc7e884862dbf56f
+702806939cff2095b2ff97a08d84bc14d1dfc5ae
+30ea107c7831a846dfe6828947249489468f3ef7
+8d51b266df630345c667bdbc07f172b906e627af
+3691b1bffd90518b4017ccefd8c15ffaa8d87d6d
+6c81ff344b4285b42f2733cadf42536addd736d2
+6c15f6261e0d7d09ca59071955ce30d09bbe97ae
+aa3acca1a17c375731214851c56020878929a068
+c4cf20cc2e3665ba0b7d948683bfa1e82aa9b7e2
+9ef570f878a8c2d9460a99ca523b835535de67d5
+7d76f893313ec0b855d1dd6ce9b8fc9bc77723cb
+e7ad68df97b2c9bcdf6e56cc017301f84a7f9b4a
+8f5a3860948e5dc213ed825fb4715f0ffa013ce3
+27e795d99164a2372106c9e1f118cc19258e41a2
+fb438b2cad9b7583f4eda4fbe6fe9e9cd1f59f10
+6c6e755b03472223c69700bb166d81d9adf080b1
+8cfbb990201cb91fff3db779885041d2b5c52c1e
+eef5130bd17ede5cacf8be5881eab0c09a538bda
+f20d8a304a9009a79a54867664bce33473947272
+aef5dcb164dba680b436bbb37faeeccbbc4fe2b4
+b412608a7f30af28fb8615e4b522b7dcecabe212
+799415d8ea5094bd6cca8c178d6d8531827da191
+ef5ba9f7f4c954dc6208e9a47fdaa730602fa27c
+6204780ab854a5443a52c343534637fc227dd70b
+58733bf4d2489d1823a432b2f515f22fa835a88b
+a8f30d02868c8ffc924271d9da99e0c180477a1c
+c8baff658f6506e04d7f530d9b266ba2d4b632b7
+87a31d871a336bc60987492515a20ef25d18d0d4
+e2929ddb475b033444f85c3cf7e5ca38e84ed7e6
+c7fb0295ad6226798e65332c841f6a1508eb9efe
+db80674c14610f0b964fb574ac32c6984cd226f5
+ebed652b9c7ae1784ab032b2023445e8b8cbaa41
+08ac1c4c2c7589f889b2bce3687a724d0c636c40
+dae2a0e1c908135eebc98a0db33ca435ebe7ad5a
+3d415472346209c9e90706dfd313728a0ea15003
+df08670661d8887644542806a8d69046e3ba87ab
+32a96ae444a08d6ae828b34539aec76a835a95e4
+4a0465f87d082b8e9a22608da161f232e8d6f464
+c7f47a4f22bcd6f11e6ee97e9687b5e917d9e495
+9b4dec196b29bcc98a377d6f433638a85177e0c9
+c0f1425ba0cdac23bc342587ce6ea6cb53515c55
+ca6f373a6c76d4a4284240fe5e88c130bd56d27c
+785966c05fb5fe10addeca3a86f1857329957fb0
+fd71a64340425384294a115d3a42bc8069ac9f67
+08784cd3a744ca0750c746910124a6056d46f608
+ee9284abb97ecdc3ed78a4807303124652924051
+c9cfae108e2aaea3b12010dfd0404f7bbffa5c2a
+d7a63c81f8bed7df99b942d88a380c100e74accd
+23fc6eff1bbf238513e2f9c76e40762f01b1737d
+485c7afff53fcb4f694a5b3cfdc09c372cf73e18
+8656b25529d3e5aabde19eb42d10aec5d8af2088
+fe954e108708531e155eadf4945fff5e432c57b3
+0f5fe6ee00187bde832acb092e42431f9fa8430c
+8827ce43536f7246e21f643fdcc1b1ad44c05a12
+869e1a290cb6ce44eada26c00d5abee0e5c2ecd5
+a2215dc789a33e1ab3be1dfcc03e8f7f02d046d5
+62a233d2e55b159001ed622fb96b9444fce9c11d
+26fff6559df5149b98c3366e7c01236daaf2b1d1
+115e024f021871b307a7a315aef720bbffe1d54c
+19719df575d3ae0d8c93c037f7f1972b9e10f1ba
+3628f33e8ef1350912bab8d4ae467c7e1f3056fd
+e123e08e23278c95e399b3b11da411325135da21
+77b3598df08e6f3a2b4ae157904e30d5aa2ad49a
+e071ff877d67787d0a6582ac3dcbcb627dec9ac4
+722a05a34115832ebdfa990a99bd999d097a0ce5
+3dcebec3361c047d19cf639879437ed5b769e7f2
+8f37fd4e1147e623fe6f8cc6d190c304467d999e
+5339c690ad044e082f8a31bfe929099d7e75531d
+82fd9658604cefb93728b198e73889872ce7d70a
+804983224e3f5cccbd52b26bebc53b88369c448c
+562f9fdd5811793c11970c856d21c7f0c32118b8
+5914ca61115649643f88ae110eaf3da4b112e6e4
+f44b0a2d303b725a7f5c82048d7423858e78e490
+de5e6430a7c1166ca82500ab7fb82cb95cc643c1
+c174eab1c3615c3ba5dbc0c6c30ac67ab6b47024
+aae0341ca8ab04c9c169f4dde3e2e943d758422a
+517dc966b1379d84a9ef741ff9ca43e281868c60
+436fd9441cf9517b6e8b5162db78031228b18d9a
+cd7e1db2eb4709309b43cc400d6619aff480484b
+5c100f2e25d49a90b25685b9d3bb17a35e325374
+362892feaf8dbf44a0429d3676f9b5e4ea6a46a2
+ca8c5f96adddb61025107907704ec344143b0088
+8c97077f3dc6794837f887a8d57bc8d3c05e8b4b
+04ba3c53b4068a8bcf31bbfc674d520ab2843a2d
+c91c677ceb1093b393d46dd21252147c3ddecd1f
+d85824d0d1dbc389c30ba584837d82e85c5bcd37
+c0f7e29dfb195770d68e6ee608c7129e72a89e23
+e55510a4c7ea27d0e47137479fcb16562f8d380f
+8845f1c8a8b45987a6fe69bcd89060ba38475d2d
+8ebf95f844971fbacb819e2e05fea4e27402a34c
+5be6327602aabcb3fafaf439f69ebc621601d30a
+698560f44a2c58c87988498dcbe51e30ea62c989
+29f215bf015e848c5af9a9c70e1e3e052016704f
+6211582a40d5a1d67e930e337ea11f1b3538ef5e
+80cb64b8ae5710be8044127b678bbc0e010e79a2
+7e613e66f3b7da299b8f4689cfa31da7bb381e31
+128131f6fd6e7bb018806ed5f150a207ae8c7e69
+f686c1c3a2fce19f177aafc281d6c724977a6dfe
+56f58b9bd5e4a5c6aec7f2c5a4a04a702fc3f2dc
+e63e6ef318e5cc205518f7fc052da7020742f55a
+c19c8562df56700121a61f5cdbc8525a46197e1f
+7ffa78b92966e11b0142829ae17c871b9f6b5c15
+426952f1145f112142141f26556313924ce7465a
+f975e857f57f0f6d96ed50006a7b4e126edf1f1a
+8a6220895e1d634d0aa0f41ce6882c98d7b495d0
+12da0f4b955b911a893158bd3beb9b24f1a0043e
+ff8441521f15f11db3c60850a1ee551b81661fef
+0b88599d7b1e25e59f2da8338520ec3325de9337
+0fda61a11326021d7ff0071b6bd8b2b3517100c8
+acdaf288f8a96f77e2c34104fadf26c04307f5fc
+16d04e701ed59f32ea3c4226b553b6f0f50c7426
+7b759405d39047b5aa0f0c22d91c3d254fbaeba1
+facb5a7732d083c66484c9b3dbb274ff1d6a1ee1
+f959116e0606392633e8d8eaeb710664e4532c6c
+febbd51aa5181f74d56f3d0e01d38e264444f825
+90ffbc94fcd43cdbd2e54f5cad75d2a7d659bdd8
+61cfcbd1b8ef945165acef5e7145762bb510453d
+5477b6eb53ccc404db0ac820d3d858052bdebbe2
+4c6156e3830087141b0014005bf955f1a87e1edc
+12dc55dc446574144eb863292c3565736ce0bfc3
+a761ce0dc6d89ad3170a3b69e3d2c71bfd014b8e
+8cf55dd9b1bd7a4c8350c81e469d92ec956af62a
+8671360c5d830f38316ccc4f63362ded7a2d20a6
+97f1a15d8196c514517e76f1d80571fa769e28b3
+85b2c0a31be506ef27e0ca124be5a89c628de120
+935dfa6867b975280d63f75cdef228372adc40ef
+63367984bfa6dcb0ae560d7bab812c622481920c
+ec10a4353082865abdb5697560390653c3902065
+b7974f532d25aa2eda5e16e5dc58d3f726373c03
+f804d65a6009874a0c4d555b6e9d8d14cbf935ef
+cf251f22dbe2c976d340eaa8469e1ea21ff88a42
+6998dad2b81635da9c328ef829a8b1125393c38b
+2a073e0f2e510318e83c16ad9141f5c1a31cf6a2
+eb6f4e52212ccb96b2aa08e0af5949dc8c67a024
+09b9f830b520e68635c45a604641874e0f2bfeb0
+17a33c4bc2856e52acf16f3f86dd7053e340ffc5
+81f4e9eee7d046992f4091cd2d82a6a82981b354
+5a6746c9041d494e8f794e4ecfb6a7c941f5ccce
+5249fba450a5865325c2b47ce5fac5a585b2ca23
+e35df1cddab6e311e0b4f0b732c555c51e8a739d
+8f95ac3d57280ec506907f000e60b9bcb065b4bf
+2750ae3dac18bcf9eecdf9127e5aedaeac19a67e
+dc4d88520f9221eea943cdc54bd89e21e52677ca
+bdfc42f3dce77e9e964ba2922c19faba2ca563ee
+c3b349b83e4fa2389ee59ea9ca036001b358ca02
+3c992e03d64ea763d4b6db96e3371143294172b8
+f40f581bb9a644dc31feeea1bdc3dd6bbc42ccca
+d59c8256b9451b83457299244fa9f81d0369081f
+b015c20c7868a98a3cee9878553502c708fd96a0
+b6e30268a7f110d767dac9144454d2c6fe49eb34
+dbfc2a5e7753d96913593c41db73a32dac062ff8

language/gpt-j/dataset.py

+import os
+import time
+import numpy as np
+import torch
+from datasets import load_dataset, load_from_disk
+from tokenizer_GPTJ import get_transformer_autotokenizer
+from torch.nn.functional import pad
+from torch.utils.data import DataLoader
+from typing import Optional, Dict, Sequence
+import io
+import utils
+import copy
+
+PROMPT_DICT = {
+    "prompt_input": (
+        "Below is an instruction that describes a task, paired with an input that provides further context. "
+        "Write a response that appropriately completes the request.\n\n"
+        "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:"
+    ),
+    "prompt_no_input": (
+        "Below is an instruction that describes a task. "
+        "Write a response that appropriately completes the request.\n\n"
+        "### Instruction:\n{instruction}\n\n### Response:"
+    ),
+}
+
+
+class Dataset:
+    def __init__(
+        self,
+        dataset_path,
+        batch_size=1,
+        pad_val=1,
+        pad_max=196,
+        total_count_override=None,
+        perf_count_override=None,
+    ):
+        print("Constructing QSL")
+
+        self.dataset = "cnn_dailymail"
+        self.model_name = "EleutherAI/gpt-j-6B"
+        self.dataset_path = dataset_path
+        self.batch_size = batch_size
+        self.pad_val = pad_val
+        self.pad_max = pad_max
+
+        self.tokenizer = get_transformer_autotokenizer(self.model_name)
+        self.tokenizer.pad_token = self.tokenizer.eos_token
+
+        self.list_data_dict = utils.jload(self.dataset_path)
+
+        prompt_input, prompt_no_input = (
+            PROMPT_DICT["prompt_input"],
+            PROMPT_DICT["prompt_no_input"],
+        )
+        self.sources = [
+            prompt_input.format_map(example) for example in self.list_data_dict
+        ]
+        self.targets = [
+            f"{example['output']}" for example in self.list_data_dict]
+
+        self.source_encoded_input_ids, self.source_encoded_attn_masks = (
+            self.encode_samples()
+        )
+
+        self.count = total_count_override or len(self.sources)
+        self.perf_count = perf_count_override or self.count
+
+    def encode_samples(self):
+        print("Encoding Samples")
+
+        total_samples = len(self.sources)
+
+        source_encoded_input_ids = []
+        source_encoded_attn_masks = []
+
+        for i in range(total_samples):
+            source_encoded = self.tokenizer(
+                self.sources[i],
+                return_tensors="pt",
+                padding=True,
+                truncation=True,
+                max_length=1919,
+            )
+            source_encoded_input_ids.append(source_encoded.input_ids)
+            source_encoded_attn_masks.append(source_encoded.attention_mask)
+
+        return source_encoded_input_ids, source_encoded_attn_masks
+
+    def LoadSamplesToRam(self, sample_list):
+        pass
+
+    def UnloadSamplesFromRam(self, sample_list):
+        pass
+
+    def __del__(self):
+        print("Finished destroying QSL.")

language/gpt-j/download_cnndm.py

+# experiment config
+import sys
+import simplejson as json
+import os
+import numpy as np
+from transformers import AutoTokenizer
+from datasets import load_dataset, concatenate_datasets
+model_id = "EleutherAI/gpt-j-6b"
+dataset_id = "cnn_dailymail"
+dataset_config = "3.0.0"
+text_column = "article"
+summary_column = "highlights"
+
+
+save_dataset_path = os.environ.get("DATASET_CNNDM_PATH", "data")
+
+# Check whether the specified path exists or not
+isExist = os.path.exists(save_dataset_path)
+if not isExist:
+    # Create a new directory because it does not exist
+    os.makedirs(save_dataset_path)
+
+# Load dataset from the hub
+dataset = load_dataset(dataset_id, name=dataset_config)
+# Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+tokenizer.padding_side = "left"
+tokenizer.pad_token = tokenizer.eos_token
+tokenizer.model_max_length = 2048
+
+
+instruction_template = "Summarize the following news article:"
+
+prompt_length = len(tokenizer(instruction_template)["input_ids"])
+max_sample_length = tokenizer.model_max_length - prompt_length
+
+
+def preprocess_function(sample, padding="max_length"):
+    # create list of samples
+    inputs = []
+
+    for i in range(0, len(sample[text_column])):
+        x = dict()
+        x["instruction"] = instruction_template
+        x["input"] = sample[text_column][i]
+        x["output"] = sample[summary_column][i]
+        inputs.append(x)
+    model_inputs = dict()
+    model_inputs["text"] = inputs
+
+    return model_inputs
+
+
+# process dataset
+tokenized_dataset = dataset.map(
+    preprocess_function, batched=True, remove_columns=list(dataset["train"].features)
+)
+
+# save dataset to disk
+
+with open(os.path.join(save_dataset_path, "cnn_eval.json"), "w") as write_f:
+    json.dump(
+        tokenized_dataset["validation"]["text"], write_f, indent=4, ensure_ascii=False
+    )
+
+
+print("Dataset saved in ", save_dataset_path)

language/gpt-j/download_gptj.py

+import sys
+import os
+import torch
+import transformers
+from transformers import AutoModelForCausalLM
+
+# if len(sys.argv)<2:
+#     print("Please provide a valid path for downloaded model")
+#     print("usage : python download_gptj.py <path_where_to_save_model>")
+#     exit()
+# else:
+#     model_path = sys.argv[1]
+#     if not os.path.exists(os.path.dirname(model_path)):
+#         print("Error : Please provide a valid path")
+#         exit()
+
+
+model_path = os.path.join(os.getcwd(), "model")
+
+if not os.path.exists(os.path.dirname(model_path)):
+    os.mkdir(model_path)
+
+model_name = "EleutherAI/gpt-j-6B"
+model = AutoModelForCausalLM.from_pretrained(
+    model_name, device_map="auto", torchscript=True
+)  # torchscript will force `return_dict=False` to avoid jit errors
+print("Loaded model")
+
+model.save_pretrained(model_path)
+
+print("Model downloaded and Saved in : ", model_path)

language/gpt-j/evaluation.py

+from dataset import Dataset
+import os
+import time
+import numpy as np
+import json
+import nltk
+import array
+import torch
+from torch.nn.functional import pad
+from torch.utils.data import DataLoader
+import evaluate
+import argparse
+import nltk
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+
+def get_args():
+    """Parse commandline."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--mlperf-accuracy-file", required=True, help="path to mlperf_log_accuracy.json"
+    )
+    parser.add_argument(
+        "--dataset-file",
+        required=True,
+        help="path to cnn_eval.json")
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="verbose messages")
+    parser.add_argument(
+        "--dtype",
+        default="int64",
+        help="dtype of the accuracy log",
+        choices=["int32", "int64"],
+    )
+    args = parser.parse_args()
+    return args
+
+
+def postprocess_text(preds, targets):
+    preds = [pred.strip() for pred in preds]
+    targets = [target.strip() for target in targets]
+
+    # rougeLSum expects newline after each sentence
+    preds = ["\n".join(nltk.sent_tokenize(pred)) for pred in preds]
+    targets = ["\n".join(nltk.sent_tokenize(target)) for target in targets]
+
+    return preds, targets
+
+
+def main():
+
+    args = get_args()
+    model_name = "EleutherAI/gpt-j-6B"
+    dataset_path = args.dataset_file
+    metric = evaluate.load("rouge")
+    nltk.download("punkt")
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_name,
+        model_max_length=2048,
+        padding_side="left",
+        use_fast=False,
+    )
+    tokenizer.pad_token = tokenizer.eos_token
+
+    data_object = Dataset(dataset_path)
+
+    targets = data_object.targets
+
+    with open(args.mlperf_accuracy_file, "r") as f:
+        results = json.load(f)
+
+    # Deduplicate the results loaded from the json
+    dedup_results = []
+    seen = set()
+    for result in results:
+        item = result["qsl_idx"]
+        if item not in seen:
+            seen.add(item)
+            dedup_results.append(result)
+    results = dedup_results
+
+    target_required = []
+    preds_token_ids = []
+
+    eval_dtype = np.int64
+    if args.dtype == "int32":
+        eval_dtype = np.int32
+
+    for pred in results:
+        qsl_idx = pred["qsl_idx"]
+        target = targets[qsl_idx]
+        target_required.append(target)
+        preds_token_ids.append(
+            np.frombuffer(
+                bytes.fromhex(
+                    pred["data"]),
+                eval_dtype))
+
+    preds_decoded_text = tokenizer.batch_decode(
+        preds_token_ids, skip_special_tokens=True
+    )
+
+    preds, targets = postprocess_text(preds_decoded_text, target_required)
+
+    result = metric.compute(
+        predictions=preds, references=targets, use_stemmer=True, use_aggregator=False
+    )
+    result = {k: f"{round(np.mean(v) * 100, 4)}" for k, v in result.items()}
+    prediction_lens = [len(pred) for pred in preds]
+    result["gen_len"] = np.sum(prediction_lens)
+    result["gen_num"] = len(preds)
+    print("\nResults\n")
+    print(result)
+
+
+if __name__ == "__main__":
+    main()

language/gpt-j/main.py

+import subprocess
+import mlperf_loadgen as lg
+import argparse
+import os
+import math
+import sys
+from backend_PyTorch import get_SUT
+from GPTJ_QDL import GPTJ_QDL
+from GPTJ_QSL import get_GPTJ_QSL
+
+# Function to parse the arguments passed during python file execution
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--backend", choices=["pytorch"], default="pytorch", help="Backend"
+    )
+    parser.add_argument(
+        "--scenario",
+        choices=["SingleStream", "Offline", "Server"],
+        default="Offline",
+        help="Scenario",
+    )
+    parser.add_argument("--model-path", default="EleutherAI/gpt-j-6B", help="")
+    parser.add_argument(
+        "--dataset-path",
+        default="./data/cnn_eval.json",
+        help="")
+    parser.add_argument(
+        "--accuracy",
+        action="store_true",
+        help="enable accuracy pass")
+    parser.add_argument(
+        "--dtype",
+        default="float32",
+        help="data type of the model, choose from float16, bfloat16 and float32",
+    )
+    parser.add_argument(
+        "--quantized",
+        action="store_true",
+        help="use quantized model (only valid for onnxruntime backend)",
+    )
+    parser.add_argument(
+        "--profile",
+        action="store_true",
+        help="enable profiling (only valid for onnxruntime backend)",
+    )
+    parser.add_argument(
+        "--gpu", action="store_true", help="use GPU instead of CPU for the inference"
+    )
+    parser.add_argument(
+        "--audit_conf",
+        default="audit.conf",
+        help="audit config for LoadGen settings during compliance runs",
+    )
+    parser.add_argument(
+        "--user_conf",
+        default="user.conf",
+        help="user config for user LoadGen settings such as target QPS",
+    )
+    parser.add_argument(
+        "--max_examples",
+        type=int,
+        default=13368,
+        help="Maximum number of examples to consider (not limited by default)",
+    )
+    parser.add_argument(
+        "--network",
+        choices=["sut", "lon", None],
+        default=None,
+        help="Loadgen network mode",
+    )
+    parser.add_argument("--node", type=str, default="")
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument(
+        "--sut_server",
+        nargs="*",
+        default=["http://localhost:8000"],
+        help="Address of the server(s) under test.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+# Function to get the amount of temporary cache generated when running the GPT-J model
+# Varies with the beam size set(Estimate: 6GB x Beam size)
+
+
+def get_temp_cache():
+    beam_size = int(os.environ.get("GPTJ_BEAM_SIZE", "4"))
+    return 6 * beam_size
+
+
+# Map the loadgen scenario as per the option given by the user
+scenario_map = {
+    "SingleStream": lg.TestScenario.SingleStream,
+    "Offline": lg.TestScenario.Offline,
+    "Server": lg.TestScenario.Server,
+    "MultiStream": lg.TestScenario.MultiStream,
+}
+
+# Main function triggered when the script is run
+
+
+def main():
+    args = get_args()
+    qsl = None
+    if args.network != "sut":
+        # Gets the Query Data Loader and Query Sample Loader
+        # Responsible for loading(Query Sample Loader) and sending samples over
+        # the network(Query Data Loader) to the server
+        qsl = get_GPTJ_QSL(
+            dataset_path=args.dataset_path, max_examples=args.max_examples
+        )
+        if args.network == "lon":
+            qdl = GPTJ_QDL(
+                sut_server_addr=args.sut_server, scenario=args.scenario, qsl=qsl
+            )
+
+        # Initiates and loads loadgen test settings and log path
+        settings = lg.TestSettings()
+        settings.scenario = scenario_map[args.scenario]
+        # mlperf conf is automatically loaded by the loadgen
+        # settings.FromConfig(args.mlperf_conf, "gptj", args.scenario)
+        settings.FromConfig(args.user_conf, "gptj", args.scenario)
+
+        # Chosing test mode Accutacy/Performance
+        if args.accuracy:
+            settings.mode = lg.TestMode.AccuracyOnly
+        else:
+            settings.mode = lg.TestMode.PerformanceOnly
+
+        # Set log path
+        log_path = os.environ.get("LOG_PATH")
+        if not log_path:
+            log_path = "build/logs"
+        if not os.path.exists(log_path):
+            os.makedirs(log_path)
+
+        log_output_settings = lg.LogOutputSettings()
+        log_output_settings.outdir = log_path
+        log_output_settings.copy_summary_to_stdout = True
+        log_settings = lg.LogSettings()
+        log_settings.log_output = log_output_settings
+        log_settings.enable_trace = True
+
+    if args.network != "lon":
+        # Gets SUT.
+        # SUT only initialised when network is either None or is SUT as it is not needed in case of client(LON).
+        # Only the server loads the model.
+        sut = get_SUT(
+            model_path=args.model_path,
+            scenario=args.scenario,
+            dtype=args.dtype,
+            use_gpu=args.gpu,
+            network=args.network,
+            dataset_path=args.dataset_path,
+            max_examples=args.max_examples,
+            qsl=qsl,  # If args.network is None, then only QSL get passed to the SUT, else it will be None
+        )
+
+    if args.network == "lon" and args.scenario == "SingleStream":
+        print(
+            "ERROR: Single stream scenario in Loadgen Over the Network is not supported!"
+        )
+
+    # If network option is LON, QDL is loaded and request is served to SUT
+    # based on the scenario given by user(Offline or SingleStream)
+    elif args.network == "lon":
+        lg.StartTestWithLogSettings(
+            qdl.qdl, qsl.qsl, settings, log_settings, args.audit_conf
+        )
+
+    # If network option is SUT, a flask server is initiated, request is
+    # processed and output is being sent back to LON client
+    elif args.network == "sut":
+        temp_cache = get_temp_cache()
+        from network_SUT import app, node, set_backend, set_semaphore
+        from systemStatus import get_cpu_memory_info, get_gpu_memory_info
+
+        # Calculating free memory inorder to determine the number of instances that can be run at a particular time
+        # lockVar contains the value of number of instances
+        # Formula:
+        #   Number of Instances = (free_memory_of_system - memory_size_taken_by_model)/temp_cache_size
+        # Based on the got value(lockVar) a semaphore is initialised
+        # Acquire and release of semaphore can be found in network_SUT.py
+        model_mem_size = sut.total_mem_size
+        if args.gpu:
+            free_mem = int(
+                os.environ.get(
+                    "CM_CUDA_DEVICE_PROP_GLOBAL_MEMORY", get_gpu_memory_info()
+                )
+            ) / (1024**3)
+        else:
+            free_mem = get_cpu_memory_info()
+        lockVar = math.floor((free_mem - model_mem_size) / temp_cache)
+        node = args.node
+        # Semaphore is set inorder to create multiple instances upon request
+        # incomming
+        set_semaphore(lockVar)
+        print(f"Set the semaphore lock variable to {lockVar}")
+        # Pass SUT as the backend
+        set_backend(sut)
+        app.run(debug=False, port=args.port, host="0.0.0.0")
+
+    else:
+        # Test not run in Loadgen Over the Network
+        print("Running LoadGen test...")
+        lg.StartTestWithLogSettings(
+            sut.sut, qsl.qsl, settings, log_settings, args.audit_conf
+        )
+
+    print("Test Done!")
+
+    if args.network != "lon":
+        print("Destroying SUT...")
+        lg.DestroySUT(sut.sut)
+
+    if args.network != "sut":
+        print("Destroying QSL...")
+        lg.DestroyQSL(qsl.qsl)
+
+
+if __name__ == "__main__":
+    main()

language/gpt-j/network_SUT.py

+from flask import Flask, request, jsonify
+import threading
+
+app = Flask(__name__)
+
+node = ""
+
+model_mem_size = None
+use_gpu = False
+free_mem = None
+
+
+# setting semaphore
+def set_semaphore(lockVar):
+    global semaphore
+    semaphore = threading.Semaphore(lockVar)
+
+
+def set_backend(b):
+    global backend
+    backend = b
+
+
+def preprocess(query):
+    """[SUT Node] A dummy preprocess."""
+    # Here may come for example batching, tokenization, resizing,
+    # normalization, etc.
+    response = query
+    return response
+
+
+def dnn_model(query):
+    # Here may come for example a call to a dnn model such as resnet, bert,
+    # etc.
+    response = backend.inference_call(query)
+    print(f"dnn model response:{response}")
+    return response
+
+
+def postprocess(query):
+    """[SUT Node] A dummy postprocess."""
+    # Here may come for example a postprocessing call, e.g., NMS,
+    # detokenization, etc.
+    response = query
+    return response
+
+
+@app.route("/predict/", methods=["POST"])
+def predict():
+    """Receives a query (e.g., a text) runs inference, and returns a prediction."""
+    semaphore.acquire()  # wait to acquire semaphore
+    try:
+        query = request.get_json(force=True)["query"]
+        result = postprocess(dnn_model(preprocess(query)))
+        print(result)
+        return jsonify(result=result)
+    finally:
+        # semaphore is released after processing
+        semaphore.release()
+
+
+@app.route("/getname/", methods=["POST", "GET"])
+def getname():
+    """Returns the name of the SUT."""
+    return jsonify(name=f"Demo SUT (Network SUT) node" +
+                   (" " + node) if node else "")

language/gpt-j/prepare-calibration.py

+import os
+import sys
+import json
+from argparse import ArgumentParser
+from datasets import load_dataset
+
+
+def get_args():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--calibration-list-file", required=True, help="Path to calibration list"
+    )
+    parser.add_argument(
+        "--output-dir", help="Output directory", default="calibration-data"
+    )
+
+    return parser.parse_args()
+
+
+dataset_id = "cnn_dailymail"
+version = "3.0.0"
+split = "train"
+
+instruction_template = "Summarize the following news article:"
+
+
+def check_path(path):
+    return os.path.exists(path)
+
+
+def prepare_calibration_data(calibration_list_file, output_dir):
+    if not check_path(calibration_list_file):
+        print("Calibration list file not found: {}".format(calibration_list_file))
+        sys.exit(1)
+
+    dataset = load_dataset("cnn_dailymail", name="3.0.0", split="train")
+    train = dict((x["id"], x) for x in dataset)
+
+    with open(calibration_list_file, "r") as fid:
+        calibration_ids = fid.read().splitlines()
+
+    inputs = []
+    for id in calibration_ids:
+        calibration_sample = train[id]
+        x = dict()
+        x["instruction"] = instruction_template
+        x["input"] = calibration_sample["article"]
+        x["output"] = calibration_sample["highlights"]
+        inputs.append(x)
+
+    if not os.path.isdir(output_dir):
+        os.makedirs(output_dir)
+
+    output_path = os.path.join(output_dir, "cnn_dailymail_calibration.json")
+    with open(output_path, "w") as write_f:
+        json.dump(inputs, write_f, indent=4, ensure_ascii=False)
+
+    print("Calibration data saved at {}".format(output_path))
+
+
+def main():
+
+    args = get_args()
+    prepare_calibration_data(args.calibration_list_file, args.output_dir)
+
+
+if __name__ == "__main__":
+    main()

language/gpt-j/requirements.txt

+transformers
+datasets
+evaluate
+accelerate
+simplejson
+nltk
+rouge_score
\ No newline at end of file