修改README，增加训练脚本，完善模型转换代码

PyTorch/NLP/BERT/tf_to_torch/cleanup_scripts/create_wiki_test_set_md5_hashes.py

+# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Creates a text file containing md5sums from test set text file, to be used for verification of test set
+
+import hashlib
+
+filename = 'wiki_test_set.txt'
+ofilename = 'wiki_test_set_md5.txt'
+
+with open(filename, mode='r', newline='\n') as ifile:
+  lines = ifile.read()
+  articles_in_file_tmp = lines.split('\n\n')
+  articles_in_file = []
+  for item in articles_in_file_tmp:
+    if item.rstrip() != '':
+      articles_in_file.append(item)
+
+with open(ofilename, mode='w', newline='\n') as ofile:
+  for item in articles_in_file:
+    ofile.write(hashlib.md5(item.encode('utf-8')).hexdigest())
+    ofile.write('\n')

PyTorch/NLP/BERT/tf_to_torch/cleanup_scripts/do_gather.py

+# Copyright 2018 MLBenchmark Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Script to package BERT dataset into files with approcimate size.
+
+Copied and modified from https://github.com/eric-haibin-lin/text-proc.git
+"""
+import argparse
+import glob
+import io
+import logging
+import multiprocessing
+import os
+import time
+
+parser = argparse.ArgumentParser(description='BERT data packaging')
+parser.add_argument(
+    '--data',
+    type=str,
+    default='~/book-corpus-feb-stn/*/*.txt',
+    help='Input files. Default is "*.txt"')
+parser.add_argument(
+    '--nworker',
+    type=int,
+    default=1,
+    help='Number of workers for parallel processing.')
+parser.add_argument(
+    '--out_dir',
+    type=str,
+    default='~/book-corpus-large-gather/',
+    help='Output dir. Default is ~/book-corpus-large-gather/')
+parser.add_argument(
+    '--num_outputs', type=int, default=500, help='number of output files')
+parser.add_argument(
+    '--input_suffix', type=str, default='.3', help='Suffix for input filenames')
+parser.add_argument(
+    '--block_size',
+    type=float,
+    default=32.0,
+    help='Block size for each output (MB)')
+
+args = parser.parse_args()
+
+input_files = sorted(glob.glob(os.path.expanduser(args.data)))
+out_dir = os.path.expanduser(args.out_dir)
+num_files = len(input_files)
+num_workers = args.nworker
+logging.basicConfig(level=logging.INFO)
+logging.info('Number of input files to process = %d', num_files)
+
+if not os.path.exists(out_dir):
+  os.makedirs(out_dir)
+
+
+def worker_fn(x):
+  """Workload for one worker."""
+  file_split, worker_id = x
+  count = 0
+  out_file = None
+  total_size = 0
+  for in_path in file_split:
+    in_file = io.open(in_path + args.input_suffix, 'r', encoding='utf-8-sig')
+    curr_size = os.path.getsize(in_path)
+    if args.block_size * 1024 * 1024 < total_size + curr_size:
+      out_file.close()
+      out_file = None
+      count += 1
+      total_size = 0
+    if not out_file:
+      out_path = os.path.join(
+          out_dir, 'part-{}-of-{}'.format(
+              str(count + 1000 * worker_id).zfill(5),
+              str(args.num_outputs).zfill(5)))
+      out_file = io.open(out_path, 'w', encoding='utf-8')
+    total_size += curr_size
+    content = in_file.read()
+    if content[-1] == content[-2] and content[-1] == '\n':
+      content = content[:-1]
+    out_file.write(content)
+
+
+if __name__ == '__main__':
+  p = multiprocessing.Pool(num_workers)
+
+  # calculate the number of splits
+  file_splits = []
+  split_size = (len(input_files) + num_workers - 1) // num_workers
+  for i in range(num_workers - 1):
+    file_splits.append((input_files[i * split_size:(i + 1) * split_size], i))
+  file_splits.append(
+      (input_files[(num_workers - 1) * split_size:], num_workers - 1))
+
+  tic = time.time()
+  p.map(worker_fn, file_splits)
+  toc = time.time()
+  logging.info('Processed %s in %.2f sec', args.data, toc - tic)

PyTorch/NLP/BERT/tf_to_torch/cleanup_scripts/do_sentence_segmentation.py

+# Copyright 2018 MLBenchmark Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Script for sentence segmentation.
+
+Copied and modified from https://github.com/eric-haibin-lin/text-proc.git
+"""
+import argparse
+import glob
+import io
+import logging
+import multiprocessing
+import os
+import time
+import nltk
+
+from nltk.tokenize import sent_tokenize
+
+parser = argparse.ArgumentParser(
+    description='Sentence segmentation for BERT documents.')
+parser.add_argument(
+    '--data',
+    type=str,
+    default='./*/*.compact',
+    help='Input files. Default is "./*/*.compact"')
+parser.add_argument(
+    '--input_suffix',
+    type=str,
+    default='.2',
+    help='Suffix for input files. Default is ".2"')
+parser.add_argument(
+    '--output_suffix',
+    type=str,
+    default='.3',
+    help='Suffix for output files. Default is ".3"')
+parser.add_argument(
+    '--nworker',
+    type=int,
+    default=72,
+    help='Number of workers for parallel processing.')
+args = parser.parse_args()
+
+# download package
+nltk.download('punkt')
+
+# arguments
+input_files = sorted(glob.glob(os.path.expanduser(args.data)))
+num_files = len(input_files)
+num_workers = args.nworker
+logging.basicConfig(level=logging.INFO)
+logging.info('Number of input files to process = %d', num_files)
+
+
+def process_one_file(one_input):
+  """Separate paragraphs into sentences, for one file."""
+  input_filename = one_input + args.input_suffix
+  output_filename = one_input + args.output_suffix
+  logging.info('Processing %s => %s', input_filename, output_filename)
+  with io.open(input_filename, 'r', encoding='utf-8') as fin:
+    with io.open(output_filename, 'w', encoding='utf-8') as fout:
+      for line in fin:
+        if len(line) == 1:
+          fout.write(u'\n')
+        sents = sent_tokenize(line)
+        for sent in sents:
+          sent_str = sent.strip()
+          # if sent_str:
+          fout.write('%s\n' % sent_str)
+      fout.write(u'\n')
+
+
+if __name__ == '__main__':
+  tic = time.time()
+  p = multiprocessing.Pool(num_workers)
+  p.map(process_one_file, input_files)
+  toc = time.time()
+  logging.info('Processed %s in %.2f sec', args.data, toc - tic)

PyTorch/NLP/BERT/tf_to_torch/cleanup_scripts/extract_test_set_articles.py

+# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import hashlib
+
+output_filename = 'wiki_test_set.txt'
+hash_file = 'wiki_test_set_md5.txt'
+
+test_articles = []
+test_hashes = []
+
+found_hashes = []
+
+file_glob = glob.glob('./results/part*', recursive=False)
+
+# Load md5sums into memory
+with open(hash_file, mode='r', newline='\n') as hfile:
+  for line in hfile:
+    test_hashes.append(line.rstrip())
+
+assert len(test_hashes) == 500, 'Incorrect number of test set articles.'
+
+with open(output_filename, mode='w', newline='\n') as ofile:
+  for filename in file_glob:
+    articles_in_file = []
+    target_article_idx = None
+    with open(filename, mode='r', newline='\n') as ifile:
+      print("file opened:", filename)
+      lines = ifile.read()
+      articles_in_file_tmp = lines.split('\n\n')
+      articles_in_file = []
+      idx = 0
+      for item in articles_in_file_tmp:
+        if item.rstrip() != '':
+          articles_in_file.append(item)
+          if hashlib.md5(item.rstrip().encode('utf-8')).hexdigest() in test_hashes:
+            print("article found at", idx)
+            target_article_idx = idx
+            found_hashes.append(hashlib.md5(item.rstrip().encode('utf-8')).hexdigest())
+            test_articles.append(item.rstrip())
+          idx += 1
+    
+    if not target_article_idx:
+      print('article not found, continuing')
+      continue
+
+    with open(filename, mode='w', newline='\n') as ifile:
+      for article in articles_in_file[:target_article_idx]:
+        ifile.write(article)
+        ifile.write('\n\n')
+
+      for article in articles_in_file[target_article_idx+1:]:
+        ifile.write(article)
+        ifile.write('\n\n')
+
+  if len(test_articles) != 500:
+    print("Entering missing article debug section.", len(found_hashes), "hashes found.")
+    missing_articles = []
+    for idx, expected_hash in enumerate(test_hashes):
+      if expected_hash not in found_hashes:
+        missing_articles.append(idx)
+        print("Missing article, reference idx:", idx)
+  
+  print(len(test_articles), "articles out of 500 found.")
+
+  assert len(test_articles) == 500, 'Not all articles were found in shards. Incomplete test set.'
+
+  for expected_hash in test_hashes:
+    idx = found_hashes.index(expected_hash)
+    ofile.write(test_articles[idx])
+    ofile.write('\n\n')
+
+print("n_articles =", len(test_articles))

PyTorch/NLP/BERT/tf_to_torch/cleanup_scripts/parallel_create_hdf5.sh

+#!/bin/bash
+# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+n_shards=${1}
+data_part=${2}
+
+cpus=$( ls -d /sys/devices/system/cpu/cpu[[:digit:]]* | wc -w )
+cpus=$((cpus / 2))
+echo "Using $cpus CPU cores"
+
+find text_shards/${n_shards}_shards/${data_part} -name "part*" | xargs --max-args=1 --max-procs=$cpus  ./create_pretraining_data_wrapper.sh

PyTorch/NLP/BERT/tf_to_torch/cleanup_scripts/process_wiki.sh

+#!/bin/bash
+# Copyright 2018 MLBenchmark Group. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# invocation script to cleanup the wiki dataset
+# Usage: ./process_wiki.sh <the wiki_?? files>
+# example: ./process_wiki.sh 'sample_data/wiki_??'
+# The resulted files will be placed in ./results
+
+inputs=$1
+
+pip install nltk
+
+# Remove doc tag and title
+python ./cleanup_file.py --data=$inputs --output_suffix='.1'
+
+# Further clean up files
+for f in ${inputs}; do
+  ./clean.sh ${f}.1 ${f}.2
+done
+
+# Sentence segmentation
+python ./do_sentence_segmentation.py --data=$inputs --input_suffix='.2' --output_suffix='.3'
+
+mkdir -p ./results
+
+## Choose file size method or number of packages by uncommenting only one of the following do_gather options
+# Gather into fixed size packages
+python ./do_gather.py --data=$inputs --input_suffix='.3' --block_size=26.92 --out_dir='./results'

PyTorch/NLP/BERT/tf_to_torch/cleanup_scripts/reshard_hdf5_files.py

+# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import h5py
+import multiprocessing
+import numpy as np
+
+hdf5_compression_method = None
+
+input_path = 'hdf5'
+input_files = glob.glob(input_path + '/part*', recursive=False)
+
+print('n_input_shards =', len(input_files))
+
+max_pred_per_seq = 76
+seq_length = 512
+n_output_shards = 1472
+n_processes = 20*2
+
+print('n_output_shards =', n_output_shards)
+
+# Allocate input storage
+f_input_ids = []
+f_input_masks = []
+f_segment_ids = []
+f_masked_lm_positions = []
+f_masked_lm_ids = []
+f_next_sentence_labels = []
+
+# Load all samples into memory (can be done in batches if necessary in future)
+def loader(idx):
+  ifile = input_files[idx]
+  print("Opening:", ifile, " --  Progress:", idx+1, '/', len(input_files))
+  h5_ifile = h5py.File(ifile, 'r')
+
+  f_input_ids = h5_ifile['input_ids'][:].astype(np.int64)
+  f_input_masks = h5_ifile['input_mask'][:].astype(np.int64)
+  f_segment_ids = h5_ifile['segment_ids'][:].astype(np.int64)
+  f_masked_lm_positions = h5_ifile['masked_lm_positions'][:].astype(np.int64)
+  f_masked_lm_ids = h5_ifile['masked_lm_ids'][:].astype(np.int64)
+  f_next_sentence_labels = h5_ifile['next_sentence_labels'][:].astype(np.int64)
+
+  h5_ifile.close()
+  
+  return f_input_ids, f_input_masks, f_segment_ids, f_masked_lm_positions, f_masked_lm_ids, f_next_sentence_labels
+
+for idx, ifile in enumerate(input_files):
+  print("Opening:", ifile, " --  Progress:", idx+1, '/', len(input_files))
+  h5_ifile = h5py.File(ifile, 'r')
+
+  f_input_ids.append(h5_ifile['input_ids'][:])
+  f_input_masks.append(h5_ifile['input_mask'][:])
+  f_segment_ids.append(h5_ifile['segment_ids'][:])
+  f_masked_lm_positions.append(h5_ifile['masked_lm_positions'][:])
+  f_masked_lm_ids.append(h5_ifile['masked_lm_ids'][:])
+  f_next_sentence_labels.append(h5_ifile['next_sentence_labels'][:])
+
+  h5_ifile.close()
+ 
+# Calculate index offsets (to access into list of np.ndarray's)
+n_samples = 0
+f_offsets = [0]
+
+print("len(f_input_ids) =", len(f_input_ids))
+
+for idx in range(len(f_input_ids) - 1):
+  f_offsets.append(f_input_ids[idx].shape[0] + f_offsets[-1])
+  n_samples += f_input_ids[idx].shape[0]
+
+n_samples += f_input_ids[-1].shape[0]
+
+f_offsets = np.array(f_offsets)
+
+print("n_samples =", n_samples)
+
+# Create random permutation
+rand_perm = np.random.permutation(n_samples)
+
+def f_retrieve(global_idx, f_ndarray):
+  idx = np.abs(f_offsets - global_idx).argmin()
+  if f_offsets[idx] > global_idx:
+    idx -= 1
+  
+  local_idx = global_idx - f_offsets[idx]
+  perm_idx = rand_perm[global_idx]
+
+  pidx = np.abs(f_offsets - perm_idx).argmin()
+  if f_offsets[pidx] > perm_idx:
+      pidx -= 1
+  
+  return f_ndarray[idx][pidx]
+
+
+# Find a "nominal" number of samples per shard (calculated to always go over by one shard size)
+# Find excess samples in last shard and distribute removal of excess over first "N" shards (could be done over last, but it doesn't matter and math is easier this way)
+#  (since 0 <= excess < nominal_shard_size, the max imbalance will be 1 sample to minimize the straggler effect)
+n_sample_per_ofile_nominal = (n_samples + n_output_shards - 1) // n_output_shards
+n_excess = n_output_shards * n_sample_per_ofile_nominal - n_samples  # Always a positive number
+
+
+# Start writing out sequentially from random permutation (mapping -> rand input to seq. output to favor disk I/O)
+# The expected program execution time will be time(sequential_read) + time(sequential_writes) + negligibile_permutation_cpu_time
+# Space requirement is approx. equal to size_of_input_bytes + size_of_permutation_array_bytes (~130GB + ~10MB)
+# This is much better than the 30 hours or so required by the previous sharding script and the >400GB memory footprint (expected input size ~130GB)
+ofile_prefix = '1472_balanced/part_'
+ofile_suffix = '_of_' + str(n_output_shards) + '.hdf5'
+
+global_index_offset = 0
+for i in range(n_output_shards):
+  ofile = ofile_prefix + str(i) + ofile_suffix
+  
+  n_samples_in_this_shard = n_sample_per_ofile_nominal
+  if i < n_excess:
+    n_samples_in_this_shard -= 1
+    
+  print("shard index:", i, ", n_samples_in_this_shard =", n_samples_in_this_shard)
+  
+  with h5py.File(ofile, 'w') as f:
+    o_input_ids = np.ndarray((n_samples_in_this_shard, seq_length))
+    o_input_masks = np.ndarray((n_samples_in_this_shard, seq_length))
+    o_segment_ids = np.ndarray((n_samples_in_this_shard, seq_length))
+    o_masked_lm_positions = np.ndarray((n_samples_in_this_shard, max_pred_per_seq))
+    o_masked_lm_ids = np.ndarray((n_samples_in_this_shard, max_pred_per_seq))
+    o_next_sentence_labels = np.ndarray((n_samples_in_this_shard))
+      
+    for local_index in range(n_samples_in_this_shard):
+      o_input_ids[local_index] = f_retrieve(global_index_offset + local_index, f_input_ids)
+      o_input_masks[local_index] = f_retrieve(global_index_offset + local_index, f_input_masks) 
+      o_segment_ids[local_index] = f_retrieve(global_index_offset + local_index, f_segment_ids)
+      o_masked_lm_positions[local_index] = f_retrieve(global_index_offset + local_index, f_masked_lm_positions)
+      o_masked_lm_ids[local_index] = f_retrieve(global_index_offset + local_index, f_masked_lm_ids)
+      o_next_sentence_labels[local_index] = f_retrieve(global_index_offset + local_index, f_next_sentence_labels)
+      
+    f.create_dataset("input_ids",            data=o_input_ids,            dtype='i4', compression=hdf5_compression_method)
+    f.create_dataset("input_mask",           data=o_input_masks,          dtype='i1', compression=hdf5_compression_method)
+    f.create_dataset("segment_ids",          data=o_segment_ids,          dtype='i1', compression=hdf5_compression_method)
+    f.create_dataset("masked_lm_positions",  data=o_masked_lm_positions,  dtype='i4', compression=hdf5_compression_method)
+    f.create_dataset("masked_lm_ids",        data=o_masked_lm_ids,        dtype='i4', compression=hdf5_compression_method)
+    f.create_dataset("next_sentence_labels", data=o_next_sentence_labels, dtype='i1', compression=hdf5_compression_method)
+      
+    global_index_offset += n_samples_in_this_shard

PyTorch/NLP/BERT/tf_to_torch/cleanup_scripts/transparency_in_test_set_generation.py

+# Copyright (c) 2019 NVIDIA CORPORATION. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+
+output_filename = 'wiki_test_set.txt'
+
+test_articles = []
+
+file_glob = glob.glob('./results/part*', recursive=False)
+
+with open(output_filename, mode='w', newline='\n') as ofile:
+  for filename in file_glob:
+    articles_in_file = []
+    with open(filename, mode='r', newline='\n') as ifile:
+      lines = ifile.read()
+      articles_in_file_tmp = lines.split('\n\n')
+      articles_in_file = []
+      for item in articles_in_file_tmp:
+        if item.rstrip() != '':
+          articles_in_file.append(item)
+    
+    target_article = min(42, len(articles_in_file) // 2)
+    test_articles.append(articles_in_file[target_article])
+
+    with open(filename, mode='w', newline='\n') as ifile:
+      for article in articles_in_file[:target_article]:
+        ifile.write(article)
+        ifile.write('\n\n')
+
+      for article in articles_in_file[target_article+1:]:
+        ifile.write(article)
+        ifile.write('\n\n')
+
+  for article in test_articles:
+    ofile.write(article)
+    ofile.write('\n\n')
+
+print("n_articles =", len(test_articles))

PyTorch/NLP/BERT/tf_to_torch/cleanup_scripts/wiki_test_set_md5.txt

+6fdd6955cc8870f3759ca33f2d3ec341
+3dfa8393f0f206ae46d0c6133ed2cb77
+a60e4942e99d034d8894b86140f63dbf
+559723e57d90806b80daf0f958e82ba7
+bc8f54aa290feb586d0d19aff74e6c87
+c65f0d516559ce89fb171ac162fd6e93
+ebad9baa817905df3fbae93b3f6af760
+e9a5f1fd447f6f7e6ba5b7cd17329796
+64aea4bae1a318d9b1377820a8ad08a7
+b2f11f1fed3009b8544ffc5d49b880d5
+81541935a9987fdd31f7a4b83ee63417
+a9a96829795abd8969500ce0293ed9da
+20d52bdb6f18b4ef7f004a9e9be5efd4
+8a6daa68cfbadcc023cd4534ec1bdec7
+fdfe086f078de5868d31abb79eb24b93
+7a73fa121b9eaee3c5b193461eece00f
+e990db6fa1f5b1c4d3bd16e0bd7f999b
+ccc8a6e2849defd36994b3f03ff6ca71
+ead01c76770d0cc1b093822e9d3fb5c4
+2082bfddbe6bc76c3727765bd28a20d3
+95ded24040b74dfbb9c11959316262a8
+8e8e65398857f7e5b4a392cfeced06c1
+7b49c71e8ff40f4db286e50b0d789491
+0185c0aca5aad797d6a976bf86cef8d8
+824b0d78d2c91a9a2211ed34c0bfec3b
+9bf4027b4b73ef5c8590283861c02fb8
+2f824ed143938e481b1b59086613e8ab
+0cb7945bdc930bf78b433fed2f1ed56a
+c65feab948291b1f3858f0b2015353ea
+d5b85ef2cbb95289a24c47d1437bd2a2
+439c939b99c722a2e641d05ebcca7f45
+76d23451c9eed70c02d044a51da67489
+391f38878a3f3ccd143ef2189599e93e
+09a800551067450a33a54102f41230ce
+2879e77fa0dd5d96f03f182405406da4
+d8c2d52db0e933f2b613d40f101f3359
+b308402dc54cae0ad754c2c2cc716d32
+2bb453a19e7ace5d5e06631508572dcc
+40e294861b2c9b59fa3a08f130bac358
+766971e091d87a9684ec294bf3861010
+1fe31747e060f7e8502071f79e61e390
+118938eac97fb8cd2359385137e70aeb
+c00a712a9ff782853f80da308aeb5a9f
+3e263d4059dd7690435a4a982b9ac03d
+9a3908e136c9c06369a56be3c919d162
+28b7f58bd0c51fb33061a80549809a30
+4c7af2f0efaa8020c2af72e1b852c45f
+840276381de485d9ae46e3fe01598512
+cdaf7a68c216a9b8ddb56b2a25155b16
+4db5173a746056d10bdbad607be573e5
+58444316e4b7dca214ee9a97886a60c4
+c44d302a256f16a0ca36741dd019e8b6
+8170c51042314fba00fb38acd3e41425
+fedcd0f67c7bbdfd83fe1702e9555ee4
+c51caa193f071cc5205a44af10563f91
+fbb573806da72e876bc492f3692164bc
+8503af5b447b0b1ac1337108c8a51184
+3a9245ff5e691d70e76719e59dd78145
+716bef52a03624c62ce008880401d2bd
+e07a1029a57240c8f990b5cb94e895d8
+e70a2302308f6cb92174ea0a57c2b394
+0c569e51a50fccfac8c79493bf2808f4
+dff4cbeb4a0ab78d38bafdffb698773e
+a13f14ca93de1d60ce458ff0a8c68a5e
+6ff95b9270af7c1933f2f9f8914dd6fd
+36d98305846346fca383985bb9b38d05
+e8c90f97d5ff07ec471664a2b8640118
+9f303e0d6762a9f234a6c3dc1739d17c
+7d904d2a149cd2b1edd6d95604961efc
+14ab1e0b332a58d38dfd534064e2302e
+ccc4222ce36c2995276cd4787483a9a5
+1021251bcaa276200f5c2492cd7c85cb
+12ebcc5932dc5f0148564a899a7a0485
+d3c97914397f103ff1e028223eb3fead
+514c56c90384b49678bd6a6e64491383
+529c8346ec57479a8344b0d8b18bef44
+fb719c3a18ecb63d4005a118ac0c9d43
+5a366cc44d64ccd1c1e582073520f1dc
+ed9d89903bf0441d78ba507a979fe225
+c35f1f1b9f3c89f071409bf38bd679ff
+b0f2a4345c27366a9ea3d8688404fff3
+4728a279ab7723b656a2f8832092fb48
+15ca01d162fc21d33fe1366bc3e6e382
+d0bb7e46bde5230ea6085e748d2eab75
+dfa7be7849fa8f895210cf1cc0d4aef8
+621e298e8eaaea1efb4ccfea6e74db30
+30913dbab44443bf2dcb16994d61f756
+e267d33f86d3f71eddbc5ce6b694940b
+31e0b3988b280e10b84349942bc6b73c
+6921e4754ff7e03568bc35646027dd91
+a00ed7dd4c077cf766f643d036c375a0
+0f99bb8d2ccda730bc52f9bd236d6da8
+96fcd9446e6908f319187eb8cb2c0221
+6eea43218c05bba16f7fcc4c29217a2a
+06696d92ee0d72f1fe0de0e57d0acfbb
+87e080ae9e760d47256b49d993b5a156
+6a6f27375020a965dbce70f20ba66400
+50c5e0807b4496f1097df681db1db13c
+8db3d283559d1967132db098a070c7ae
+2aafc07a25214cc81454c76012aa6f71
+71464c311dba3de441dddad13b3e1b5c
+9dc2e3a17fe0145269ef3b53f6636000
+9f8e97bdb1bd93435fb36a9f91397f7e
+84d9aa1360aa1ee3e5ad8f1dc8601fd3
+f7b2b8213414bed3f782091c78e23458
+f3d9943caa7e88b74f62a57c6b7a99b5
+c5b681acfa39cddb621780d612b6f04d
+cfb7764aacda35032dc3035c68768d9d
+d0fcd40ba3b6ee750e81bb0d8d26ee0e
+9915b311c6e60cc803c82e7a1396ef72
+fe976d569b21f8a274eb19d34fdf6abf
+930743c57dd08f4c5358d387aea3a9c4
+3e47ef4a7b2e4d567eb7f003bc889345
+388c382201e24cb00cd70238f3aa320c
+85c9d809b8cd23557e51fd8600ab12b3
+40fa8113d19a0a5aaad6a3cbb231f985
+a2bcd0770b21bccb19ecd038af397127
+13b3e62f64adee0c3b56d4a18822dc86
+e9ae8b54a6e2293ce5b900eb5237aa8f
+5a655eb1844280aa251b431ea95dc9d9
+286e3301805776fdbaabe66a27d6f31d
+46ad12a5a321831d96ea6938336c1af4
+00e1435bc160e1f3fe5551b573d0d4f6
+bc08dccc046933a84036e8bffe76caf1
+bc304b4cab22f18f69db726bd8ebb957
+db4dc4c0dc5f5d77fc824035953580a7
+3841c69cc9222d641b122f2813f626b4
+3639713d8d3233260de05dbf9ab35e0f
+f42431deb4c6d4442fe8bc9baf881721
+94e8454b59a54c34066ab7fe6edbaa77
+5d395f49810523c7b385dca4237c8639
+e6e32b94a81875dfaf9d76dbe9caabe2
+4c035f29d3eff2640c4dc686a570a8a6
+ec36030d66567d9290901d6a351739bc
+2a575f0b6f2b9a2f27b44d6c15a1b9a8
+f3a2e0cdbb0f9c0588c9a40d2350cd32
+6d80c2b2df52371d4f2cf78edbe48c1b
+08bce1556ee7b4ae2a0ad073408c029f
+3022941b88cbdf4d4919d5f2104f733b
+2cdcff72716a3df47b1f99c8fe9855c9
+0270e1e9e5a3bf9834b8081205c09e83
+cccca8f97d29041180ad41d0babca69e
+c20a71242793c23f8c411beb04cece9c
+e076dba3436472fec1a37b6bc64e6937
+10a800735554b66e47b4d92e090500ae
+3abde50ae3190f503d66e04471a6f7f9
+2d80390ab4ad8f60c4bda35f332a95b0
+ae280a6944181dc6e36b7810380d8d53
+12fe7c869965131574b08647b4eb18a6
+2748b34fb98aa02ee0a466f4bb345dde
+99854d9b81abe0309c01ea378e2e494b
+47b5033b3d94307379fd84447a0bc9b3
+5c6b045edb18edf721fc4970dbf0a8a8
+bbd2f05e6c1a6ee49e509cc0599557f0
+96a97ea5c594ffc07934d73726869e09
+470a7d5a509d40ee94aeab3ccb7a1b9f
+8d317899d5a338f953a71ce3ba533720
+7bb5eaaa662fb044f26f2ee98032fdba
+50eb0e9a12fdcf160cc1b34c9444fc8a
+565c133f5c2ebe7ef4bf8da520d0b383
+acd42e501e248184189c75e39d46e93c
+93ec505a151708f1153aac247c643460
+d38a2e336ff378909bac2e6c33c7d6ed
+0e0f7d46009701f7a11d6bb563a5c64f
+72b47a203ed91d4be197a80c830ea943
+a1ec56db71dac654bb41d21369d31ed8
+ccb22a0252a2d7e645e82a2de00bd5de
+b082904731b10e11e48d9ad1898d7a19
+1bf9792e64206c1f8de378d3f6fb9441
+11821796dbee85d486af5ddb3c4e36bf
+d3864c99cd0861883982825b6a24d06b
+43643db013a6a742fd44dc3f3b50ce48
+9d9020684b39102a272f821842bdfc53
+408a1f4f1aca905a4c4ca3ba259971c0
+9b9af46e2f1aac9723ac0633c89fc9e9
+9375ae7534aaf2e0fa03fb5fabd36e95
+12fae3437faf03f35924171fec19444c
+228119c228d88e874d7cb532ccd41b4e
+a5f38f4924fd04f539b9ba9e7ddcd8db
+3800f0ef981b20fd8e0c680f1792ade5
+6b282a2febc7aa4b854a52d354f84298
+bddcea5b9b979cb30612eace9425aa8e
+0caec4c292e323a1ad4f00efa5af8ef9
+412b313da76ad3b69f162b37ffee3c8c
+e169d983ddc88f197fdb2f2fc01bc5fd
+afad5340e2eb1b855aeaac1dccdfa87b
+dded03dd1c7d5ea765ea167066beded2
+4f109d8fca2574043a12749902b2a32b
+157a529afe0b30b053df507e08967da3
+9f6adf4e7d39691360c1dee8343c474d
+026ac04d80a4026708faaac4ba2656ae
+9339c82715a7db5993fbdc4c9b32352b
+0b35688ed7f25491063c0cb15604ca72
+a9b27e86934c5d9145cb8102417d27d0
+c91b729c5cd8b2af8b7e5550b6a40736
+9cbb6d61f23cf100b80dcd96b7c21ca2
+dcc225d228ee826085e00da81cc1d7ad
+975820ececda8fde06bdd1b3d8f840bc
+be717909b320d3ec5856113fa6b5c0ca
+7b031cc2091711e26846a89707a8d6d3
+28ee5e7d680e2acf2fdd849a133c08b0
+a626cbab988bd1ee00df4a27a5c6abbd
+0164633aaa832e157fa184e9b44d8869
+35ad24475c09bd1fcf6f7085c09b8d2a
+f2599893199bb21b82628da5dfc73f42
+cdbb3fa6e5975f7d0437a674fd4d5a5a
+16efe5db25538a83c1dd9285adc5c832
+ea4478c9fbefbe4ce0a6b768ab13a8a8
+7f41f538dcb147068a4b4af4e6428e14
+78f77023d9b12fce717a8730dc1d255c
+9eff58bdf4ea153c3cb8320e664d26a6
+881eba9434a50d78f4dff3792cdc7109
+6184e6a65b3c9b5aae861707ceb2e6b3
+5a3e9534d1a922130f4a81e4c9e12ef9
+f26c21c36566913bd1163e2ca4c65edf
+a5bc220cf54f11a05b7e29a3f1559a67
+c482da8a997097f2511972c1b06b191a
+00b2535dbc4027ea6e8d77c87b8a79f1
+f9f036028c1bb8168d1d81636a44d08d
+1bd22e4e8d17dd1d46e9a0bb906c17a9
+7ca745385e174a84535591b6fd8a299a
+4e69ac846165c75c5bd79d576eb748b5
+f37e2f540bf85b325479e92861b2060f
+99ebd53fa2efac4a8c25a22dda86024b
+14f0a5769a9ab2f4343dc6986648a078
+2f5c82d734f8043fb7973f832f35a52b
+53f09829e135baed5b9fc5f263cf3e26
+8532d668ddd7dbb97bb632291a40eec9
+1aa6ae22b32fbbe27dc1c7b86312476d
+f1af49dd16f3e5e8417231c464e4c0f8
+eb28131afe674e915d12d203d500b013
+83ce4897ef829bad25226d790ba8748a
+10065b05e2bd75ae3a86df67def9737d
+0569a17859c118c4580b755acb96e369
+e91bef949a51f161f805257af8c4af2a
+1bb4218976f9b24208f2b1c2971b12a6
+1090179d3c73386bce9d074821adb00d
+be726fdf9be6d92b3c4a387d09c785ff
+fccf0d590254333b8a73926a53f14512
+5b0aea0cb1fd0dda1faece08ba3d0c75
+5e427be2f880f2252fec8e618d7c258c
+6004934e0ece2c4f9b67551cf8e70ac1
+8ab35b6efc21bd6ecffcd664add3e86c
+e5f3f8134bc2a220c16f9b6039256679
+ecfc8c90f7bf6146b0ef06c167d43826
+3f2751e8c9d42d8a5f2c165add02aa33
+c8c5ba576b59358fb57458d875e9d6f3
+0406f83629b7184d30dc615a4047b7b1
+a9d13c992aa677a6033077cc77e976e4
+42bf0811de4708e9efbaca3178146cad
+63ca73c9075777e86347321690ee873a
+a9eeb71517fc5ae39e692ca745cf9b40
+d74d1f6b518c874f604ca48018dfa94d
+f9534fd5ed4416844a607c8c361ec941
+cf3eaf5abd4fd0c1c8a990aea70a864a
+615c8acefdc03d1a2f97de6281f7d4df
+c690a4fb998e5f08a083e70080f48bf5
+eec5c8850ba1a1c06383abdaa7e604af
+fdb0bd1ccdb70f4dc05365adb64b796e
+c5db3caaae50a53fb11fc6392aa27008
+d4322d92fb9cf134f921e0c87b59532e
+e276f8d589db27c8cc1a4f2e19facd62
+13a83e57f5d520884ee270a3aa0ceae3
+ccebb66e38da56ec96cd0485ef0ffe10
+904ea3d1a8c49aa713548ff91d85b765
+a687301c5c0d7d3a593943983f48b5d4
+c3dad8f45dbd69616c17b99135a2f4f8
+e6b5f86624946d0fbf29e7841a9dcdfb
+d2bdd8f762c0c77085718f2acd7abb70
+de036bc0329283dce5026bf50eef751a
+26b3d4aad34e06a03465052bb8ebb3f7
+14ffab94008a762d98c9c76fbf8fa027
+93d3a1090391f041a16415e26b5ed329
+f6b762de3d5d038b36584adc8cf5c65f
+a51c016a79225be223539c2aae1febde
+fdd8ec3c940316ef3b9e013095c95ff9
+dbaaec1305c5accfaa9b32531d73f140
+ca44e453af31d98be2e771e2f79a9c28
+3fe975d671c158b0c324d563aac62bdb
+13b6cf5885a8973cd0da2b681340641d
+cd06827ff67d8d8aafad37e7a260ae87
+1c05e2bdf6c79b09f2b120da9a2f9d6b
+149dc5463e94295fdb10be3ac6318e44
+c38776c3368d0c9725f5e5ffdde3d059
+84aa7c6e980771445dbe72a8d8adf3b6
+7f531a48cab40e956c4e6473ff9bee34
+964ae32151d97f0beb7a02cece43d28e
+34cd680625d07930ed56b7d50b1f8264
+ff839138b2248ebd5d04e066e21fafdd
+8749b5247509490c003e64abd13af0f2
+89e82878fec6f362c0eb1774b19907b1
+48482ee11c9307ddfdf11f2446034d3a
+a0ae4f19124ff13edce269ec4284784d
+acb35d06a1c7ef0aa1917d620f39fd6b
+b8bfd4b29918bf8e97745824ba6ee539
+91f5c9d467277e811b07654b8610edf1
+ce0e0b2359f6c8e863edfa9f66e5c684
+18d928b2786ea06bbe1541f8e14600c6
+c0f6082f728cc806364216226469716c
+c078d55886f6358bc02dad3b1808ba75
+9c759aa7aeb39ff005b6891d19157191
+5856ca59f0c843ef7e2ae51c9b3929bd
+222f7f765ec5f4878484bbbbb225e5e4
+aa452dd7e37069b045737296a7b153ca
+4908b7a763ffe65becd643984b4b1559
+12f841f723965dced73c241206b4eb51
+c211a42b53714433331bc25c1132616b
+f365575c7f304ccd628a5522547f3187
+84bdb2f10263a0a8deecc49dac15edb9
+f97cc86771f369f4d786010fe25a011d
+9a327e0884dd2a784ceb02edf9054be6
+f404b3154f7df36745e2da025a782fe8
+a173764788263b4cf6be26aa1d7be2eb
+24c762eed3d4f045b36e3d61a2d38c65
+d2004b91375dfb05916af5e52b69dc4c
+c054d19e2d6de0f08d23021cb07789c4
+14878968ee402024226ddf6349d49572
+47cb200e3fb12e5cb7090e20dee96542
+798648a37568e135758e133d0f44bbbb
+a5db574ac7840ea1c751fc47cd163732
+f12467ec777a45190f86f74a64dbf556
+30a6249d91a8aa1209d7c19b95f52f69
+265d8873c643098e847b784688baf663
+73f2e840477567ad0534fc5cfdf36fcc
+d1a8b3662fceaecc62cc36f57ca317bd
+aa212c1ac005b3c12f9660a519c3e271
+65928e222a0a04a07779ab2aa1e9a56f
+33a7efa054d361467ec859772a5e6e6d
+0660181de57d94f6b647d5d1f3a4525a
+84852fa4fafed236348d3915ea64d55d
+3ff67d7f2aba5a9926cf6d69e149d84f
+c2118f6ea57f616fdb446b486c2434b6
+c77cfdc46be685b4fddff7c66b5463aa
+420045214ef1ac72fecf8f593e43be1e
+c10684a0abb27d4f8e0e55bd6cad51f1
+6a29741ee72b4a4f25ffdd37b02a06d1
+a8fa35202ef2d5ab93a8d2ae5b8e7f48
+90b7fda3d8a86b2d41f35015f6d7c786
+16ee44588b3b5cd1b79d1fcc34c698a5
+8594e4fa8f45ef1c91a2b5059bfa394c
+a8ac81415a9299a8fc49a015a5f65e31
+57a660d77d0a181718a5aba6486652d0
+c588bcead1d3004908f5c399d67cd5c6
+4fdf7ad7838fe4f9e11aadcf7289c5bb
+2d836d62d55644b9a9f5977f92def6bb
+ccfd127b2e018dd2631d0e466fc2f50a
+8d1b778619146f481b4926ded8460842
+ecf26014aad849e56c215a3924856e78
+4a2666f05b7d1e1374ed94680627d21b
+a5f50af56ee65bed5203f5abf96f7056
+1f1d43cbc07c9b8ac374ae7f866bab8b
+1d8ed3ed31180b83d126c2e722ed2ff8
+6d30763579eef8bf801bbd2e9ec0aa4a
+d806adb8d93434f461929eee327d02eb
+12cff4861273a826d21c3d6a06e589cc
+92d3eccc237425b3f4b143e70aba2b55
+0337e0909b5096b599ea2eb38165f55b
+369153ef2049a590c4e448343be694f5
+d787bb062ba78e07baf5507d6bcfd166
+439a55d197ad1cd8fbcf799f00d8edce
+5ab02c8112ba4c69a384babc858930e5
+bf3857a080d8dc08b64705ec9a61c063
+e4d0daa78fffe1528e2849af6409e4b3
+d76d2c2a090427e50ef1fbd4ce740e69
+8652f47129678666f7c35647e5baf8a9
+32d80af6f8fd4a2b8fd859e007866c52
+9ef2c52ebb1c59aa61adeb81b7e1a225
+26eec265ceb92e47ae8e4b097815a64f
+8ee178231860224779bf26d2048287a5
+a21fe5464e91b48cebca122b4a2d1509
+c81bc7c94c8e47193b5382a365766233
+e4c7ec4cad948f8fbb64c4e8c8eba818
+31ed712ca7250db0492b779e994a19bd
+5a3a9cf83d5e6f023414237c2dbe2d22
+67749ee082eef8916064e2da2db5bd38
+2c6e3d0d4f2378ae8714edb534fd92c1
+52cd3daf60ea7af3bc86b8f15df140f3
+5e2f04ed330fcd99f97dffd5dbba06da
+9a3550ded0110dc9b49dc6cd01352a82
+e54397613e9e3c30e1df0fe11eed778a
+089720356c98a43ffc128649955e89d7
+1b6faa0e760ce872fd4ef830cef92cc1
+8c4aec42909d3362fc8ca948922d6eaa
+5b4274bbf2ece5adb315622e542a05ed
+bb0ca62a4f72f8446cce7d17d48fc8fe
+9dd727099f4f9af19bebefa66b0ed91a
+c0499f4f283cfb8311c263e4e6c8e076
+175576949a4e83d6f71b68ba3f2df0c2
+87b750219e1de1b8f7b4090b5d49ea55
+1c91b444a8bf6eba2a93282ae76da534
+5cc82cf35ac62e2b9231a49a9c543b51
+7db89d819bf3e572de6f146ab2020805
+484f880698e9c7aa8b02ba5572e8ecb9
+51a08f9e6bbaf1784ef120690b27ef49
+d9af04e4a34000dce7b2e74a93dde00d
+31b6bab72467d80d8b05df2645fd4941
+3180fadb00d3697c6e116d447f063c5f
+bd7a1f591624f6c7a4528bebf8f0f3a0
+815bd4094c179982bdbc1c4d86cb7f74
+ddd141b2ce436a6a93afbcc392e0197c
+eed862c52d624243092d17daef1d0039
+cf9366953f9946c36bcdfd5d5a71b69c
+40940bd3b2473354a347bc12ef0a69b1
+238f11ee51ff9cef603b4f1bf50351a4
+35fc9cd588a03170dd53eda9055cef7f
+1288daf454c0bbcb723d1ed8f468833b
+095b863c9899a7d73c891a8dbb432c21
+7536994dd6d261787e8fe17fb0dc2145
+fb7fb33436971d4012caaf25e691c89c
+79ca953f1686174f773ebf01667ce961
+25ff19480644a774619228e0d2437df3
+cd64fbeb065c9016cfbe0466c7c152e4
+79d49665084d817e5421d6c73d4bd70f
+dc756475ea1a1d231fe907553ea6bfff
+934405395545a2d3c80765c864fd5bb3
+e8f4844d1d87351efe475358ae7b7ce2
+bbde90430bd773d7159c22900de12c88
+b5492af6912d89654cfea83a97a31a1a
+75c58ff13445a3bd2d6372375ae11464
+ef26897db8b01cd0cb0bfeec73fbff1b
+b634032ffc68d973fa7688fec1aac819
+92fac2e05dbe150ff4065c44020da1e1
+4a4a2224494b176cdf5eaf9fe191e842
+0d550934377243ac903094d2863b58d5
+f56043e73b46ec87e5ae628bf11833ed
+aa0ec28a6cb2d5c8ca7fdfc839e07fe1
+09e2662aeb1e7de54a2330c14c834e50
+a21b67cad1537c5421bdb0861747c5eb
+0de5a9b480b8fbadf88975a747a41147
+022de796aad39353d5fc8613959f63e3
+102b1386bb39edc570f92ad4b86ad5e5
+091b5c292fd3d58edd526d8c9f551fd9
+bee833013813e33e0973f89b2283b062
+8bd81bec074d92da9a2ba3e6c3a003dc
+2c127b04612d717fd6cdabd2030e5099
+75f5a33c76e00be172fbd8bfcc07ae34
+d1861153a7abebeb463b7a18d86a25f3
+17a764f5c7942944e3f150f3239519a1
+1c761d14ee5363056c2978c54a50bb16
+bca13d60398010fa09dacccebbfe0534
+aaccf98095f00b789378f40d2da63b75
+8b92e8b59f4c8e4f638d80bdd5c6111e
+ebf8e90e54c45a52075f0a7e04835c89
+d740321a1a9b27e670dc531c6dcbcebf
+511c6906febda60ca21e69210084282d
+d3e087d30120be7d56fa45d5d7e89d48
+7f5c9cf62a3f027389f083262042a810
+1244fb2a4c1b74c381ac4c20b811bfb1
+77a428a05c300ca5cdb3fa27da7b918a
+6360b1deaecb1e05afc7417bfe3e0cb5
+371472a9e16c6bee6212080a5e3f9b76
+1f0c168858cfa8515c9795e5b88dff2b
+4fa1d004ec68f7832e5cb769b609fabd
+0bb735f1e48e9d3e2a497ac674c8e1ee
+3c621b49a27fd3c9071ecd50cb236daa
+ad2b4273f31e56095e8d92a6435315ed
+2c381a8e6619b7c712829aff5023d8ae
+26a2b4be6422c36ca4fc3a780fea1a28
+2613758f009a0a1a5f026778ef2b2e4e
+3bb39d1768e2370be299ccd8ad844b3d
+9147beac122dc839bc045753e058f06d
+b58dee54d8b5e0d95c64c59409edd3d7
+ae6937bc589b15647792a10e3dea84f5
+6742316fb5cba4da00c617277a9261a3
+97e995e50629de305d4b75587f0fbbed
+616175a38a149c242082361dc55382ef
+879e09049f4f7dbb25f2b674b07cc48b
+f215f0a18c2d72df279dc9f5e2f01628
+bfb93fad7fbc25b064af430f49168ad4
+16098399c3915af025e65ef57eeac6a0
+07f62d76e08ca84d6aaea9263de9a9c6
+84f5a0248c80d708a88b2936382a870c
+c98f935e91e46e4e02f50ba9b2427848
+11dbb2ba75cb39c36caa72f5f788917c
+2b0655e5a1e376b4e0f3096bf4f33b0e
+461765fb8967dcdfa864d51b0ce66602
+1130da968809d16ccbd428a2a956fb87
+8184f25887a039b6d740fa4993177492
+efc329b236ff31fb09704531ebd98d68
+72be393988f99181bc50514be845b63f
+2997c759dad97d8805daa79eb6cfa3b0
+87c5ee9e9a8fe583937f26dec0555df9
+693d55a4cf3c4c5f45d7ab668927bb2d
+2ada19592e01b22a856b0ffcdc429aec
+d1439c04e7b7b42bf559daf018dc9d29
+e2bec322df3e4948bd417ddd3ab2b479
+140a2d87b9a4f5a092747deeeb385ddd
+562ba8b5cec9123258331a5801cf2631
+678302487c96f554a0d0143a58bf90e6
+b2d3a92f7cd12a5d13d52f6088c255b8
+56d861b31167fd8f10f81de2ce545c93
+4377d95b12276ae44c3c04b361c59615
+3b491798c958a64b61c045ac2c057158
+82b9a4b9c02ee170de846d1932bdd065
+438f461305a864eeeab3afaf80905bae
+00012582033a06cef8cc4a80d9e52a54
+9f93bb52850140c8c1b062eebefdc559
+95ae848a11b95b79363a4cc002c71f0d
+58f4aa1dd606031abecb28f41ddcfa4b
+7ee7d12102d12a9b2f21dcf1a62571c9

PyTorch/NLP/BERT/tf_to_torch/config_DGX1_1x8x6x6.sh

+## DL params
+export BATCHSIZE=36
+export LR=4.0e-4
+export GRADIENT_STEPS=6
+export MAX_STEPS=13889
+export WARMUP_PROPORTION=0.0
+export PHASE=2
+export MAX_SAMPLES_TERMINATION=4500000
+export EXTRA_PARAMS="--unpad"
+
+## System run parms
+export DGXNNODES=1
+export DGXSYSTEM=$(basename $(readlink -f ${BASH_SOURCE[0]}) | sed 's/^config_//' | sed 's/\.sh$//' )
+export WALLTIME=04:00:00
+
+## System config params
+export DGXNGPU=8
+export DGXSOCKETCORES=20
+export DGXNSOCKET=2
+export DGXHT=2         # HT is on is 2, HT off is 1

PyTorch/NLP/BERT/tf_to_torch/config_DGX2_1x16x27x1.sh

+## DL params
+export BATCHSIZE=27
+export LR=4.0e-4
+export GRADIENT_STEPS=1
+export MAX_STEPS=8103
+export WARMUP_PROPORTION=0.0
+export PHASE=2
+export MAX_SAMPLES_TERMINATION=4500000
+export EXTRA_PARAMS="--unpad"
+
+## System run parms
+export DGXNNODES=1
+export DGXSYSTEM=$(basename $(readlink -f ${BASH_SOURCE[0]}) | sed 's/^config_//' | sed 's/\.sh$//' )
+export WALLTIME=02:00:00
+
+## System config params
+export DGXNGPU=16
+export DGXSOCKETCORES=24
+export DGXNSOCKET=2
+export DGXHT=2         # HT is on is 2, HT off is 1

PyTorch/NLP/BERT/tf_to_torch/config_DGX2_32x16x18x1.sh

+## DL params
+export BATCHSIZE=18
+export LR=2.0e-3
+export GRADIENT_STEPS=1
+export MAX_STEPS=750
+export WARMUP_PROPORTION=0.0
+export OPT_LAMB_BETA_1=0.86
+export OPT_LAMB_BETA_2=0.975
+export PHASE=2
+export MAX_SAMPLES_TERMINATION=7500000
+export EXTRA_PARAMS=""
+
+## System run parms
+export DGXNNODES=32
+export DGXSYSTEM=$(basename $(readlink -f ${BASH_SOURCE[0]}) | sed 's/^config_//' | sed 's/\.sh$//' )
+export WALLTIME=00:30:00
+
+## System config params
+export DGXNGPU=16
+export DGXSOCKETCORES=24
+export DGXNSOCKET=2
+export DGXHT=2         # HT is on is 2, HT off is 1

PyTorch/NLP/BERT/tf_to_torch/config_DGX2_96x16x18x1.sh

+## DL params
+export BATCHSIZE=18
+export LR=3.0e-3
+export GRADIENT_STEPS=1
+export MAX_STEPS=550
+export WARMUP_PROPORTION=0.0
+export OPT_LAMB_BETA_1=0.87
+export OPT_LAMB_BETA_2=0.98
+export PHASE=2
+export MAX_SAMPLES_TERMINATION=13000000
+export EXTRA_PARAMS=""
+
+## System run parms
+export DGXNNODES=96
+export DGXSYSTEM=$(basename $(readlink -f ${BASH_SOURCE[0]}) | sed 's/^config_//' | sed 's/\.sh$//' )
+export WALLTIME=00:45:00
+
+## System config params
+export DGXNGPU=16
+export DGXSOCKETCORES=24
+export DGXNSOCKET=2
+export DGXHT=2         # HT is on is 2, HT off is 1

PyTorch/NLP/BERT/tf_to_torch/config_DGXA100_128x8x10x1.sh

+## DL params
+export BATCHSIZE=10
+export LR=2e-3
+export GRADIENT_STEPS=1
+export MAX_STEPS=750
+export WARMUP_PROPORTION=0.0
+export OPT_LAMB_BETA_1=0.86
+export OPT_LAMB_BETA_2=0.975
+export PHASE=2
+export MAX_SAMPLES_TERMINATION=12000000
+export EXTRA_PARAMS=""
+## System run parms
+export DGXNNODES=128
+export DGXSYSTEM=$(basename $(readlink -f ${BASH_SOURCE[0]}) | sed 's/^config_//' | sed 's/\.sh$//' )
+export WALLTIME=00:50:00
+
+## System config params
+export DGXNGPU=8
+export DGXSOCKETCORES=24
+export DGXNSOCKET=2
+export DGXHT=2         # HT is on is 2, HT off is 1

PyTorch/NLP/BERT/tf_to_torch/config_DGXA100_1x8x32x1.sh

+## DL params
+export BATCHSIZE=32
+export LR=3.5e-4
+export GRADIENT_STEPS=1
+export MAX_STEPS=13700
+export WARMUP_PROPORTION=0.0
+export PHASE=2
+export MAX_SAMPLES_TERMINATION=4500000
+export EXTRA_PARAMS="--unpad"
+
+## System run parms
+export DGXNNODES=1
+export DGXSYSTEM=$(basename $(readlink -f ${BASH_SOURCE[0]}) | sed 's/^config_//' | sed 's/\.sh$//' )
+export WALLTIME=02:00:00
+
+## System config params
+export DGXNGPU=8
+export DGXSOCKETCORES=24
+export DGXNSOCKET=2
+export DGXHT=2         # HT is on is 2, HT off is 1

PyTorch/NLP/BERT/tf_to_torch/config_DGXA100_256x8x4x1.sh

+## DL params
+# steps500_lr0.004_wup0.1_b1_0.878_b2_0.974
+export BATCHSIZE=4
+export LR=0.00288293
+export GRADIENT_STEPS=1
+export MAX_STEPS=560
+export WARMUP_STEPS=287
+export START_WARMUP_STEP=-76
+export OPT_LAMB_BETA_1=0.88
+export OPT_LAMB_BETA_2=0.88
+export TARGET_MLM_ACCURACY=0.706
+export WEIGHT_DECAY_RATE=0.0166629
+export PHASE=2
+export MAX_SAMPLES_TERMINATION=12000000
+export INIT_LOSS_SCALE=16384
+export EVAL_ITER_START_SAMPLES=3047424
+export EVAL_ITER_SAMPLES=507904
+export EXTRA_PARAMS=" "
+
+## System run parms
+export DGXNNODES=256
+export DGXSYSTEM=$(basename $(readlink -f ${BASH_SOURCE[0]}) | sed 's/^config_//' | sed 's/\.sh$//' )
+export WALLTIME=00:50:00
+
+## System config params
+export DGXNGPU=8
+export DGXSOCKETCORES=24
+export DGXNSOCKET=2
+export DGXHT=2         # HT is on is 2, HT off is 1
+
+export CONTAINER_PRELOAD_LUSTRE=1
+export USE_DDP=1