initial llama

0211193c · zhuwenwen · 0211193c · 0211193c · 0211193c · 0211193c
Commit 0211193c authored Aug 17, 2023 by zhuwenwen
20 changed files
--- a/3rdparty/Megatron-LM/megatron/data/blendable_dataset.py
+++ b/3rdparty/Megatron-LM/megatron/data/blendable_dataset.py
--- a/3rdparty/Megatron-LM/megatron/data/data_samplers.py
+++ b/3rdparty/Megatron-LM/megatron/data/data_samplers.py
--- a/3rdparty/Megatron-LM/megatron/data/dataset_utils.py
+++ b/3rdparty/Megatron-LM/megatron/data/dataset_utils.py
--- a/3rdparty/Megatron-LM/megatron/data/gpt_dataset.py
+++ b/3rdparty/Megatron-LM/megatron/data/gpt_dataset.py
--- a/3rdparty/Megatron-LM/megatron/data/helpers.cpp
+++ b/3rdparty/Megatron-LM/megatron/data/helpers.cpp
--- a/3rdparty/Megatron-LM/megatron/data/ict_dataset.py
+++ b/3rdparty/Megatron-LM/megatron/data/ict_dataset.py
--- a/3rdparty/Megatron-LM/megatron/data/image_folder.py
+++ b/3rdparty/Megatron-LM/megatron/data/image_folder.py
--- a/3rdparty/Megatron-LM/megatron/data/indexed_dataset.py
+++ b/3rdparty/Megatron-LM/megatron/data/indexed_dataset.py
--- a/3rdparty/Megatron-LM/megatron/data/orqa_wiki_dataset.py
+++ b/3rdparty/Megatron-LM/megatron/data/orqa_wiki_dataset.py
--- a/3rdparty/Megatron-LM/megatron/data/realm_dataset_utils.py
+++ b/3rdparty/Megatron-LM/megatron/data/realm_dataset_utils.py
--- a/3rdparty/Megatron-LM/megatron/data/realm_index.py
+++ b/3rdparty/Megatron-LM/megatron/data/realm_index.py
--- a/3rdparty/Megatron-LM/megatron/data/t5_dataset.py
+++ b/3rdparty/Megatron-LM/megatron/data/t5_dataset.py
--- a/3rdparty/Megatron-LM/megatron/data/test/test_indexed_dataset.py
+++ b/3rdparty/Megatron-LM/megatron/data/test/test_indexed_dataset.py
--- a/3rdparty/Megatron-LM/megatron/data/test/test_preprocess_data.sh
+++ b/3rdparty/Megatron-LM/megatron/data/test/test_preprocess_data.sh
+#!/bin/bash
+
+IMPL=cached
+python ../preprocess_data.py \
+       --input test_samples.json \
+       --vocab vocab.txt \
+       --dataset-impl ${IMPL} \
+       --output-prefix test_samples_${IMPL} \
+       --workers 1 \
+       --log-interval 2
--- a/3rdparty/Megatron-LM/megatron/data/vit_dataset.py
+++ b/3rdparty/Megatron-LM/megatron/data/vit_dataset.py
--- a/3rdparty/Megatron-LM/megatron/dist_signal_handler.py
+++ b/3rdparty/Megatron-LM/megatron/dist_signal_handler.py
--- a/3rdparty/Megatron-LM/megatron/fp16_deprecated/loss_scaler.py
+++ b/3rdparty/Megatron-LM/megatron/fp16_deprecated/loss_scaler.py
--- a/3rdparty/Megatron-LM/megatron/fused_kernels/__init__.py
+++ b/3rdparty/Megatron-LM/megatron/fused_kernels/__init__.py
--- a/3rdparty/Megatron-LM/megatron/fused_kernels/compat.h
+++ b/3rdparty/Megatron-LM/megatron/fused_kernels/compat.h
--- a/3rdparty/Megatron-LM/megatron/fused_kernels/layer_norm_cuda.cpp
+++ b/3rdparty/Megatron-LM/megatron/fused_kernels/layer_norm_cuda.cpp