all

523ec9cc · wangsen · 523ec9cc · 523ec9cc · 523ec9cc · 523ec9cc
Commit 523ec9cc authored Sep 03, 2024 by wangsen
20 changed files
--- a/megatron/core/__pycache__/__init__.cpython-310.pyc
+++ b/megatron/core/__pycache__/__init__.cpython-310.pyc
--- a/megatron/core/__pycache__/enums.cpython-310.pyc
+++ b/megatron/core/__pycache__/enums.cpython-310.pyc
--- a/megatron/core/__pycache__/inference_params.cpython-310.pyc
+++ b/megatron/core/__pycache__/inference_params.cpython-310.pyc
--- a/megatron/core/__pycache__/jit.cpython-310.pyc
+++ b/megatron/core/__pycache__/jit.cpython-310.pyc
--- a/megatron/core/__pycache__/model_parallel_config.cpython-310.pyc
+++ b/megatron/core/__pycache__/model_parallel_config.cpython-310.pyc
--- a/megatron/core/__pycache__/package_info.cpython-310.pyc
+++ b/megatron/core/__pycache__/package_info.cpython-310.pyc
--- a/megatron/core/__pycache__/packed_seq_params.cpython-310.pyc
+++ b/megatron/core/__pycache__/packed_seq_params.cpython-310.pyc
--- a/megatron/core/__pycache__/parallel_state.cpython-310.pyc
+++ b/megatron/core/__pycache__/parallel_state.cpython-310.pyc
--- a/megatron/core/__pycache__/timers.cpython-310.pyc
+++ b/megatron/core/__pycache__/timers.cpython-310.pyc
--- a/megatron/core/__pycache__/utils.cpython-310.pyc
+++ b/megatron/core/__pycache__/utils.cpython-310.pyc
--- a/megatron/core/datasets/Makefile
+++ b/megatron/core/datasets/Makefile
+CXXFLAGS += -O3 -Wall -shared -std=c++11 -fPIC -fdiagnostics-color
+CPPFLAGS += $(shell python3 -m pybind11 --includes)
+LIBNAME = helpers
+LIBEXT = $(shell python3-config --extension-suffix)
+
+default: $(LIBNAME)$(LIBEXT)
+
+%$(LIBEXT): %.cpp
+	$(CXX) $(CXXFLAGS) $(CPPFLAGS) $< -o $@
--- a/megatron/core/datasets/__init__.py
+++ b/megatron/core/datasets/__init__.py
--- a/megatron/core/datasets/__pycache__/__init__.cpython-310.pyc
+++ b/megatron/core/datasets/__pycache__/__init__.cpython-310.pyc
--- a/megatron/core/datasets/__pycache__/blended_dataset.cpython-310.pyc
+++ b/megatron/core/datasets/__pycache__/blended_dataset.cpython-310.pyc
--- a/megatron/core/datasets/__pycache__/blended_megatron_dataset_builder.cpython-310.pyc
+++ b/megatron/core/datasets/__pycache__/blended_megatron_dataset_builder.cpython-310.pyc
--- a/megatron/core/datasets/__pycache__/blended_megatron_dataset_config.cpython-310.pyc
+++ b/megatron/core/datasets/__pycache__/blended_megatron_dataset_config.cpython-310.pyc
--- a/megatron/core/datasets/__pycache__/gpt_dataset.cpython-310.pyc
+++ b/megatron/core/datasets/__pycache__/gpt_dataset.cpython-310.pyc
--- a/megatron/core/datasets/__pycache__/indexed_dataset.cpython-310.pyc
+++ b/megatron/core/datasets/__pycache__/indexed_dataset.cpython-310.pyc
--- a/megatron/core/datasets/__pycache__/megatron_dataset.cpython-310.pyc
+++ b/megatron/core/datasets/__pycache__/megatron_dataset.cpython-310.pyc
--- a/megatron/core/datasets/__pycache__/megatron_tokenizer.cpython-310.pyc
+++ b/megatron/core/datasets/__pycache__/megatron_tokenizer.cpython-310.pyc