初版适配

K100AI_finetune.sh

@@ -35,6 +35,8 @@ VALID_NAMES="[databricks-dolly-15k]"
 # CONCAT_SAMPLING_PROBS="[0.3,0.7]" # "[1]" # 只有一个数据集设置为1
 CONCAT_SAMPLING_PROBS="[1]"
 
+# 可能需要导入环境变量
+export LD_PRELOAD=/usr/local/lib/python3.10/site-packages/transformer_engine.libs/libgalaxyhip-8e217ef3.so.5.2.24472.1059-0a6afed7
 # 运行训练脚本
 torchrun --nproc_per_node 8 \
    /workspace/nemo_main/NeMo-2.0.0.rc0.beta/examples/nlp/language_modeling/tuning/megatron_gpt_finetuning.py \

NeMo-2.0.0.rc0.beta/requirements/requirements.txt

+huggingface_hub>=0.20.3
+numba
+numpy>=1.22
+onnx>=1.7.0
+python-dateutil
+ruamel.yaml
+scikit-learn
+setuptools>=65.5.1
+tensorboard
+text-unidecode
+torch
+tqdm>=4.41.0
+wget
+wrapt

NeMo-2.0.0.rc0.beta/requirements/requirements_asr.txt

+braceexpand
+editdistance
+g2p_en
+ipywidgets
+jiwer
+kaldi-python-io
+kaldiio
+lhotse>=1.20.0
+librosa>=0.10.0
+marshmallow
+matplotlib
+packaging
+pyannote.core
+pyannote.metrics
+pydub
+pyloudnorm
+resampy
+ruamel.yaml
+scipy>=0.14
+soundfile
+sox
+texterrors

NeMo-2.0.0.rc0.beta/requirements/requirements_common.txt

+datasets
+inflect
+pandas
+sacremoses>=0.0.43
+sentencepiece<1.0.0

NeMo-2.0.0.rc0.beta/requirements/requirements_docs.txt

+boto3
+Jinja2
+latexcodec
+numpy
+pydata-sphinx-theme
+Sphinx
+sphinx-book-theme
+sphinx-copybutton
+sphinxcontrib-bibtex
+sphinxext-opengraph
+urllib3
+wrapt

NeMo-2.0.0.rc0.beta/requirements/requirements_lightning.txt

+hydra-core>1.3,<=1.3.2
+omegaconf<=2.3
+pytorch-lightning>=2.2.1
+torchmetrics>=0.11.0
+transformers>=4.36.0
+wandb
+webdataset>=0.2.86

NeMo-2.0.0.rc0.beta/requirements/requirements_multimodal.txt

+addict
+clip
+diffusers>=0.19.3
+einops_exts
+imageio
+kornia
+nerfacc>=0.5.3
+open_clip_torch
+PyMCubes
+taming-transformers
+torchdiffeq
+torchsde
+trimesh

NeMo-2.0.0.rc0.beta/requirements/requirements_nlp.txt

+boto3
+einops
+faiss-cpu
+fasttext
+flask_restful
+ftfy
+gdown
+h5py
+ijson
+jieba
+markdown2
+matplotlib>=3.3.2
+megatron_core==0.5.0
+nltk>=3.6.5
+opencc<1.1.7
+pangu
+rapidfuzz
+rouge_score
+sacrebleu  # manually install sacrebleu[ja] for Japanese support; MeCab is unsupported in Python 3.11+
+sentence_transformers
+tensorstore<0.1.46
+zarr

NeMo-2.0.0.rc0.beta/requirements/requirements_slu.txt

+jiwer>=2.0.0
+progress>=1.5
+tabulate>=0.8.7
+textdistance>=4.1.5
+tqdm

NeMo-2.0.0.rc0.beta/requirements/requirements_test.txt

+black==19.10b0
+click==8.0.2
+isort>5.1.0,<6.0.0
+parameterized
+pytest
+pytest-runner
+ruamel.yaml
+sphinx
+sphinxcontrib-bibtex
+wandb
+wget
+wrapt

NeMo-2.0.0.rc0.beta/requirements/requirements_tts.txt

+attrdict
+einops
+jieba
+kornia
+librosa
+matplotlib
+# pynini does not currently support aarch, disable nemo_text_processing for now
+nemo_text_processing; 'arm' not in platform_machine and 'aarch' not in platform_machine
+nltk
+pandas
+pypinyin
+pypinyin-dict