Add run_bert_ict.sh

run_bert_ict.sh

+#!/bin/bash
+LENGTH=512
+CHKPT="chkpts/debug"
+COMMAND="/home/scratch.gcf/adlr-utils/release/cluster-interface/latest/mp_launch python pretrain_bert_ict.py \
+       --num-layers 6 \
+       --hidden-size 768\
+       --num-attention-heads 12 \
+       --batch-size 1 \
+       --checkpoint-activations \
+       --seq-length $LENGTH \
+       --max-position-embeddings $LENGTH \
+       --train-iters 1000 \
+       --no-save-optim --no-save-rng \
+       --save $CHKPT \
+       --resume-dataloader \
+       --train-data /home/universal-lm-data.cosmos549/datasets/wikipedia/wikidump_lines.json \
+       --presplit-sentences \
+       --loose-json \
+       --text-key text \
+       --data-loader lazy \
+       --tokenizer-type BertWordPieceTokenizer \
+       --cache-dir cache \
+       --split 58,1,1 \
+       --distributed-backend nccl \
+       --lr 0.00015 \
+       --num-workers 0 \
+       --no-load-optim --finetune \
+       --lr-decay-style cosine \
+       --weight-decay 1e-2 \
+       --clip-grad 1.0 \
+       --warmup .01 \
+       --save-interval 1000 \
+       --fp16 --adlr-autoresume --adlr-autoresume-interval 5000"
+submit_job --image 'http://gitlab-master.nvidia.com/adlr/megatron-lm/megatron:rouge_score' --mounts /home/universal-lm-data.cosmos549,/home/raulp -c "${COMMAND}" --name ict_test --partition interactive --gpu 8 --nodes 2 --autoresume_timer 300 -i