Skip to content

GitLab

Unverified Commit fb4f530b authored by Cade Daniel's avatar Cade Daniel Committed by GitHub
Browse files

[CI] [nightly benchmark] Do not re-download sharegpt dataset if exists (#6706)

parent 79319ced
Show whitespace changes
Inline Side-by-side
Showing with 10 additions and 1 deletion
.buildkite/nightly-benchmarks/run-benchmarks-suite.sh
View file @ fb4f530b
...@@ -34,6 +34,15 @@ check_hf_token() { ...@@ -34,6 +34,15 @@ check_hf_token() {
fi fi
} }
ensure_sharegpt_downloaded() {
local FILE=ShareGPT_V3_unfiltered_cleaned_split.json
if [ ! -f "$FILE" ]; then
wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/$FILE
else
echo "$FILE already exists."
fi
}
json2args() { json2args() {
# transforms the JSON string to command line args, and '_' is replaced to '-' # transforms the JSON string to command line args, and '_' is replaced to '-'
# example: # example:
...@@ -350,7 +359,7 @@ main() { ...@@ -350,7 +359,7 @@ main() {
# prepare for benchmarking # prepare for benchmarking
cd benchmarks || exit 1 cd benchmarks || exit 1
wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json ensure_sharegpt_downloaded
declare -g RESULTS_FOLDER=results/ declare -g RESULTS_FOLDER=results/
mkdir -p $RESULTS_FOLDER mkdir -p $RESULTS_FOLDER
QUICK_BENCHMARK_ROOT=../.buildkite/nightly-benchmarks/ QUICK_BENCHMARK_ROOT=../.buildkite/nightly-benchmarks/
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment