conflict changed

17b04444 · cardy20 · c69f6c38 · c69f6c38 · c69f6c38 · c69f6c38
Commit 17b04444 authored Jun 03, 2023 by cardy20
Hide whitespace changes
Inline Side-by-side

Showing with 0 additions and 36 deletions

get_ngram.sh get_ngram.sh +0 -4

get_ngram2.sh get_ngram2.sh +0 -4

ngrams.log ngrams.log +0 -28

No files found.
--- a/get_ngram.sh
+++ b/get_ngram.sh
-export PYTHONPATH=$PWD
-python3 scripts/clean_training_data/generate_13_grams.py \
-        -dir /fsx/polyglot/massivetext_large_data/ \
-        -sdir /fsx/lime12/ngram_train2/ -n 13 -buckets 500
--- a/get_ngram2.sh
+++ b/get_ngram2.sh
-export PYTHONPATH=$PWD
-python3 scripts/clean_training_data/generate_13_grams.py \
-        -dir /fsx/kevinai/data/ko/merged_raw/ \
-        -sdir /fsx/lime12/ngram_merged_raw -n 13 -buckets 500
\ No newline at end of file
--- a/ngrams.log
+++ b/ngrams.log
-INFO - 05/29/23 02:24:05 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 02:24:05 - 0:00:00 - Starting at pile document index 0
-INFO - 05/29/23 02:26:29 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 02:26:29 - 0:00:00 - Starting at pile document index 106000
-INFO - 05/29/23 02:29:19 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 02:29:19 - 0:00:00 - Starting at pile document index 0
-INFO - 05/29/23 02:31:50 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 02:31:50 - 0:00:00 - Starting at pile document index 0
-INFO - 05/29/23 02:32:22 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 02:32:22 - 0:00:00 - ngrams already generated and bucketed, skipping
-INFO - 05/29/23 02:34:01 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 02:34:01 - 0:00:00 - Starting at pile document index 0
-INFO - 05/29/23 02:34:58 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 02:34:58 - 0:00:00 - Starting at pile document index 0
-INFO - 05/29/23 07:12:33 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 07:12:33 - 0:00:00 - Starting at pile document index 0
-INFO - 05/29/23 07:26:46 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 07:26:46 - 0:00:00 - Starting at pile document index 0
-INFO - 05/29/23 07:30:21 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 07:30:21 - 0:00:00 - Starting at pile document index 0
-INFO - 05/29/23 07:31:54 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 07:31:54 - 0:00:00 - Starting at pile document index 0
-INFO - 05/29/23 13:27:39 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 13:27:39 - 0:00:00 - Starting at pile document index 8432000
-INFO - 05/29/23 13:30:28 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 13:30:28 - 0:00:00 - Starting at pile document index 0
-INFO - 05/29/23 14:27:00 - 0:00:00 - Generating 13-grams and bucketing.
-INFO - 05/29/23 14:27:00 - 0:00:00 - Starting at pile document index 0