Commit 32e1332a authored by VictorSanh's avatar VictorSanh
Browse files

[distil] fix once for all general logger for scripts

parent b62abe87
......@@ -21,8 +21,12 @@ import random
import time
import numpy as np
from pytorch_transformers import BertTokenizer
import logging
from examples.distillation.utils import logger
logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt = '%m/%d/%Y %H:%M:%S',
level = logging.INFO)
logger = logging.getLogger(__name__)
def main():
parser = argparse.ArgumentParser(description="Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).")
......@@ -74,4 +78,4 @@ def main():
if __name__ == "__main__":
main()
\ No newline at end of file
main()
......@@ -18,8 +18,12 @@ Preprocessing script before training DistilBERT.
from collections import Counter
import argparse
import pickle
import logging
from examples.distillation.utils import logger
logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt = '%m/%d/%Y %H:%M:%S',
level = logging.INFO)
logger = logging.getLogger(__name__)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment