Skip to content

GitLab

Commit a8e071c6 authored by chrislarson1's avatar chrislarson1
Browse files

added notebook to check correctness of the pytorch->tensorflow conversion

parent 0a4fb0da
Show whitespace changes
Inline Side-by-side
Showing with 1630 additions and 0 deletions
notebooks/Comparing-PT-and-TF-models.ipynb 0 → 100644
View file @ a8e071c6
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Pytorch to Tensorflow Conversion Test Notebook\n",
"\n",
"To run this notebook follow these steps, modifying the **Config** section as necessary:\n",
"\n",
"1. Point `pt_model_dir` to your local directory containing the pytorch Bert model to be converted.\n",
"2. Point `tf_bert_dir` to your clone of Google's Bert implementation which can be found here: https://github.com/google-research/bert.\n",
"\n",
"Note: \n",
"1. This feature currently only supports the base BERT models (uncased/cased).\n",
"2. Tensorflow model will be dumped in `tf_model_dir`."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Config"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"\n",
"model_cls = 'BertModel'\n",
"model_typ = 'bert-base-uncased'\n",
"token_cls = 'BertTokenizer'\n",
"max_seq = 12\n",
"CLS = \"[CLS]\"\n",
"SEP = \"[SEP]\"\n",
"MASK = \"[MASK]\"\n",
"CLS_IDX = 0\n",
"layer_idxs = tuple(range(12))\n",
"input_text = \"jim henson was a puppeteer\"\n",
"\n",
"pt_model_dir = \"/home/ubuntu/.pytorch-pretrained-BERT-cache/{}\".format(model_typ)\n",
"tf_bert_dir = \"/home/ubuntu/bert\"\n",
"\n",
"pt_vocab_file = os.path.join(pt_model_dir, \"vocab.txt\")\n",
"pt_init_ckpt = os.path.join(pt_model_dir, model_typ.replace(\"-\", \"_\") + \".bin\")\n",
"tf_model_dir = os.path.join(pt_model_dir, 'tf')\n",
"tf_vocab_file = os.path.join(tf_model_dir, \"vocab.txt\")\n",
"tf_init_ckpt = os.path.join(tf_model_dir, model_typ.replace(\"-\", \"_\") + \".ckpt\")\n",
"tf_config_file = os.path.join(tf_model_dir, \"bert_config.json\")\n",
"\n",
"if not os.path.isdir(tf_model_dir): \n",
" os.makedirs(tf_model_dir, exist_ok=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Tokenization"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def tokenize(text, tokenizer):\n",
" text = text.strip().lower()\n",
" tok_ids = tokenizer.tokenize(text)\n",
" if len(tok_ids) > max_seq - 2:\n",
" tok_ids = tok_ids[:max_seq - 2]\n",
" tok_ids.insert(CLS_IDX, CLS)\n",
" tok_ids.append(SEP)\n",
" input_ids = tokenizer.convert_tokens_to_ids(tok_ids)\n",
" mask_ids = [1] * len(input_ids)\n",
" seg_ids = [0] * len(input_ids)\n",
" padding = [0] * (max_seq - len(input_ids))\n",
" input_ids += padding\n",
" mask_ids += padding\n",
" seg_ids += padding\n",
" return input_ids, mask_ids, seg_ids"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Pytorch execution"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 231508/231508 [00:00<00:00, 41092464.26B/s]\n",
"100%|██████████| 407873900/407873900 [00:07<00:00, 58092479.52B/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Pytorch embedding shape: (1, 768)\n"
]
}
],
"source": [
"import numpy as np\n",
"import torch\n",
"from pytorch_pretrained_bert import (BertConfig,\n",
" BertModel, \n",
" BertTokenizer, \n",
" BertForSequenceClassification)\n",
"\n",
"# Save Vocab\n",
"pt_tokenizer = BertTokenizer.from_pretrained(\n",
" pretrained_model_name_or_path=model_typ, \n",
" cache_dir=pt_model_dir)\n",
"pt_tokenizer.save_vocabulary(pt_model_dir)\n",
"pt_tokenizer.save_vocabulary(tf_model_dir)\n",
"\n",
"# Save Model\n",
"pt_model = BertModel.from_pretrained(\n",
" pretrained_model_name_or_path=model_typ, \n",
" cache_dir=pt_model_dir).to('cpu')\n",
"pt_model.eval()\n",
"pt_model.config.hidden_dropout_prob = 0.0\n",
"pt_model.config.attention_probs_dropout_prob = 0.0\n",
"pt_model.config.to_json_file(tf_config_file)\n",
"torch.save(pt_model.state_dict(), pt_init_ckpt)\n",
"\n",
"# Inputs\n",
"input_ids_pt, mask_ids_pt, seg_ids_pt = tokenize(input_text, pt_tokenizer)\n",
"\n",
"# PT Embedding\n",
"tok_tensor = torch.tensor(input_ids_pt).to('cpu').unsqueeze(0)\n",
"seg_tensor = torch.tensor(seg_ids_pt).to('cpu').unsqueeze(0)\n",
"msk_tensor = torch.tensor(mask_ids_pt).to('cpu').unsqueeze(0)\n",
"attn_blks, nsp_logits = pt_model(tok_tensor, seg_tensor, msk_tensor)\n",
"pt_embedding = nsp_logits.detach().numpy() \n",
"print(\"Pytorch embedding shape: {}\".format(pt_embedding.shape))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Pytorch &rarr; Tensorflow conversion"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/nlp/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Colocations handled automatically by placer.\n",
"bert/embeddings/word_embeddings initialized\n",
"bert/embeddings/position_embeddings initialized\n",
"bert/embeddings/token_type_embeddings initialized\n",
"bert/embeddings/LayerNorm/gamma initialized\n",
"bert/embeddings/LayerNorm/beta initialized\n",
"bert/encoder/layer_0/attention/self/query/kernel initialized\n",
"bert/encoder/layer_0/attention/self/query/bias initialized\n",
"bert/encoder/layer_0/attention/self/key/kernel initialized\n",
"bert/encoder/layer_0/attention/self/key/bias initialized\n",
"bert/encoder/layer_0/attention/self/value/kernel initialized\n",
"bert/encoder/layer_0/attention/self/value/bias initialized\n",
"bert/encoder/layer_0/attention/output/dense/kernel initialized\n",
"bert/encoder/layer_0/attention/output/dense/bias initialized\n",
"bert/encoder/layer_0/attention/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_0/attention/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_0/intermediate/dense/kernel initialized\n",
"bert/encoder/layer_0/intermediate/dense/bias initialized\n",
"bert/encoder/layer_0/output/dense/kernel initialized\n",
"bert/encoder/layer_0/output/dense/bias initialized\n",
"bert/encoder/layer_0/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_0/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_1/attention/self/query/kernel initialized\n",
"bert/encoder/layer_1/attention/self/query/bias initialized\n",
"bert/encoder/layer_1/attention/self/key/kernel initialized\n",
"bert/encoder/layer_1/attention/self/key/bias initialized\n",
"bert/encoder/layer_1/attention/self/value/kernel initialized\n",
"bert/encoder/layer_1/attention/self/value/bias initialized\n",
"bert/encoder/layer_1/attention/output/dense/kernel initialized\n",
"bert/encoder/layer_1/attention/output/dense/bias initialized\n",
"bert/encoder/layer_1/attention/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_1/attention/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_1/intermediate/dense/kernel initialized\n",
"bert/encoder/layer_1/intermediate/dense/bias initialized\n",
"bert/encoder/layer_1/output/dense/kernel initialized\n",
"bert/encoder/layer_1/output/dense/bias initialized\n",
"bert/encoder/layer_1/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_1/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_2/attention/self/query/kernel initialized\n",
"bert/encoder/layer_2/attention/self/query/bias initialized\n",
"bert/encoder/layer_2/attention/self/key/kernel initialized\n",
"bert/encoder/layer_2/attention/self/key/bias initialized\n",
"bert/encoder/layer_2/attention/self/value/kernel initialized\n",
"bert/encoder/layer_2/attention/self/value/bias initialized\n",
"bert/encoder/layer_2/attention/output/dense/kernel initialized\n",
"bert/encoder/layer_2/attention/output/dense/bias initialized\n",
"bert/encoder/layer_2/attention/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_2/attention/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_2/intermediate/dense/kernel initialized\n",
"bert/encoder/layer_2/intermediate/dense/bias initialized\n",
"bert/encoder/layer_2/output/dense/kernel initialized\n",
"bert/encoder/layer_2/output/dense/bias initialized\n",
"bert/encoder/layer_2/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_2/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_3/attention/self/query/kernel initialized\n",
"bert/encoder/layer_3/attention/self/query/bias initialized\n",
"bert/encoder/layer_3/attention/self/key/kernel initialized\n",
"bert/encoder/layer_3/attention/self/key/bias initialized\n",
"bert/encoder/layer_3/attention/self/value/kernel initialized\n",
"bert/encoder/layer_3/attention/self/value/bias initialized\n",
"bert/encoder/layer_3/attention/output/dense/kernel initialized\n",
"bert/encoder/layer_3/attention/output/dense/bias initialized\n",
"bert/encoder/layer_3/attention/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_3/attention/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_3/intermediate/dense/kernel initialized\n",
"bert/encoder/layer_3/intermediate/dense/bias initialized\n",
"bert/encoder/layer_3/output/dense/kernel initialized\n",
"bert/encoder/layer_3/output/dense/bias initialized\n",
"bert/encoder/layer_3/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_3/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_4/attention/self/query/kernel initialized\n",
"bert/encoder/layer_4/attention/self/query/bias initialized\n",
"bert/encoder/layer_4/attention/self/key/kernel initialized\n",
"bert/encoder/layer_4/attention/self/key/bias initialized\n",
"bert/encoder/layer_4/attention/self/value/kernel initialized\n",
"bert/encoder/layer_4/attention/self/value/bias initialized\n",
"bert/encoder/layer_4/attention/output/dense/kernel initialized\n",
"bert/encoder/layer_4/attention/output/dense/bias initialized\n",
"bert/encoder/layer_4/attention/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_4/attention/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_4/intermediate/dense/kernel initialized\n",
"bert/encoder/layer_4/intermediate/dense/bias initialized\n",
"bert/encoder/layer_4/output/dense/kernel initialized\n",
"bert/encoder/layer_4/output/dense/bias initialized\n",
"bert/encoder/layer_4/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_4/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_5/attention/self/query/kernel initialized\n",
"bert/encoder/layer_5/attention/self/query/bias initialized\n",
"bert/encoder/layer_5/attention/self/key/kernel initialized\n",
"bert/encoder/layer_5/attention/self/key/bias initialized\n",
"bert/encoder/layer_5/attention/self/value/kernel initialized\n",
"bert/encoder/layer_5/attention/self/value/bias initialized\n",
"bert/encoder/layer_5/attention/output/dense/kernel initialized\n",
"bert/encoder/layer_5/attention/output/dense/bias initialized\n",
"bert/encoder/layer_5/attention/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_5/attention/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_5/intermediate/dense/kernel initialized\n",
"bert/encoder/layer_5/intermediate/dense/bias initialized\n",
"bert/encoder/layer_5/output/dense/kernel initialized\n",
"bert/encoder/layer_5/output/dense/bias initialized\n",
"bert/encoder/layer_5/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_5/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_6/attention/self/query/kernel initialized\n",
"bert/encoder/layer_6/attention/self/query/bias initialized\n",
"bert/encoder/layer_6/attention/self/key/kernel initialized\n",
"bert/encoder/layer_6/attention/self/key/bias initialized\n",
"bert/encoder/layer_6/attention/self/value/kernel initialized\n",
"bert/encoder/layer_6/attention/self/value/bias initialized\n",
"bert/encoder/layer_6/attention/output/dense/kernel initialized\n",
"bert/encoder/layer_6/attention/output/dense/bias initialized\n",
"bert/encoder/layer_6/attention/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_6/attention/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_6/intermediate/dense/kernel initialized\n",
"bert/encoder/layer_6/intermediate/dense/bias initialized\n",
"bert/encoder/layer_6/output/dense/kernel initialized\n",
"bert/encoder/layer_6/output/dense/bias initialized\n",
"bert/encoder/layer_6/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_6/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_7/attention/self/query/kernel initialized\n",
"bert/encoder/layer_7/attention/self/query/bias initialized\n",
"bert/encoder/layer_7/attention/self/key/kernel initialized\n",
"bert/encoder/layer_7/attention/self/key/bias initialized\n",
"bert/encoder/layer_7/attention/self/value/kernel initialized\n",
"bert/encoder/layer_7/attention/self/value/bias initialized\n",
"bert/encoder/layer_7/attention/output/dense/kernel initialized\n",
"bert/encoder/layer_7/attention/output/dense/bias initialized\n",
"bert/encoder/layer_7/attention/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_7/attention/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_7/intermediate/dense/kernel initialized\n",
"bert/encoder/layer_7/intermediate/dense/bias initialized\n",
"bert/encoder/layer_7/output/dense/kernel initialized\n",
"bert/encoder/layer_7/output/dense/bias initialized\n",
"bert/encoder/layer_7/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_7/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_8/attention/self/query/kernel initialized\n",
"bert/encoder/layer_8/attention/self/query/bias initialized\n",
"bert/encoder/layer_8/attention/self/key/kernel initialized\n",
"bert/encoder/layer_8/attention/self/key/bias initialized\n",
"bert/encoder/layer_8/attention/self/value/kernel initialized\n",
"bert/encoder/layer_8/attention/self/value/bias initialized\n",
"bert/encoder/layer_8/attention/output/dense/kernel initialized\n",
"bert/encoder/layer_8/attention/output/dense/bias initialized\n",
"bert/encoder/layer_8/attention/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_8/attention/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_8/intermediate/dense/kernel initialized\n",
"bert/encoder/layer_8/intermediate/dense/bias initialized\n",
"bert/encoder/layer_8/output/dense/kernel initialized\n",
"bert/encoder/layer_8/output/dense/bias initialized\n",
"bert/encoder/layer_8/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_8/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_9/attention/self/query/kernel initialized\n",
"bert/encoder/layer_9/attention/self/query/bias initialized\n",
"bert/encoder/layer_9/attention/self/key/kernel initialized\n",
"bert/encoder/layer_9/attention/self/key/bias initialized\n",
"bert/encoder/layer_9/attention/self/value/kernel initialized\n",
"bert/encoder/layer_9/attention/self/value/bias initialized\n",
"bert/encoder/layer_9/attention/output/dense/kernel initialized\n",
"bert/encoder/layer_9/attention/output/dense/bias initialized\n",
"bert/encoder/layer_9/attention/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_9/attention/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_9/intermediate/dense/kernel initialized\n",
"bert/encoder/layer_9/intermediate/dense/bias initialized\n",
"bert/encoder/layer_9/output/dense/kernel initialized\n",
"bert/encoder/layer_9/output/dense/bias initialized\n",
"bert/encoder/layer_9/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_9/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_10/attention/self/query/kernel initialized\n",
"bert/encoder/layer_10/attention/self/query/bias initialized\n",
"bert/encoder/layer_10/attention/self/key/kernel initialized\n",
"bert/encoder/layer_10/attention/self/key/bias initialized\n",
"bert/encoder/layer_10/attention/self/value/kernel initialized\n",
"bert/encoder/layer_10/attention/self/value/bias initialized\n",
"bert/encoder/layer_10/attention/output/dense/kernel initialized\n",
"bert/encoder/layer_10/attention/output/dense/bias initialized\n",
"bert/encoder/layer_10/attention/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_10/attention/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_10/intermediate/dense/kernel initialized\n",
"bert/encoder/layer_10/intermediate/dense/bias initialized\n",
"bert/encoder/layer_10/output/dense/kernel initialized\n",
"bert/encoder/layer_10/output/dense/bias initialized\n",
"bert/encoder/layer_10/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_10/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_11/attention/self/query/kernel initialized\n",
"bert/encoder/layer_11/attention/self/query/bias initialized\n",
"bert/encoder/layer_11/attention/self/key/kernel initialized\n",
"bert/encoder/layer_11/attention/self/key/bias initialized\n",
"bert/encoder/layer_11/attention/self/value/kernel initialized\n",
"bert/encoder/layer_11/attention/self/value/bias initialized\n",
"bert/encoder/layer_11/attention/output/dense/kernel initialized\n",
"bert/encoder/layer_11/attention/output/dense/bias initialized\n",
"bert/encoder/layer_11/attention/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_11/attention/output/LayerNorm/beta initialized\n",
"bert/encoder/layer_11/intermediate/dense/kernel initialized\n",
"bert/encoder/layer_11/intermediate/dense/bias initialized\n",
"bert/encoder/layer_11/output/dense/kernel initialized\n",
"bert/encoder/layer_11/output/dense/bias initialized\n",
"bert/encoder/layer_11/output/LayerNorm/gamma initialized\n",
"bert/encoder/layer_11/output/LayerNorm/beta initialized\n",
"bert/pooler/dense/kernel initialized\n",
"bert/pooler/dense/bias initialized\n"
]
}
],
"source": [
"from pytorch_pretrained_bert.convert_pytorch_checkpoint_to_tf import main\n",
"\n",
"main([\n",
" '--model_name', model_typ, \n",
" '--pytorch_model_path', pt_init_ckpt,\n",
" '--tf_cache_dir', tf_model_dir,\n",
" '--cache_dir', pt_model_dir\n",
"])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Tensorflow execution"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
"For more information, please see:\n",
" * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
" * https://github.com/tensorflow/addons\n",
"If you depend on functionality not listed there, please file an issue.\n",
"\n",
"WARNING:tensorflow:From /home/ubuntu/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Use keras.layers.dense instead.\n",
"WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/nlp/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Use standard file APIs to check for files with this prefix.\n",
"INFO:tensorflow:Restoring parameters from /home/ubuntu/.pytorch-pretrained-BERT-cache/bert-base-uncased/tf/bert_base_uncased.ckpt\n",
"Tensorflow embedding shape: (1, 768)\n"
]
}
],
"source": [
"import tensorflow as tf\n",
"sys.path.insert(0, tf_bert_dir)\n",
"import modeling\n",
"import tokenization\n",
"\n",
"tf.reset_default_graph()\n",
"\n",
"# Process text\n",
"tf_tokenizer = tokenization.FullTokenizer(vocab_file=tf_vocab_file)\n",
"\n",
"# Graph inputs\n",
"input_ids_tf, mask_ids_tf, seg_ids_tf = tokenize(input_text, tf_tokenizer)\n",
"config = modeling.BertConfig.from_json_file(\n",
" os.path.join(tf_model_dir, 'bert_config.json'))\n",
"input_tensor = tf.placeholder(\n",
" dtype=tf.int32,\n",
" shape=[1, None],\n",
" name='input_ids')\n",
"mask_tensor = tf.placeholder(\n",
" dtype=tf.int32,\n",
" shape=[1, None],\n",
" name='mask_ids')\n",
"seg_tensor = tf.placeholder(\n",
" dtype=tf.int32,\n",
" shape=[1, None],\n",
" name='seg_ids')\n",
"tf_model = modeling.BertModel(\n",
" config=config,\n",
" is_training=False,\n",
" input_ids=input_tensor,\n",
" input_mask=mask_tensor,\n",
" token_type_ids=seg_tensor,\n",
" use_one_hot_embeddings=False)\n",
"output_layer = tf_model.get_pooled_output()\n",
"\n",
"# Load tf model\n",
"session = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))\n",
"vars_to_load = [v for v in tf.global_variables()]\n",
"session.run(tf.variables_initializer(var_list=vars_to_load))\n",
"saver = tf.train.Saver(vars_to_load)\n",
"saver.restore(session, save_path=tf_init_ckpt)\n",
"\n",
"# TF Embedding\n",
"fetches = output_layer\n",
"feed_dict = {\n",
" input_tensor: [input_ids_tf],\n",
" mask_tensor: [mask_ids_tf],\n",
" seg_tensor: [seg_ids_tf]\n",
"}\n",
"tf_embedding = session.run(fetches=fetches, feed_dict=feed_dict)\n",
"print(\"Tensorflow embedding shape: {}\".format(tf_embedding.shape))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Compare Tokenization"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"TOKEN_IDS_PT: [101, 3958, 27227, 2001, 1037, 13997, 11510, 102, 0, 0, 0, 0]\n",
"TOKEN_IDS_TF: [101, 3958, 27227, 2001, 1037, 13997, 11510, 102, 0, 0, 0, 0]\n",
"SEG_IDS_PT: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
"SEG_IDS_TF: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
"MASK_IDS_PT: [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0]\n",
"MASK_IDS_TF: [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0]\n"
]
}
],
"source": [
"print(\"TOKEN_IDS_PT: {}\".format(input_ids_pt))\n",
"print(\"TOKEN_IDS_TF: {}\".format(input_ids_tf))\n",
"print(\"SEG_IDS_PT: {}\".format(seg_ids_pt))\n",
"print(\"SEG_IDS_TF: {}\".format(seg_ids_tf))\n",
"print(\"MASK_IDS_PT: {}\".format(mask_ids_pt))\n",
"print(\"MASK_IDS_TF: {}\".format(mask_ids_tf))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Compare Model Weights"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"bert/embeddings/word_embeddings\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (30522, 768) values: [-0.01018257 -0.06154883 -0.02649689 -0.0420608 0.00116716]\n",
"TF: shape: (30522, 768) values: [-0.01018257 -0.06154883 -0.02649689 -0.0420608 0.00116716]\n",
"\n",
"bert/embeddings/token_type_embeddings\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (2, 768) values: [0.00043164 0.01098826 0.00370439 0.00150542 0.00057812]\n",
"TF: shape: (2, 768) values: [0.00043164 0.01098826 0.00370439 0.00150542 0.00057812]\n",
"\n",
"bert/embeddings/position_embeddings\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (512, 768) values: [ 0.01750538 -0.02563101 -0.03664156 -0.02528613 0.00797095]\n",
"TF: shape: (512, 768) values: [ 0.01750538 -0.02563101 -0.03664156 -0.02528613 0.00797095]\n",
"\n",
"bert/embeddings/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.02591471 -0.0195513 0.02423946 0.08904593 -0.06281059]\n",
"TF: shape: (768,) values: [-0.02591471 -0.0195513 0.02423946 0.08904593 -0.06281059]\n",
"\n",
"bert/embeddings/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.9260566 0.8851115 0.85807985 0.8616906 0.8937205 ]\n",
"TF: shape: (768,) values: [0.9260566 0.8851115 0.85807985 0.8616906 0.8937205 ]\n",
"\n",
"bert/encoder/layer_0/attention/self/query/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.01640572 -0.03257025 0.01046295 -0.04442816 -0.02256124]\n",
"TF: shape: (768, 768) values: [-0.01640572 -0.03257025 0.01046295 -0.04442816 -0.02256124]\n",
"\n",
"bert/encoder/layer_0/attention/self/query/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.58488506 -0.3312432 -0.43010172 0.37446147 -0.29811692]\n",
"TF: shape: (768,) values: [ 0.58488506 -0.3312432 -0.43010172 0.37446147 -0.29811692]\n",
"\n",
"bert/encoder/layer_0/attention/self/key/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.00807745 0.02652155 -0.01866494 0.01797846 0.00450485]\n",
"TF: shape: (768, 768) values: [ 0.00807745 0.02652155 -0.01866494 0.01797846 0.00450485]\n",
"\n",
"bert/encoder/layer_0/attention/self/key/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.00104306 0.00035106 -0.0024626 -0.00010567 -0.00119283]\n",
"TF: shape: (768,) values: [ 0.00104306 0.00035106 -0.0024626 -0.00010567 -0.00119283]\n",
"\n",
"bert/encoder/layer_0/attention/self/value/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.01144261 -0.02663044 0.01911472 -0.02206182 -0.00287949]\n",
"TF: shape: (768, 768) values: [ 0.01144261 -0.02663044 0.01911472 -0.02206182 -0.00287949]\n",
"\n",
"bert/encoder/layer_0/attention/self/value/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.01184616 -0.01596605 -0.00251847 0.01736802 0.00449983]\n",
"TF: shape: (768,) values: [-0.01184616 -0.01596605 -0.00251847 0.01736802 0.00449983]\n",
"\n",
"bert/encoder/layer_0/attention/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.00581949 0.03170148 -0.06135742 -0.01706108 -0.00759045]\n",
"TF: shape: (768, 768) values: [ 0.00581949 0.03170148 -0.06135742 -0.01706108 -0.00759045]\n",
"\n",
"bert/encoder/layer_0/attention/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.00511063 -0.0166625 0.02812938 -0.01166061 0.01942627]\n",
"TF: shape: (768,) values: [ 0.00511063 -0.0166625 0.02812938 -0.01166061 0.01942627]\n",
"\n",
"bert/encoder/layer_0/attention/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.25779155 -0.03077853 -0.2772697 -0.38847703 0.36841765]\n",
"TF: shape: (768,) values: [ 0.25779155 -0.03077853 -0.2772697 -0.38847703 0.36841765]\n",
"\n",
"bert/encoder/layer_0/attention/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.9803408 0.959969 0.96368986 0.9603653 0.9801324 ]\n",
"TF: shape: (768,) values: [0.9803408 0.959969 0.96368986 0.9603653 0.9801324 ]\n",
"\n",
"bert/encoder/layer_0/intermediate/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 3072) values: [-0.01010427 -0.060398 -0.01468864 0.00311493 0.02862451]\n",
"TF: shape: (768, 3072) values: [-0.01010427 -0.060398 -0.01468864 0.00311493 0.02862451]\n",
"\n",
"bert/encoder/layer_0/intermediate/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072,) values: [-0.11498757 -0.09629171 -0.12399033 -0.129036 -0.06369043]\n",
"TF: shape: (3072,) values: [-0.11498757 -0.09629171 -0.12399033 -0.129036 -0.06369043]\n",
"\n",
"bert/encoder/layer_0/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072, 768) values: [-0.03710171 0.0648794 0.00758566 -0.05224452 -0.04348791]\n",
"TF: shape: (3072, 768) values: [-0.03710171 0.0648794 0.00758566 -0.05224452 -0.04348791]\n",
"\n",
"bert/encoder/layer_0/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.04801027 0.19766568 0.02154854 0.02880666 0.0444298 ]\n",
"TF: shape: (768,) values: [-0.04801027 0.19766568 0.02154854 0.02880666 0.0444298 ]\n",
"\n",
"bert/encoder/layer_0/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.10142924 -0.00499344 0.04274083 0.09324206 -0.10700516]\n",
"TF: shape: (768,) values: [-0.10142924 -0.00499344 0.04274083 0.09324206 -0.10700516]\n",
"\n",
"bert/encoder/layer_0/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.7835125 0.8072406 0.7670588 0.73706394 0.76303864]\n",
"TF: shape: (768,) values: [0.7835125 0.8072406 0.7670588 0.73706394 0.76303864]\n",
"\n",
"bert/encoder/layer_1/attention/self/query/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.03132744 -0.01340016 -0.07761582 0.0655639 -0.00337808]\n",
"TF: shape: (768, 768) values: [ 0.03132744 -0.01340016 -0.07761582 0.0655639 -0.00337808]\n",
"\n",
"bert/encoder/layer_1/attention/self/query/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.27827993 0.17387655 -0.2497937 -0.8809636 0.41262135]\n",
"TF: shape: (768,) values: [-0.27827993 0.17387655 -0.2497937 -0.8809636 0.41262135]\n",
"\n",
"bert/encoder/layer_1/attention/self/key/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.03353037 0.04007257 0.05320328 -0.02166729 -0.03581231]\n",
"TF: shape: (768, 768) values: [-0.03353037 0.04007257 0.05320328 -0.02166729 -0.03581231]\n",
"\n",
"bert/encoder/layer_1/attention/self/key/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.00504407 0.00136887 -0.00394336 0.00646125 -0.00148919]\n",
"TF: shape: (768,) values: [-0.00504407 0.00136887 -0.00394336 0.00646125 -0.00148919]\n",
"\n",
"bert/encoder/layer_1/attention/self/value/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.00464159 0.06674305 -0.00970626 -0.0276653 -0.01597566]\n",
"TF: shape: (768, 768) values: [-0.00464159 0.06674305 -0.00970626 -0.0276653 -0.01597566]\n",
"\n",
"bert/encoder/layer_1/attention/self/value/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.00381288 0.02650839 -0.0059689 -0.00508269 -0.01293722]\n",
"TF: shape: (768,) values: [ 0.00381288 0.02650839 -0.0059689 -0.00508269 -0.01293722]\n",
"\n",
"bert/encoder/layer_1/attention/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.01390745 -0.01100563 0.01303005 -0.01969771 0.0125082 ]\n",
"TF: shape: (768, 768) values: [-0.01390745 -0.01100563 0.01303005 -0.01969771 0.0125082 ]\n",
"\n",
"bert/encoder/layer_1/attention/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.02946591 0.05715097 0.01293636 0.01920356 0.00805334]\n",
"TF: shape: (768,) values: [0.02946591 0.05715097 0.01293636 0.01920356 0.00805334]\n",
"\n",
"bert/encoder/layer_1/attention/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.08583715 0.14199966 -0.0856637 -0.18797271 0.21056814]\n",
"TF: shape: (768,) values: [ 0.08583715 0.14199966 -0.0856637 -0.18797271 0.21056814]\n",
"\n",
"bert/encoder/layer_1/attention/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.896962 0.87148863 0.8531161 0.8690647 0.9488987 ]\n",
"TF: shape: (768,) values: [0.896962 0.87148863 0.8531161 0.8690647 0.9488987 ]\n",
"\n",
"bert/encoder/layer_1/intermediate/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 3072) values: [ 0.01841293 -0.02650284 -0.09708428 -0.01734244 -0.05529237]\n",
"TF: shape: (768, 3072) values: [ 0.01841293 -0.02650284 -0.09708428 -0.01734244 -0.05529237]\n",
"\n",
"bert/encoder/layer_1/intermediate/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072,) values: [-0.15203774 -0.10449131 -0.08440229 -0.09323178 -0.08511415]\n",
"TF: shape: (3072,) values: [-0.15203774 -0.10449131 -0.08440229 -0.09323178 -0.08511415]\n",
"\n",
"bert/encoder/layer_1/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072, 768) values: [-0.02372648 0.03326349 0.08291997 -0.01519038 0.01868557]\n",
"TF: shape: (3072, 768) values: [-0.02372648 0.03326349 0.08291997 -0.01519038 0.01868557]\n",
"\n",
"bert/encoder/layer_1/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.02514724 0.09868994 -0.027811 0.03749462 0.01086514]\n",
"TF: shape: (768,) values: [-0.02514724 0.09868994 -0.027811 0.03749462 0.01086514]\n",
"\n",
"bert/encoder/layer_1/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.07662535 -0.10506564 0.03191236 0.07633785 -0.11187791]\n",
"TF: shape: (768,) values: [-0.07662535 -0.10506564 0.03191236 0.07633785 -0.11187791]\n",
"\n",
"bert/encoder/layer_1/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.9017883 0.8868776 0.8862677 0.85865664 0.87496454]\n",
"TF: shape: (768,) values: [0.9017883 0.8868776 0.8862677 0.85865664 0.87496454]\n",
"\n",
"bert/encoder/layer_2/attention/self/query/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.08433672 0.09580533 0.07543895 -0.01126779 -0.01354045]\n",
"TF: shape: (768, 768) values: [ 0.08433672 0.09580533 0.07543895 -0.01126779 -0.01354045]\n",
"\n",
"bert/encoder/layer_2/attention/self/query/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.0371241 0.03406003 0.27713948 -0.21613775 -0.05275448]\n",
"TF: shape: (768,) values: [ 0.0371241 0.03406003 0.27713948 -0.21613775 -0.05275448]\n",
"\n",
"bert/encoder/layer_2/attention/self/key/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.04794507 0.02517631 -0.01319554 -0.02094732 0.09073472]\n",
"TF: shape: (768, 768) values: [ 0.04794507 0.02517631 -0.01319554 -0.02094732 0.09073472]\n",
"\n",
"bert/encoder/layer_2/attention/self/key/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.00037404 -0.00125881 -0.00114734 -0.00157741 0.00037122]\n",
"TF: shape: (768,) values: [-0.00037404 -0.00125881 -0.00114734 -0.00157741 0.00037122]\n",
"\n",
"bert/encoder/layer_2/attention/self/value/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.01119406 -0.01488636 -0.02960914 0.04746444 0.00428481]\n",
"TF: shape: (768, 768) values: [-0.01119406 -0.01488636 -0.02960914 0.04746444 0.00428481]\n",
"\n",
"bert/encoder/layer_2/attention/self/value/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.02728729 0.04979054 0.08326469 0.04150949 0.600959 ]\n",
"TF: shape: (768,) values: [-0.02728729 0.04979054 0.08326469 0.04150949 0.600959 ]\n",
"\n",
"bert/encoder/layer_2/attention/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.00517425 0.01197957 0.0393172 -0.0063884 -0.02673388]\n",
"TF: shape: (768, 768) values: [ 0.00517425 0.01197957 0.0393172 -0.0063884 -0.02673388]\n",
"\n",
"bert/encoder/layer_2/attention/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.01754025 0.1226335 -0.05733554 0.06844623 0.00879776]\n",
"TF: shape: (768,) values: [ 0.01754025 0.1226335 -0.05733554 0.06844623 0.00879776]\n",
"\n",
"bert/encoder/layer_2/attention/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.1490809 0.12386955 -0.19382021 -0.26515856 0.32723007]\n",
"TF: shape: (768,) values: [ 0.1490809 0.12386955 -0.19382021 -0.26515856 0.32723007]\n",
"\n",
"bert/encoder/layer_2/attention/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8983343 0.88877076 0.86283594 0.8584952 0.9587886 ]\n",
"TF: shape: (768,) values: [0.8983343 0.88877076 0.86283594 0.8584952 0.9587886 ]\n",
"\n",
"bert/encoder/layer_2/intermediate/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 3072) values: [-0.01619919 0.00662888 0.01492284 -0.01280748 0.01318596]\n",
"TF: shape: (768, 3072) values: [-0.01619919 0.00662888 0.01492284 -0.01280748 0.01318596]\n",
"\n",
"bert/encoder/layer_2/intermediate/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072,) values: [-0.08474881 -0.12850781 -0.11550345 -0.09513011 -0.02519853]\n",
"TF: shape: (3072,) values: [-0.08474881 -0.12850781 -0.11550345 -0.09513011 -0.02519853]\n",
"\n",
"bert/encoder/layer_2/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072, 768) values: [-0.07225161 -0.0129784 0.00618811 -0.01593373 -0.02160194]\n",
"TF: shape: (3072, 768) values: [-0.07225161 -0.0129784 0.00618811 -0.01593373 -0.02160194]\n",
"\n",
"bert/encoder/layer_2/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.06319264 0.06169628 -0.03041368 0.00924282 0.06277442]\n",
"TF: shape: (768,) values: [-0.06319264 0.06169628 -0.03041368 0.00924282 0.06277442]\n",
"\n",
"bert/encoder/layer_2/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.1139038 -0.11665309 0.07883061 0.07796711 -0.14219187]\n",
"TF: shape: (768,) values: [-0.1139038 -0.11665309 0.07883061 0.07796711 -0.14219187]\n",
"\n",
"bert/encoder/layer_2/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8813261 0.85744697 0.8511922 0.85261875 0.8329574 ]\n",
"TF: shape: (768,) values: [0.8813261 0.85744697 0.8511922 0.85261875 0.8329574 ]\n",
"\n",
"bert/encoder/layer_3/attention/self/query/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.05855456 -0.00111438 -0.00828963 0.04117409 -0.07591715]\n",
"TF: shape: (768, 768) values: [ 0.05855456 -0.00111438 -0.00828963 0.04117409 -0.07591715]\n",
"\n",
"bert/encoder/layer_3/attention/self/query/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.09740101 -0.19290674 0.04332267 0.17937997 -0.08023558]\n",
"TF: shape: (768,) values: [ 0.09740101 -0.19290674 0.04332267 0.17937997 -0.08023558]\n",
"\n",
"bert/encoder/layer_3/attention/self/key/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.02562077 0.02507281 -0.03361562 0.05613289 -0.05435724]\n",
"TF: shape: (768, 768) values: [ 0.02562077 0.02507281 -0.03361562 0.05613289 -0.05435724]\n",
"\n",
"bert/encoder/layer_3/attention/self/key/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.00188639 -0.00379197 -0.01020415 0.00969649 -0.00094182]\n",
"TF: shape: (768,) values: [ 0.00188639 -0.00379197 -0.01020415 0.00969649 -0.00094182]\n",
"\n",
"bert/encoder/layer_3/attention/self/value/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.00539032 0.00959642 0.01325458 0.00490616 0.0129908 ]\n",
"TF: shape: (768, 768) values: [-0.00539032 0.00959642 0.01325458 0.00490616 0.0129908 ]\n",
"\n",
"bert/encoder/layer_3/attention/self/value/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.04573824 0.05405985 0.00681163 0.00655945 0.01141771]\n",
"TF: shape: (768,) values: [0.04573824 0.05405985 0.00681163 0.00655945 0.01141771]\n",
"\n",
"bert/encoder/layer_3/attention/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.01850341 0.03148198 0.02705758 -0.0004669 0.01367511]\n",
"TF: shape: (768, 768) values: [ 0.01850341 0.03148198 0.02705758 -0.0004669 0.01367511]\n",
"\n",
"bert/encoder/layer_3/attention/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.01981483 0.03566506 -0.05016088 0.02958186 0.04989756]\n",
"TF: shape: (768,) values: [ 0.01981483 0.03566506 -0.05016088 0.02958186 0.04989756]\n",
"\n",
"bert/encoder/layer_3/attention/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.09815404 0.00063774 -0.01257733 -0.26485074 0.22568701]\n",
"TF: shape: (768,) values: [ 0.09815404 0.00063774 -0.01257733 -0.26485074 0.22568701]\n",
"\n",
"bert/encoder/layer_3/attention/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.91457725 0.88453823 0.8340887 0.84203583 0.95247847]\n",
"TF: shape: (768,) values: [0.91457725 0.88453823 0.8340887 0.84203583 0.95247847]\n",
"\n",
"bert/encoder/layer_3/intermediate/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 3072) values: [-0.02733567 0.03307878 -0.01331292 -0.00032527 0.03252084]\n",
"TF: shape: (768, 3072) values: [-0.02733567 0.03307878 -0.01331292 -0.00032527 0.03252084]\n",
"\n",
"bert/encoder/layer_3/intermediate/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072,) values: [-0.11436842 -0.15038085 -0.07842971 0.01335877 -0.09492484]\n",
"TF: shape: (3072,) values: [-0.11436842 -0.15038085 -0.07842971 0.01335877 -0.09492484]\n",
"\n",
"bert/encoder/layer_3/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072, 768) values: [-0.01751153 0.01631314 -0.02660011 0.03569947 -0.01394763]\n",
"TF: shape: (3072, 768) values: [-0.01751153 0.01631314 -0.02660011 0.03569947 -0.01394763]\n",
"\n",
"bert/encoder/layer_3/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.03873252 0.08414765 -0.0399323 0.01997361 0.12924597]\n",
"TF: shape: (768,) values: [-0.03873252 0.08414765 -0.0399323 0.01997361 0.12924597]\n",
"\n",
"bert/encoder/layer_3/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.08049371 -0.06923949 -0.03357155 0.05231095 -0.09717073]\n",
"TF: shape: (768,) values: [-0.08049371 -0.06923949 -0.03357155 0.05231095 -0.09717073]\n",
"\n",
"bert/encoder/layer_3/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.827748 0.83012533 0.82399255 0.81772 0.80794513]\n",
"TF: shape: (768,) values: [0.827748 0.83012533 0.82399255 0.81772 0.80794513]\n",
"\n",
"bert/encoder/layer_4/attention/self/query/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.08296382 0.02076941 0.06525186 -0.02659729 0.03491377]\n",
"TF: shape: (768, 768) values: [ 0.08296382 0.02076941 0.06525186 -0.02659729 0.03491377]\n",
"\n",
"bert/encoder/layer_4/attention/self/query/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.07045844 -0.13412629 -0.0514146 0.00061329 0.1248519 ]\n",
"TF: shape: (768,) values: [ 0.07045844 -0.13412629 -0.0514146 0.00061329 0.1248519 ]\n",
"\n",
"bert/encoder/layer_4/attention/self/key/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.06941643 0.08133814 -0.0453992 0.0668715 -0.06014847]\n",
"TF: shape: (768, 768) values: [ 0.06941643 0.08133814 -0.0453992 0.0668715 -0.06014847]\n",
"\n",
"bert/encoder/layer_4/attention/self/key/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.00588725 -0.00235185 0.00281131 0.00173088 -0.00546653]\n",
"TF: shape: (768,) values: [-0.00588725 -0.00235185 0.00281131 0.00173088 -0.00546653]\n",
"\n",
"bert/encoder/layer_4/attention/self/value/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.06889665 0.06645385 0.01232084 0.0132611 -0.01595679]\n",
"TF: shape: (768, 768) values: [ 0.06889665 0.06645385 0.01232084 0.0132611 -0.01595679]\n",
"\n",
"bert/encoder/layer_4/attention/self/value/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.01126871 -0.02704018 0.0301532 0.02332082 -0.04233487]\n",
"TF: shape: (768,) values: [-0.01126871 -0.02704018 0.0301532 0.02332082 -0.04233487]\n",
"\n",
"bert/encoder/layer_4/attention/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.02285513 -0.04172142 -0.0146292 0.04862929 -0.0442014 ]\n",
"TF: shape: (768, 768) values: [ 0.02285513 -0.04172142 -0.0146292 0.04862929 -0.0442014 ]\n",
"\n",
"bert/encoder/layer_4/attention/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.03054528 0.00479777 -0.02729505 -0.0325212 -0.00525727]\n",
"TF: shape: (768,) values: [ 0.03054528 0.00479777 -0.02729505 -0.0325212 -0.00525727]\n",
"\n",
"bert/encoder/layer_4/attention/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.00903359 0.0052285 -0.02841488 -0.22355485 0.28281343]\n",
"TF: shape: (768,) values: [ 0.00903359 0.0052285 -0.02841488 -0.22355485 0.28281343]\n",
"\n",
"bert/encoder/layer_4/attention/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8849676 0.86927813 0.8114595 0.80269504 0.94864094]\n",
"TF: shape: (768,) values: [0.8849676 0.86927813 0.8114595 0.80269504 0.94864094]\n",
"\n",
"bert/encoder/layer_4/intermediate/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 3072) values: [-0.00639783 0.06198016 -0.03184223 0.00485356 -0.02453273]\n",
"TF: shape: (768, 3072) values: [-0.00639783 0.06198016 -0.03184223 0.00485356 -0.02453273]\n",
"\n",
"bert/encoder/layer_4/intermediate/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072,) values: [-0.08770327 -0.11779705 -0.11764182 -0.00192611 -0.1335473 ]\n",
"TF: shape: (3072,) values: [-0.08770327 -0.11779705 -0.11764182 -0.00192611 -0.1335473 ]\n",
"\n",
"bert/encoder/layer_4/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072, 768) values: [-0.05421264 0.0221118 -0.02674172 0.03672203 -0.02399626]\n",
"TF: shape: (3072, 768) values: [-0.05421264 0.0221118 -0.02674172 0.03672203 -0.02399626]\n",
"\n",
"bert/encoder/layer_4/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.05068972 0.04838871 0.01156022 0.05381602 0.08857913]\n",
"TF: shape: (768,) values: [-0.05068972 0.04838871 0.01156022 0.05381602 0.08857913]\n",
"\n",
"bert/encoder/layer_4/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.04338909 -0.0781464 -0.01518662 0.04936362 -0.12378412]\n",
"TF: shape: (768,) values: [-0.04338909 -0.0781464 -0.01518662 0.04936362 -0.12378412]\n",
"\n",
"bert/encoder/layer_4/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8734387 0.8576282 0.8339444 0.8450325 0.8105372]\n",
"TF: shape: (768,) values: [0.8734387 0.8576282 0.8339444 0.8450325 0.8105372]\n",
"\n",
"bert/encoder/layer_5/attention/self/query/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.00858843 -0.03920127 0.02552994 -0.02786552 0.02436485]\n",
"TF: shape: (768, 768) values: [-0.00858843 -0.03920127 0.02552994 -0.02786552 0.02436485]\n",
"\n",
"bert/encoder/layer_5/attention/self/query/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.00859117 -0.01642405 -0.04391079 0.01085692 0.02925887]\n",
"TF: shape: (768,) values: [-0.00859117 -0.01642405 -0.04391079 0.01085692 0.02925887]\n",
"\n",
"bert/encoder/layer_5/attention/self/key/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.00352847 0.02330176 -0.00369894 -0.03904612 0.00294574]\n",
"TF: shape: (768, 768) values: [ 0.00352847 0.02330176 -0.00369894 -0.03904612 0.00294574]\n",
"\n",
"bert/encoder/layer_5/attention/self/key/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.01087186 -0.01176561 0.00016575 -0.01163023 0.00946616]\n",
"TF: shape: (768,) values: [-0.01087186 -0.01176561 0.00016575 -0.01163023 0.00946616]\n",
"\n",
"bert/encoder/layer_5/attention/self/value/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.06134222 0.04238288 0.02796064 -0.01284983 0.03683741]\n",
"TF: shape: (768, 768) values: [ 0.06134222 0.04238288 0.02796064 -0.01284983 0.03683741]\n",
"\n",
"bert/encoder/layer_5/attention/self/value/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.05061118 -0.02954445 -0.0034053 -0.00025261 0.0437019 ]\n",
"TF: shape: (768,) values: [ 0.05061118 -0.02954445 -0.0034053 -0.00025261 0.0437019 ]\n",
"\n",
"bert/encoder/layer_5/attention/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.00739815 0.0533964 -0.03736389 -0.04999201 0.01693069]\n",
"TF: shape: (768, 768) values: [-0.00739815 0.0533964 -0.03736389 -0.04999201 0.01693069]\n",
"\n",
"bert/encoder/layer_5/attention/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.0021682 0.01711399 -0.04201518 0.01605333 0.00552063]\n",
"TF: shape: (768,) values: [-0.0021682 0.01711399 -0.04201518 0.01605333 0.00552063]\n",
"\n",
"bert/encoder/layer_5/attention/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.06841327 -0.0146848 0.09792476 -0.23284538 0.2785602 ]\n",
"TF: shape: (768,) values: [-0.06841327 -0.0146848 0.09792476 -0.23284538 0.2785602 ]\n",
"\n",
"bert/encoder/layer_5/attention/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8908311 0.87884724 0.81637293 0.8047641 0.96539867]\n",
"TF: shape: (768,) values: [0.8908311 0.87884724 0.81637293 0.8047641 0.96539867]\n",
"\n",
"bert/encoder/layer_5/intermediate/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 3072) values: [-0.03246041 0.07251058 -0.08201726 0.00772481 0.02532209]\n",
"TF: shape: (768, 3072) values: [-0.03246041 0.07251058 -0.08201726 0.00772481 0.02532209]\n",
"\n",
"bert/encoder/layer_5/intermediate/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072,) values: [-0.09689714 -0.27696273 -0.13047501 -0.10892326 -0.1057625 ]\n",
"TF: shape: (3072,) values: [-0.09689714 -0.27696273 -0.13047501 -0.10892326 -0.1057625 ]\n",
"\n",
"bert/encoder/layer_5/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072, 768) values: [ 0.0642072 -0.01738782 -0.05095377 0.00523853 0.04425264]\n",
"TF: shape: (3072, 768) values: [ 0.0642072 -0.01738782 -0.05095377 0.00523853 0.04425264]\n",
"\n",
"bert/encoder/layer_5/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.0007217 0.06006297 0.0016595 0.03848181 0.06703516]\n",
"TF: shape: (768,) values: [-0.0007217 0.06006297 0.0016595 0.03848181 0.06703516]\n",
"\n",
"bert/encoder/layer_5/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.00278729 -0.05594506 -0.0631047 0.06023621 -0.18672828]\n",
"TF: shape: (768,) values: [-0.00278729 -0.05594506 -0.0631047 0.06023621 -0.18672828]\n",
"\n",
"bert/encoder/layer_5/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8621183 0.8515807 0.82654256 0.81729776 0.7985204 ]\n",
"TF: shape: (768,) values: [0.8621183 0.8515807 0.82654256 0.81729776 0.7985204 ]\n",
"\n",
"bert/encoder/layer_6/attention/self/query/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.02527807 -0.01429243 0.01467054 0.08624706 -0.00188593]\n",
"TF: shape: (768, 768) values: [-0.02527807 -0.01429243 0.01467054 0.08624706 -0.00188593]\n",
"\n",
"bert/encoder/layer_6/attention/self/query/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.17319514 0.27564248 0.16801168 -0.10946485 0.1643271 ]\n",
"TF: shape: (768,) values: [-0.17319514 0.27564248 0.16801168 -0.10946485 0.1643271 ]\n",
"\n",
"bert/encoder/layer_6/attention/self/key/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.05886372 0.00706217 0.0398422 0.00882155 -0.04571463]\n",
"TF: shape: (768, 768) values: [ 0.05886372 0.00706217 0.0398422 0.00882155 -0.04571463]\n",
"\n",
"bert/encoder/layer_6/attention/self/key/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.00424696 -0.0001192 0.0046079 -0.00315606 0.00434314]\n",
"TF: shape: (768,) values: [-0.00424696 -0.0001192 0.0046079 -0.00315606 0.00434314]\n",
"\n",
"bert/encoder/layer_6/attention/self/value/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.01720381 0.01170722 0.02346902 -0.02284313 -0.03173028]\n",
"TF: shape: (768, 768) values: [-0.01720381 0.01170722 0.02346902 -0.02284313 -0.03173028]\n",
"\n",
"bert/encoder/layer_6/attention/self/value/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.03492057 0.01813157 -0.00182878 -0.01420629 -0.00508944]\n",
"TF: shape: (768,) values: [-0.03492057 0.01813157 -0.00182878 -0.01420629 -0.00508944]\n",
"\n",
"bert/encoder/layer_6/attention/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.0323688 -0.00689882 0.07379091 0.01121114 -0.02059202]\n",
"TF: shape: (768, 768) values: [ 0.0323688 -0.00689882 0.07379091 0.01121114 -0.02059202]\n",
"\n",
"bert/encoder/layer_6/attention/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.00648672 -0.05935453 -0.05673229 -0.01152384 -0.02766573]\n",
"TF: shape: (768,) values: [-0.00648672 -0.05935453 -0.05673229 -0.01152384 -0.02766573]\n",
"\n",
"bert/encoder/layer_6/attention/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.06793639 0.03157783 0.15647687 -0.15025291 0.14727171]\n",
"TF: shape: (768,) values: [-0.06793639 0.03157783 0.15647687 -0.15025291 0.14727171]\n",
"\n",
"bert/encoder/layer_6/attention/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8882361 0.8704905 0.80289173 0.77365315 0.92333615]\n",
"TF: shape: (768,) values: [0.8882361 0.8704905 0.80289173 0.77365315 0.92333615]\n",
"\n",
"bert/encoder/layer_6/intermediate/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 3072) values: [ 0.04492201 0.05160861 0.09041415 -0.00742628 0.048133 ]\n",
"TF: shape: (768, 3072) values: [ 0.04492201 0.05160861 0.09041415 -0.00742628 0.048133 ]\n",
"\n",
"bert/encoder/layer_6/intermediate/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072,) values: [-0.09301704 -0.158612 -0.10633879 -0.09706812 -0.17319229]\n",
"TF: shape: (3072,) values: [-0.09301704 -0.158612 -0.10633879 -0.09706812 -0.17319229]\n",
"\n",
"bert/encoder/layer_6/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072, 768) values: [-0.00085372 -0.00974195 0.00684915 0.00038686 0.06610142]\n",
"TF: shape: (3072, 768) values: [-0.00085372 -0.00974195 0.00684915 0.00038686 0.06610142]\n",
"\n",
"bert/encoder/layer_6/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.03254414 0.05681704 0.03720434 0.01936359 0.09134153]\n",
"TF: shape: (768,) values: [-0.03254414 0.05681704 0.03720434 0.01936359 0.09134153]\n",
"\n",
"bert/encoder/layer_6/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.0117129 -0.03209404 -0.08646043 0.03760341 -0.13841423]\n",
"TF: shape: (768,) values: [-0.0117129 -0.03209404 -0.08646043 0.03760341 -0.13841423]\n",
"\n",
"bert/encoder/layer_6/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8674175 0.8657014 0.8151861 0.82301307 0.8305737 ]\n",
"TF: shape: (768,) values: [0.8674175 0.8657014 0.8151861 0.82301307 0.8305737 ]\n",
"\n",
"bert/encoder/layer_7/attention/self/query/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.00075523 -0.01501983 0.04090893 0.01884826 0.04670674]\n",
"TF: shape: (768, 768) values: [-0.00075523 -0.01501983 0.04090893 0.01884826 0.04670674]\n",
"\n",
"bert/encoder/layer_7/attention/self/query/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.0010344 -0.00423982 0.3117479 0.04494623 -0.01260845]\n",
"TF: shape: (768,) values: [ 0.0010344 -0.00423982 0.3117479 0.04494623 -0.01260845]\n",
"\n",
"bert/encoder/layer_7/attention/self/key/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.02781927 -0.00906972 0.02121989 0.0298591 0.05854786]\n",
"TF: shape: (768, 768) values: [ 0.02781927 -0.00906972 0.02121989 0.0298591 0.05854786]\n",
"\n",
"bert/encoder/layer_7/attention/self/key/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.00074918 0.00731079 0.00089338 0.00345652 0.00043817]\n",
"TF: shape: (768,) values: [-0.00074918 0.00731079 0.00089338 0.00345652 0.00043817]\n",
"\n",
"bert/encoder/layer_7/attention/self/value/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.01080035 -0.03468366 0.03167168 0.01583073 0.0327719 ]\n",
"TF: shape: (768, 768) values: [-0.01080035 -0.03468366 0.03167168 0.01583073 0.0327719 ]\n",
"\n",
"bert/encoder/layer_7/attention/self/value/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.02824226 0.01605172 0.00067929 -0.04553111 0.0076044 ]\n",
"TF: shape: (768,) values: [-0.02824226 0.01605172 0.00067929 -0.04553111 0.0076044 ]\n",
"\n",
"bert/encoder/layer_7/attention/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.05496112 0.01006968 0.02206531 -0.01873116 0.02149118]\n",
"TF: shape: (768, 768) values: [-0.05496112 0.01006968 0.02206531 -0.01873116 0.02149118]\n",
"\n",
"bert/encoder/layer_7/attention/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.00349772 -0.05831751 -0.0594084 -0.0342187 0.02965918]\n",
"TF: shape: (768,) values: [ 0.00349772 -0.05831751 -0.0594084 -0.0342187 0.02965918]\n",
"\n",
"bert/encoder/layer_7/attention/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.02826844 0.04427591 0.05678326 -0.0475907 0.16136196]\n",
"TF: shape: (768,) values: [-0.02826844 0.04427591 0.05678326 -0.0475907 0.16136196]\n",
"\n",
"bert/encoder/layer_7/attention/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8742141 0.870608 0.79147685 0.7595279 0.9223656 ]\n",
"TF: shape: (768,) values: [0.8742141 0.870608 0.79147685 0.7595279 0.9223656 ]\n",
"\n",
"bert/encoder/layer_7/intermediate/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 3072) values: [ 0.03598932 -0.12225644 0.03019998 0.05691092 0.03717208]\n",
"TF: shape: (768, 3072) values: [ 0.03598932 -0.12225644 0.03019998 0.05691092 0.03717208]\n",
"\n",
"bert/encoder/layer_7/intermediate/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072,) values: [-0.12465011 -0.08639494 -0.06206005 -0.08012587 -0.08773767]\n",
"TF: shape: (3072,) values: [-0.12465011 -0.08639494 -0.06206005 -0.08012587 -0.08773767]\n",
"\n",
"bert/encoder/layer_7/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072, 768) values: [-0.02190432 -0.02279165 0.03279508 0.01011065 -0.07793335]\n",
"TF: shape: (3072, 768) values: [-0.02190432 -0.02279165 0.03279508 0.01011065 -0.07793335]\n",
"\n",
"bert/encoder/layer_7/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.04282642 0.03700675 0.06142357 -0.04787201 0.02958163]\n",
"TF: shape: (768,) values: [-0.04282642 0.03700675 0.06142357 -0.04787201 0.02958163]\n",
"\n",
"bert/encoder/layer_7/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.03142036 -0.04358427 -0.05132087 -0.01788123 -0.16399944]\n",
"TF: shape: (768,) values: [-0.03142036 -0.04358427 -0.05132087 -0.01788123 -0.16399944]\n",
"\n",
"bert/encoder/layer_7/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.83858097 0.8179645 0.80693793 0.81225365 0.7844832 ]\n",
"TF: shape: (768,) values: [0.83858097 0.8179645 0.80693793 0.81225365 0.7844832 ]\n",
"\n",
"bert/encoder/layer_8/attention/self/query/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [0.0448719 0.02289526 0.03083764 0.03048073 0.02436891]\n",
"TF: shape: (768, 768) values: [0.0448719 0.02289526 0.03083764 0.03048073 0.02436891]\n",
"\n",
"bert/encoder/layer_8/attention/self/query/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.25132924 -0.23753347 0.02581017 0.00901509 0.18424493]\n",
"TF: shape: (768,) values: [-0.25132924 -0.23753347 0.02581017 0.00901509 0.18424493]\n",
"\n",
"bert/encoder/layer_8/attention/self/key/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.01999719 0.00711403 0.03949134 -0.0102224 0.03152475]\n",
"TF: shape: (768, 768) values: [-0.01999719 0.00711403 0.03949134 -0.0102224 0.03152475]\n",
"\n",
"bert/encoder/layer_8/attention/self/key/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 5.5668897e-05 3.4638541e-03 -1.7605867e-03 -6.1321147e-03\n",
" -4.4074579e-04]\n",
"TF: shape: (768,) values: [ 5.5668897e-05 3.4638541e-03 -1.7605867e-03 -6.1321147e-03\n",
" -4.4074579e-04]\n",
"\n",
"bert/encoder/layer_8/attention/self/value/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.00736056 -0.01795213 0.00104576 -0.00034653 0.03190543]\n",
"TF: shape: (768, 768) values: [-0.00736056 -0.01795213 0.00104576 -0.00034653 0.03190543]\n",
"\n",
"bert/encoder/layer_8/attention/self/value/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.02892835 0.00642501 -0.03608712 0.00264269 -0.0245198 ]\n",
"TF: shape: (768,) values: [ 0.02892835 0.00642501 -0.03608712 0.00264269 -0.0245198 ]\n",
"\n",
"bert/encoder/layer_8/attention/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.03971623 0.05307067 -0.01298818 0.00946693 -0.00121235]\n",
"TF: shape: (768, 768) values: [ 0.03971623 0.05307067 -0.01298818 0.00946693 -0.00121235]\n",
"\n",
"bert/encoder/layer_8/attention/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.01468131 -0.05406622 -0.06289103 0.004484 0.0240819 ]\n",
"TF: shape: (768,) values: [ 0.01468131 -0.05406622 -0.06289103 0.004484 0.0240819 ]\n",
"\n",
"bert/encoder/layer_8/attention/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.06004262 0.0457275 0.08688109 -0.14416659 -0.05500487]\n",
"TF: shape: (768,) values: [-0.06004262 0.0457275 0.08688109 -0.14416659 -0.05500487]\n",
"\n",
"bert/encoder/layer_8/attention/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8907534 0.89116573 0.811639 0.7810443 0.9045574 ]\n",
"TF: shape: (768,) values: [0.8907534 0.89116573 0.811639 0.7810443 0.9045574 ]\n",
"\n",
"bert/encoder/layer_8/intermediate/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 3072) values: [-0.01962814 -0.01482586 -0.02292624 0.03397145 0.02457482]\n",
"TF: shape: (768, 3072) values: [-0.01962814 -0.01482586 -0.02292624 0.03397145 0.02457482]\n",
"\n",
"bert/encoder/layer_8/intermediate/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072,) values: [-0.08129632 -0.1691108 -0.10681771 -0.10392351 -0.13120006]\n",
"TF: shape: (3072,) values: [-0.08129632 -0.1691108 -0.10681771 -0.10392351 -0.13120006]\n",
"\n",
"bert/encoder/layer_8/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072, 768) values: [-0.04683433 -0.02690669 0.02979059 0.02223369 -0.00130287]\n",
"TF: shape: (3072, 768) values: [-0.04683433 -0.02690669 0.02979059 0.02223369 -0.00130287]\n",
"\n",
"bert/encoder/layer_8/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.09155537 -0.04465394 0.05649116 -0.09628641 0.11875238]\n",
"TF: shape: (768,) values: [-0.09155537 -0.04465394 0.05649116 -0.09628641 0.11875238]\n",
"\n",
"bert/encoder/layer_8/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.06043394 -0.06657387 -0.05341128 -0.00374733 -0.10855272]\n",
"TF: shape: (768,) values: [-0.06043394 -0.06657387 -0.05341128 -0.00374733 -0.10855272]\n",
"\n",
"bert/encoder/layer_8/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.84467345 0.84421015 0.82582206 0.84553087 0.8207573 ]\n",
"TF: shape: (768,) values: [0.84467345 0.84421015 0.82582206 0.84553087 0.8207573 ]\n",
"\n",
"bert/encoder/layer_9/attention/self/query/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.08004542 -0.0143706 -0.04219061 -0.05175152 -0.01147588]\n",
"TF: shape: (768, 768) values: [ 0.08004542 -0.0143706 -0.04219061 -0.05175152 -0.01147588]\n",
"\n",
"bert/encoder/layer_9/attention/self/query/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.14508031 0.40926442 -0.3281781 -0.02869792 -0.26104516]\n",
"TF: shape: (768,) values: [-0.14508031 0.40926442 -0.3281781 -0.02869792 -0.26104516]\n",
"\n",
"bert/encoder/layer_9/attention/self/key/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.01337681 0.00615428 -0.0455939 0.03379053 -0.01992556]\n",
"TF: shape: (768, 768) values: [-0.01337681 0.00615428 -0.0455939 0.03379053 -0.01992556]\n",
"\n",
"bert/encoder/layer_9/attention/self/key/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.0051302 0.0083288 0.00377641 0.00928865 -0.00418182]\n",
"TF: shape: (768,) values: [-0.0051302 0.0083288 0.00377641 0.00928865 -0.00418182]\n",
"\n",
"bert/encoder/layer_9/attention/self/value/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.02485976 -0.0301923 0.00984638 -0.02495162 0.01074037]\n",
"TF: shape: (768, 768) values: [-0.02485976 -0.0301923 0.00984638 -0.02495162 0.01074037]\n",
"\n",
"bert/encoder/layer_9/attention/self/value/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.04229928 -0.02636711 0.0060447 0.00222829 0.04979481]\n",
"TF: shape: (768,) values: [-0.04229928 -0.02636711 0.0060447 0.00222829 0.04979481]\n",
"\n",
"bert/encoder/layer_9/attention/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.01258144 0.00871274 0.00482882 -0.00675888 -0.04390825]\n",
"TF: shape: (768, 768) values: [-0.01258144 0.00871274 0.00482882 -0.00675888 -0.04390825]\n",
"\n",
"bert/encoder/layer_9/attention/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.02457753 0.05051134 -0.06890804 -0.00962795 0.00864793]\n",
"TF: shape: (768,) values: [ 0.02457753 0.05051134 -0.06890804 -0.00962795 0.00864793]\n",
"\n",
"bert/encoder/layer_9/attention/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.08963391 -0.06362236 0.0676669 -0.09895685 0.08318913]\n",
"TF: shape: (768,) values: [-0.08963391 -0.06362236 0.0676669 -0.09895685 0.08318913]\n",
"\n",
"bert/encoder/layer_9/attention/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.85100883 0.82569736 0.7927931 0.7660444 0.8912934 ]\n",
"TF: shape: (768,) values: [0.85100883 0.82569736 0.7927931 0.7660444 0.8912934 ]\n",
"\n",
"bert/encoder/layer_9/intermediate/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 3072) values: [ 0.06290598 0.0203122 -0.05384256 0.05442941 0.00484769]\n",
"TF: shape: (768, 3072) values: [ 0.06290598 0.0203122 -0.05384256 0.05442941 0.00484769]\n",
"\n",
"bert/encoder/layer_9/intermediate/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072,) values: [-0.10818483 -0.00169527 -0.08962701 -0.10280421 -0.14310956]\n",
"TF: shape: (3072,) values: [-0.10818483 -0.00169527 -0.08962701 -0.10280421 -0.14310956]\n",
"\n",
"bert/encoder/layer_9/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072, 768) values: [ 0.05487705 0.01644666 0.00436198 -0.00490768 -0.03238423]\n",
"TF: shape: (3072, 768) values: [ 0.05487705 0.01644666 0.00436198 -0.00490768 -0.03238423]\n",
"\n",
"bert/encoder/layer_9/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.08755219 -0.01910074 -0.02988298 -0.08150438 0.09897955]\n",
"TF: shape: (768,) values: [-0.08755219 -0.01910074 -0.02988298 -0.08150438 0.09897955]\n",
"\n",
"bert/encoder/layer_9/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.04136161 -0.02113917 -0.07581077 -0.00809791 -0.09790538]\n",
"TF: shape: (768,) values: [-0.04136161 -0.02113917 -0.07581077 -0.00809791 -0.09790538]\n",
"\n",
"bert/encoder/layer_9/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8250572 0.83477134 0.7794141 0.81264955 0.7827918 ]\n",
"TF: shape: (768,) values: [0.8250572 0.83477134 0.7794141 0.81264955 0.7827918 ]\n",
"\n",
"bert/encoder/layer_10/attention/self/query/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.00071212 -0.00853064 0.01776993 0.03189976 0.02183623]\n",
"TF: shape: (768, 768) values: [ 0.00071212 -0.00853064 0.01776993 0.03189976 0.02183623]\n",
"\n",
"bert/encoder/layer_10/attention/self/query/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.03667567 -0.01449654 -0.03822913 0.00118343 -0.05489838]\n",
"TF: shape: (768,) values: [-0.03667567 -0.01449654 -0.03822913 0.00118343 -0.05489838]\n",
"\n",
"bert/encoder/layer_10/attention/self/key/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.0494106 0.05531096 -0.02459413 -0.06019118 -0.02829785]\n",
"TF: shape: (768, 768) values: [-0.0494106 0.05531096 -0.02459413 -0.06019118 -0.02829785]\n",
"\n",
"bert/encoder/layer_10/attention/self/key/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.00692997 0.00855893 0.00670777 -0.0052475 -0.00017074]\n",
"TF: shape: (768,) values: [-0.00692997 0.00855893 0.00670777 -0.0052475 -0.00017074]\n",
"\n",
"bert/encoder/layer_10/attention/self/value/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.01911842 0.04858809 -0.02608485 0.00794924 -0.02246636]\n",
"TF: shape: (768, 768) values: [ 0.01911842 0.04858809 -0.02608485 0.00794924 -0.02246636]\n",
"\n",
"bert/encoder/layer_10/attention/self/value/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.0133503 -0.01224133 -0.0051834 -0.00232528 0.00148614]\n",
"TF: shape: (768,) values: [-0.0133503 -0.01224133 -0.0051834 -0.00232528 0.00148614]\n",
"\n",
"bert/encoder/layer_10/attention/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.05904732 0.02616 0.00794104 -0.02889086 -0.03692576]\n",
"TF: shape: (768, 768) values: [-0.05904732 0.02616 0.00794104 -0.02889086 -0.03692576]\n",
"\n",
"bert/encoder/layer_10/attention/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.02089205 0.01458059 0.05217785 0.0324267 0.00907548]\n",
"TF: shape: (768,) values: [0.02089205 0.01458059 0.05217785 0.0324267 0.00907548]\n",
"\n",
"bert/encoder/layer_10/attention/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.10986238 -0.04332284 0.02603893 -0.06236923 0.14469369]\n",
"TF: shape: (768,) values: [-0.10986238 -0.04332284 0.02603893 -0.06236923 0.14469369]\n",
"\n",
"bert/encoder/layer_10/attention/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8515822 0.81392974 0.836747 0.78040504 0.88091415]\n",
"TF: shape: (768,) values: [0.8515822 0.81392974 0.836747 0.78040504 0.88091415]\n",
"\n",
"bert/encoder/layer_10/intermediate/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 3072) values: [-0.07061081 0.06997397 0.01433633 0.04150929 0.02865192]\n",
"TF: shape: (768, 3072) values: [-0.07061081 0.06997397 0.01433633 0.04150929 0.02865192]\n",
"\n",
"bert/encoder/layer_10/intermediate/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072,) values: [-0.13879126 -0.06401426 -0.1408043 -0.15043251 -0.10193057]\n",
"TF: shape: (3072,) values: [-0.13879126 -0.06401426 -0.1408043 -0.15043251 -0.10193057]\n",
"\n",
"bert/encoder/layer_10/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072, 768) values: [ 0.02918765 0.02609882 -0.02259856 0.01636725 -0.00038442]\n",
"TF: shape: (3072, 768) values: [ 0.02918765 0.02609882 -0.02259856 0.01636725 -0.00038442]\n",
"\n",
"bert/encoder/layer_10/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.01799502 0.10970547 -0.02384165 -0.03350981 0.10491351]\n",
"TF: shape: (768,) values: [-0.01799502 0.10970547 -0.02384165 -0.03350981 0.10491351]\n",
"\n",
"bert/encoder/layer_10/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.00999107 -0.0217309 -0.0854177 -0.01109101 -0.07902174]\n",
"TF: shape: (768,) values: [ 0.00999107 -0.0217309 -0.0854177 -0.01109101 -0.07902174]\n",
"\n",
"bert/encoder/layer_10/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.8272796 0.8597452 0.79116803 0.81267637 0.8273501 ]\n",
"TF: shape: (768,) values: [0.8272796 0.8597452 0.79116803 0.81267637 0.8273501 ]\n",
"\n",
"bert/encoder/layer_11/attention/self/query/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.04141425 -0.06491017 -0.03202523 0.06226195 0.02193764]\n",
"TF: shape: (768, 768) values: [-0.04141425 -0.06491017 -0.03202523 0.06226195 0.02193764]\n",
"\n",
"bert/encoder/layer_11/attention/self/query/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.0501296 0.11886728 0.2186807 0.08720991 -0.20476632]\n",
"TF: shape: (768,) values: [ 0.0501296 0.11886728 0.2186807 0.08720991 -0.20476632]\n",
"\n",
"bert/encoder/layer_11/attention/self/key/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.02634268 -0.01357682 -0.06076496 0.04210597 0.01783857]\n",
"TF: shape: (768, 768) values: [ 0.02634268 -0.01357682 -0.06076496 0.04210597 0.01783857]\n",
"\n",
"bert/encoder/layer_11/attention/self/key/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.0007798 -0.00065806 -0.00010521 0.00119144 -0.00180091]\n",
"TF: shape: (768,) values: [-0.0007798 -0.00065806 -0.00010521 0.00119144 -0.00180091]\n",
"\n",
"bert/encoder/layer_11/attention/self/value/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.03520973 -0.00678078 -0.02883583 -0.01011515 0.04519828]\n",
"TF: shape: (768, 768) values: [ 0.03520973 -0.00678078 -0.02883583 -0.01011515 0.04519828]\n",
"\n",
"bert/encoder/layer_11/attention/self/value/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.01502306 -0.00530942 0.00023572 0.00205218 -0.00578036]\n",
"TF: shape: (768,) values: [ 0.01502306 -0.00530942 0.00023572 0.00205218 -0.00578036]\n",
"\n",
"bert/encoder/layer_11/attention/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [ 0.02361419 0.03112707 -0.00063031 0.04209773 -0.02434015]\n",
"TF: shape: (768, 768) values: [ 0.02361419 0.03112707 -0.00063031 0.04209773 -0.02434015]\n",
"\n",
"bert/encoder/layer_11/attention/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [ 0.02566087 0.0028438 -0.00475678 0.02149458 -0.01755187]\n",
"TF: shape: (768,) values: [ 0.02566087 0.0028438 -0.00475678 0.02149458 -0.01755187]\n",
"\n",
"bert/encoder/layer_11/attention/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.03134411 0.01207957 -0.04636396 -0.03013046 0.07944281]\n",
"TF: shape: (768,) values: [-0.03134411 0.01207957 -0.04636396 -0.03013046 0.07944281]\n",
"\n",
"bert/encoder/layer_11/attention/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.85203767 0.8020145 0.8554237 0.8150477 0.8441815 ]\n",
"TF: shape: (768,) values: [0.85203767 0.8020145 0.8554237 0.8150477 0.8441815 ]\n",
"\n",
"bert/encoder/layer_11/intermediate/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 3072) values: [ 0.05871898 -0.01124212 0.00206979 -0.04366514 -0.00716808]\n",
"TF: shape: (768, 3072) values: [ 0.05871898 -0.01124212 0.00206979 -0.04366514 -0.00716808]\n",
"\n",
"bert/encoder/layer_11/intermediate/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072,) values: [-0.09762521 -0.06175711 -0.05153917 -0.08580919 -0.09734315]\n",
"TF: shape: (3072,) values: [-0.09762521 -0.06175711 -0.05153917 -0.08580919 -0.09734315]\n",
"\n",
"bert/encoder/layer_11/output/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (3072, 768) values: [-0.022382 0.01073206 -0.01357213 0.02484621 0.01403091]\n",
"TF: shape: (3072, 768) values: [-0.022382 0.01073206 -0.01357213 0.02484621 0.01403091]\n",
"\n",
"bert/encoder/layer_11/output/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.06574099 0.04207807 0.01201084 0.00229322 0.05551811]\n",
"TF: shape: (768,) values: [-0.06574099 0.04207807 0.01201084 0.00229322 0.05551811]\n",
"\n",
"bert/encoder/layer_11/output/LayerNorm/beta\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.00634605 -0.01989403 0.04628465 0.01585056 -0.04256899]\n",
"TF: shape: (768,) values: [-0.00634605 -0.01989403 0.04628465 0.01585056 -0.04256899]\n",
"\n",
"bert/encoder/layer_11/output/LayerNorm/gamma\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [0.6384234 0.6300364 0.66570055 0.6126921 0.63756436]\n",
"TF: shape: (768,) values: [0.6384234 0.6300364 0.66570055 0.6126921 0.63756436]\n",
"\n",
"bert/pooler/dense/kernel\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768, 768) values: [-0.00127425 0.00199868 -0.03863145 -0.00139355 0.00691627]\n",
"TF: shape: (768, 768) values: [-0.00127425 0.00199868 -0.03863145 -0.00139355 0.00691627]\n",
"\n",
"bert/pooler/dense/bias\n",
"|sum(pt_wts - tf_wts)| = 0.0\n",
"PT: shape: (768,) values: [-0.03597581 -0.00389536 0.05181352 0.02224747 -0.00493723]\n",
"TF: shape: (768,) values: [-0.03597581 -0.00389536 0.05181352 0.02224747 -0.00493723]\n",
"\n"
]
}
],
"source": [
"tensors_to_transopse = (\n",
" \"dense.weight\",\n",
" \"attention.self.query\",\n",
" \"attention.self.key\",\n",
" \"attention.self.value\"\n",
")\n",
"var_map = (\n",
" ('layer.', 'layer_'),\n",
" ('word_embeddings.weight', 'word_embeddings'),\n",
" ('position_embeddings.weight', 'position_embeddings'),\n",
" ('token_type_embeddings.weight', 'token_type_embeddings'),\n",
" ('.', '/'),\n",
" ('LayerNorm/weight', 'LayerNorm/gamma'),\n",
" ('LayerNorm/bias', 'LayerNorm/beta'),\n",
" ('weight', 'kernel')\n",
")\n",
"\n",
"def to_tf_var_name(name:str):\n",
" for patt, repl in iter(var_map):\n",
" name = name.replace(patt, repl)\n",
" return 'bert/{}'.format(name)\n",
"\n",
"tf_vars = {v.name: session.run(fetches=v) for v in tf.global_variables()}\n",
"pt_vars = {}\n",
"for v, T in pt_model.state_dict().items():\n",
" T = T.detach().numpy()\n",
" if any([x in v for x in tensors_to_transopse]):\n",
" T = T.T\n",
" pt_vars.update({to_tf_var_name(v): T})\n",
"\n",
"for var_name in tf_vars:\n",
" \n",
" pt = pt_vars[var_name.strip(\":0\")]\n",
" tf = tf_vars[var_name]\n",
"\n",
" print(var_name.strip(\":0\"))\n",
" \n",
" # Assert equivalence\n",
" print(\"|sum(pt_wts - tf_wts)| = {}\".format(\n",
" np.abs(np.sum(pt - tf, keepdims=False))\n",
" ))\n",
" assert not np.sum(pt - tf, keepdims=False)\n",
" \n",
" if len(pt.shape) == 2:\n",
" print(\"PT: shape: {0} values: {1}\".format(pt.shape, pt[0, :5]))\n",
" print(\"TF: shape: {0} values: {1}\".format(tf.shape, tf[0, :5]))\n",
" else:\n",
" print(\"PT: shape: {0} values: {1}\".format(pt.shape, pt[:5]))\n",
" print(\"TF: shape: {0} values: {1}\".format(tf.shape, tf[:5]))\n",
" print()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Compare Layer-12 Projections"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"MSE: 2.7155439966009e-05\n",
"PT-values: [-0.876663 -0.41088238 -0.12200808 0.44941 0.19445966]\n",
"TF-values: [-0.8742865 -0.40621698 -0.10585472 0.444904 0.1825743 ]\n"
]
}
],
"source": [
"# Mean Squared Error (MSE) between last projection of each model\n",
"MSE = np.mean((pt_embedding - tf_embedding) ** 2, keepdims=False)\n",
"print(\"MSE: {}\".format(MSE))\n",
"print(\"PT-values: {}\".format(pt_embedding[0, :5]))\n",
"print(\"TF-values: {}\".format(tf_embedding[0, :5]))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "nlp",
"language": "python",
"name": "nlp"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment