v

libs/configs/default.py

+from libs.utils.vocab import DocTypeVocab
+from libs.utils.counter import Counter
+
+
+train_lrc_paths = [\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/extract_lrc/test/infos.lrc", \
+        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/extract_lrc/train/infos.lrc", \
+            "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/extract_lrc/val/infos.lrc", \
+                "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/info_lrc/test/infos.lrc", \
+                    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/info_lrc/train/infos.lrc", \
+                        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/info_lrc/val/infos.lrc"
+    ]
+
+train_sorted_lrc_paths = [\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/extract_sorted_lrc/test/infos.lrc", \
+        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/extract_sorted_lrc/train/infos.lrc", \
+            "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/extract_sorted_lrc/val/infos.lrc", \
+                "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/extract_sorted_lrc/test/infos.lrc", \
+                    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/extract_sorted_lrc/train/infos.lrc", \
+                        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/extract_sorted_lrc/val/infos.lrc"
+    ]
+
+train_image_dirs = [\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/rvl-cdip/images/", \
+        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/rvl-cdip/images/", \
+            "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/rvl-cdip/images/", \
+                "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/DocBank_500K_images/", \
+                    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/DocBank_500K_images/", \
+                        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/DocBank_500K_images/"
+    ]
+
+train_rvlcdip_lrc_paths = [\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/extract_sorted_lrc/test/infos.lrc", \
+        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/extract_sorted_lrc/train/infos.lrc", \
+            "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/extract_sorted_lrc/val/infos.lrc",
+    ]
+
+train_rvlcdip_image_dirs = [\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/rvl-cdip/images/", \
+        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/rvl-cdip/images/", \
+            "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/rvl-cdip/images/", \
+    ]
+
+train_docbank_lrc_paths =[\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/info_lrc/test/infos.lrc", \
+        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/info_lrc/train/infos.lrc", \
+            "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/info_lrc/val/infos.lrc"
+    ]
+
+train_docbank_image_dirs = [\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/DocBank_500K_images/", \
+        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/DocBank_500K_images/", \
+            "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/DocBank_500K_images/"
+
+    ]
+
+train_ucfs_180w_lrc_paths = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/extract_lrc/180w/train/infos.lrc"
+    ]
+
+train_ucfs_180w_image_dirs = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/pdf/images"
+    ]
+
+train_ucfs_700w_lrc_paths = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/extract_lrc/700w/train/infos.lrc"
+    ]
+
+train_ucfs_700w_image_dirs = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/pdf/images"
+    ]
+
+train_ucfs_180w_word_lrc_paths = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/extract_lrc/180w/train/word_level/infos.lrc"
+    ]
+
+train_ucfs_180w_word_image_dirs = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/pdf/images"
+    ]
+
+train_ucfs_700w_word_lrc_paths = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/extract_lrc/700w/train/word_level/infos.lrc"
+    ]
+
+train_ucfs_700w_word_image_dirs = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/pdf/images"
+    ]
+
+train_ucfs_700w_word_v2_lrc_paths = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/extract_lrc/700w/train/word_level_v2/infos.lrc"
+    ]
+
+train_ucfs_700w_word_v2_image_dirs = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/pdf/images"
+    ]
+
+train_ucfs_900w_word_lrc_paths = [\
+    "/yrfs2/cv1/jszhang6/zrzhang6/DocumentPretrain/dataset/ucfs/extract_lrc/900w/train/word_level/infos.lrc"
+    ]
+
+train_ucfs_900w_word_image_dirs = [\
+    "/yrfs2/cv1/jszhang6/zrzhang6/DocumentPretrain/dataset/ucfs/images/images"
+    ]
+
+# valid path
+valid_lrc_paths = [\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/extract_lrc/small_test/infos.lrc",\
+        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/info_lrc/small_test/infos.lrc"
+    ]
+
+valid_sorted_lrc_paths = [\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/extract_sorted_lrc/small_test/infos.lrc",\
+        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/extract_sorted_lrc/small_test/infos.lrc"
+    ]
+
+valid_image_dirs = [\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/rvl-cdip/images/", \
+        "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/DocBank_500K_images/"
+    ]
+
+valid_rvlcdip_lrc_paths = [\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/extract_sorted_lrc/small_test/infos.lrc",
+    ]
+
+valid_rvlcdip_image_dirs=[\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/RVL_CDIP/rvl-cdip/images/"
+    ]
+
+valid_docbank_lrc_paths = [\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/extract_sorted_lrc/small_test/infos.lrc"
+    ]
+
+valid_docbank_image_dirs = [\
+    "/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/DocBank/DocBank_500K_images/"
+    ]
+
+valid_ucfs_180w_lrc_paths = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/extract_lrc/180w/test/infos.lrc"
+    ]
+
+valid_ucfs_180w_image_dirs = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/pdf/images"
+    ]
+
+valid_ucfs_700w_lrc_paths = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/extract_lrc/700w/test/infos.lrc"
+    ]
+
+valid_ucfs_700w_image_dirs = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/pdf/images"
+    ]
+
+valid_ucfs_180w_word_lrc_paths = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/extract_lrc/180w/test/word_level/infos.lrc"
+    ]
+
+valid_ucfs_180w_word_image_dirs = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/pdf/images"
+    ]
+
+valid_ucfs_700w_word_lrc_paths = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/extract_lrc/700w/test/word_level/infos.lrc"
+    ]
+
+valid_ucfs_700w_word_image_dirs = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/pdf/images"
+    ]
+
+valid_ucfs_700w_word_v2_lrc_paths = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/extract_lrc/700w/test/word_level_v2/infos.lrc"
+    ]
+
+valid_ucfs_700w_word_v2_image_dirs = [\
+    "/yrfs1/hyperbrain/hyyang11/zrzhang6/DocPretrain/dataset/ucfs/pdf/images"
+    ]
+
+valid_ucfs_900w_word_lrc_paths = [\
+    "/yrfs2/cv1/jszhang6/zrzhang6/DocumentPretrain/dataset/ucfs/extract_lrc/900w/test/word_level/infos.lrc"
+    ]
+
+valid_ucfs_900w_word_image_dirs = [\
+    "/yrfs2/cv1/jszhang6/zrzhang6/DocumentPretrain/dataset/ucfs/images/images"
+    ]
+
+
+# extract by easyocr and tokenize by robert-sentence-bert
+train_rvlcdip_sentence_lrc_paths = ["/work1/cv1/jszhang6/DocumentPretrain/dataset/rvlcdip/extract_lrc/line/v2/extract_lrc/infos.lrc"]
+# train_rvlcdip_sentence_lrc_paths = ["/work1/cv1/jszhang6/DocumentPretrain/dataset/rvlcdip/extract_lrc/line/v1/extract_lrc/small_val.lrc"]
+train_rvlcdip_sentence_image_dirs = [None]
+
+valid_rvlcdip_sentence_lrc_paths = ["/work1/cv1/jszhang6/DocumentPretrain/dataset/rvlcdip/extract_lrc/line/v1/extract_lrc/small_val.lrc"]
+valid_rvlcdip_sentence_image_dirs = [None]
+
+train_ucfs_sentence_200w_lrc_paths = ["/work1/cv1/jszhang6/DocumentPretrain/dataset/ucfs/word/v1/extract_lrc/infos.lrc"]
+train_ucfs_sentence_200w_image_dirs = [None]
+
+valid_ucfs_sentence_200w_lrc_paths = ["/work1/cv1/jszhang6/DocumentPretrain/dataset/ucfs/word/v1/extract_lrc/infos.lrc"]
+valid_ucfs_sentence_200w_image_dirs = [None]
+
+# dataset path dict
+datasets = dict(
+    # train
+    train_lrc_paths=train_lrc_paths,
+    train_sorted_lrc_paths=train_sorted_lrc_paths,
+    train_image_dirs=train_image_dirs,
+    train_rvlcdip_lrc_paths=train_rvlcdip_lrc_paths,
+    train_rvlcdip_image_dirs=train_rvlcdip_image_dirs,
+    train_docbank_lrc_paths=train_docbank_lrc_paths,
+    train_docbank_image_dirs=train_docbank_image_dirs,
+    train_ucfs_180w_lrc_paths=train_ucfs_180w_lrc_paths,
+    train_ucfs_180w_image_dirs=train_ucfs_180w_image_dirs,
+    train_ucfs_700w_lrc_paths=train_ucfs_700w_lrc_paths,
+    train_ucfs_700w_image_dirs=train_ucfs_700w_image_dirs,
+    train_ucfs_180w_word_lrc_paths=train_ucfs_180w_word_lrc_paths,
+    train_ucfs_180w_word_image_dirs=train_ucfs_180w_word_image_dirs,
+    train_ucfs_700w_word_lrc_paths=train_ucfs_700w_word_lrc_paths,
+    train_ucfs_700w_word_image_dirs=train_ucfs_700w_word_image_dirs,
+    train_ucfs_700w_word_v2_lrc_paths=train_ucfs_700w_word_v2_lrc_paths,
+    train_ucfs_700w_word_v2_image_dirs=train_ucfs_700w_word_v2_image_dirs,
+    train_ucfs_900w_word_lrc_paths=train_ucfs_900w_word_lrc_paths,
+    train_ucfs_900w_word_image_dirs=train_ucfs_900w_word_image_dirs,
+
+    # valid
+    valid_lrc_paths=valid_lrc_paths,
+    valid_sorted_lrc_paths=valid_sorted_lrc_paths,
+    valid_image_dirs=valid_image_dirs,
+    valid_rvlcdip_lrc_paths=valid_rvlcdip_lrc_paths,
+    valid_rvlcdip_image_dirs=valid_rvlcdip_image_dirs,
+    valid_docbank_lrc_paths=valid_docbank_lrc_paths,
+    valid_docbank_image_dirs=valid_docbank_image_dirs,
+    valid_ucfs_180w_lrc_paths=valid_ucfs_180w_lrc_paths,
+    valid_ucfs_180w_image_dirs=valid_ucfs_180w_image_dirs,
+    valid_ucfs_700w_lrc_paths=valid_ucfs_700w_lrc_paths,
+    valid_ucfs_700w_image_dirs=valid_ucfs_700w_image_dirs,
+    valid_ucfs_180w_word_lrc_paths=valid_ucfs_180w_word_lrc_paths,
+    valid_ucfs_180w_word_image_dirs=valid_ucfs_180w_word_image_dirs,
+    valid_ucfs_700w_word_lrc_paths=valid_ucfs_700w_word_lrc_paths,
+    valid_ucfs_700w_word_image_dirs=valid_ucfs_700w_word_image_dirs,
+    valid_ucfs_700w_word_v2_lrc_paths=valid_ucfs_700w_word_v2_lrc_paths,
+    valid_ucfs_700w_word_v2_image_dirs=valid_ucfs_700w_word_v2_image_dirs,
+    valid_ucfs_900w_word_lrc_paths=valid_ucfs_900w_word_lrc_paths,
+    valid_ucfs_900w_word_image_dirs=valid_ucfs_900w_word_image_dirs,
+
+    # sentence level
+    train_rvlcdip_sentence_lrc_paths=train_rvlcdip_sentence_lrc_paths,
+    train_rvlcdip_sentence_image_dirs=train_rvlcdip_sentence_image_dirs,
+    valid_rvlcdip_sentence_lrc_paths=valid_rvlcdip_sentence_lrc_paths,
+    valid_rvlcdip_sentence_image_dirs=valid_rvlcdip_sentence_image_dirs,
+    
+    train_ucfs_sentence_200w_lrc_paths=train_ucfs_sentence_200w_lrc_paths,
+    train_ucfs_sentence_200w_image_dirs=train_ucfs_sentence_200w_image_dirs,
+    valid_ucfs_sentence_200w_lrc_paths=valid_ucfs_sentence_200w_lrc_paths,
+    valid_ucfs_sentence_200w_image_dirs=valid_ucfs_sentence_200w_image_dirs
+)
+
+# document type vocab
+doctype_vocab = DocTypeVocab()
+
+# counter for show each item loss
+counter = Counter(cache_nums=1000)
\ No newline at end of file

libs/configs/finetune_funsd.py

+from libs.utils.vocab import FunsdTokenTypeVocab
+
+train_lrc_paths = ["/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/funsd/dataset/extract_word_lrc/train/infos.lrc"]
+train_image_dirs = ["/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/funsd/dataset/training_data/images/"]
+
+valid_lrc_paths = ["/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/funsd/dataset/extract_word_lrc/test/infos.lrc"]
+valid_image_dirs = ["/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/funsd/dataset/testing_data/images/"]
+
+funsd_vocab = FunsdTokenTypeVocab()
+
+max_length = 1024
\ No newline at end of file

libs/configs/finetune_funsd_line_bbox.py

+from libs.utils.vocab import FunsdTokenTypeVocab
+
+train_lrc_paths = ["/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/funsd/dataset/extract_word_lrc/train/infos.lrc"]
+train_image_dirs = ["/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/funsd/dataset/training_data/images/"]
+
+valid_lrc_paths = ["/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/funsd/dataset/extract_word_lrc/test/infos.lrc"]
+valid_image_dirs = ["/yrfs1/intern/zrzhang6/DocumentPretrain/dataset/funsd/dataset/testing_data/images/"]
+
+funsd_vocab = FunsdTokenTypeVocab()
+
+max_length = 512
\ No newline at end of file

libs/configs/finetune_funsd_sentence.py

+from libs.utils.vocab import FunsdEntityVocab
+
+train_lrc_paths = ["/yrfs1/intern/pfhu6/GraphDoc/dataset/funsd/train.lrc"]
+train_image_dirs = ["/yrfs1/intern/pfhu6/GraphDoc/dataset/funsd/dataset/training_data/images/"]
+
+valid_lrc_paths = ["/yrfs1/intern/pfhu6/GraphDoc/dataset/funsd/val.lrc"]
+valid_image_dirs = ["/yrfs1/intern/pfhu6/GraphDoc/dataset/funsd/dataset/testing_data/images/"]
+
+funsd_vocab = FunsdEntityVocab()
\ No newline at end of file

libs/configs/graphdoc/config.json

+{
+  "attention_probs_dropout_prob": 0.1,
+  "coordinate_size": 128,
+  "fast_qkv": true,
+  "gradient_checkpointing": false,
+  "has_relative_attention_bias": false,
+  "has_spatial_attention_bias": false,
+  "has_visual_segment_embedding": true,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "image_feature_pool_shape": [
+    7,
+    7,
+    256
+  ],
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_2d_position_embeddings": 1024,
+  "max_position_embeddings": 512,
+  "max_rel_2d_pos": 256,
+  "max_rel_pos": 128,
+  "model_type": "layoutclmV30",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "shape_size": 128,
+  "rel_2d_pos_bins": 64,
+  "rel_pos_bins": 32,
+  "type_vocab_size": 2,
+  "vocab_size": 30522,
+  "use_mlm":true,
+  "mlm_alpha":1,
+  "mlm_prob":0.15,
+  "use_rel_2d":true,
+  "local_atten":true,
+  "rel_topk":36,
+  "abs_emb_type":"Linear",
+  "datasets":["rvlcdip_sentence"],
+  "sentence_model":"/yrfs2/cv1/jszhang6/zrzhang6/PretrainModel/Transformer/roberta-base-nli-stsb-mean-tokens/"
+}
\ No newline at end of file

libs/configs/graphdoc/gitattributes

+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tar.gz filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text