Skip to content

GitLab

"git@developer.sourcefind.cn:OpenDAS/pytorch3d.git" did not exist on "1cd0cbffb82d250ab95e783f0c3dfca2ea401a9e"
Unverified Commit 67d93458 authored by Rhett Ying's avatar Rhett Ying Committed by GitHub
Browse files

[GraphBolt] update OnDiskDataset tutorial (#6793)

parent 3cf82462
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 74 additions and 105 deletions
notebooks/stochastic_training/ondisk_dataset_homograph.ipynb
View file @ 67d93458
...@@ -152,28 +152,28 @@ ...@@ -152,28 +152,28 @@
"node_feat_0 = np.random.rand(num_nodes, 5)\n", "node_feat_0 = np.random.rand(num_nodes, 5)\n",
"print(f\"Part of node feature [feat_0]: {node_feat_0[:10, :]}\")\n", "print(f\"Part of node feature [feat_0]: {node_feat_0[:10, :]}\")\n",
"np.save(node_feat_0_path, node_feat_0)\n", "np.save(node_feat_0_path, node_feat_0)\n",
"print(f\"Node feature [feat_0] is saved to {node_feat_0_path}\")\n", "print(f\"Node feature [feat_0] is saved to {node_feat_0_path}\\n\")\n",
"\n", "\n",
"# Generate another node feature in torch tensor\n", "# Generate another node feature in torch tensor\n",
"node_feat_1_path = os.path.join(base_dir, \"node-feat-1.pt\")\n", "node_feat_1_path = os.path.join(base_dir, \"node-feat-1.pt\")\n",
"node_feat_1 = torch.rand(num_nodes, 5)\n", "node_feat_1 = torch.rand(num_nodes, 5)\n",
"print(f\"Part of node feature [feat_1]: {node_feat_1[:10, :]}\")\n", "print(f\"Part of node feature [feat_1]: {node_feat_1[:10, :]}\")\n",
"torch.save(node_feat_1, node_feat_1_path)\n", "torch.save(node_feat_1, node_feat_1_path)\n",
"print(f\"Node feature [feat_1] is saved to {node_feat_1_path}\")\n", "print(f\"Node feature [feat_1] is saved to {node_feat_1_path}\\n\")\n",
"\n", "\n",
"# Generate edge feature in numpy array.\n", "# Generate edge feature in numpy array.\n",
"edge_feat_0_path = os.path.join(base_dir, \"edge-feat-0.npy\")\n", "edge_feat_0_path = os.path.join(base_dir, \"edge-feat-0.npy\")\n",
"edge_feat_0 = np.random.rand(num_edges, 5)\n", "edge_feat_0 = np.random.rand(num_edges, 5)\n",
"print(f\"Part of edge feature [feat_0]: {edge_feat_0[:10, :]}\")\n", "print(f\"Part of edge feature [feat_0]: {edge_feat_0[:10, :]}\")\n",
"np.save(edge_feat_0_path, edge_feat_0)\n", "np.save(edge_feat_0_path, edge_feat_0)\n",
"print(f\"Edge feature [feat_0] is saved to {edge_feat_0_path}\")\n", "print(f\"Edge feature [feat_0] is saved to {edge_feat_0_path}\\n\")\n",
"\n", "\n",
"# Generate another edge feature in torch tensor\n", "# Generate another edge feature in torch tensor\n",
"edge_feat_1_path = os.path.join(base_dir, \"edge-feat-1.pt\")\n", "edge_feat_1_path = os.path.join(base_dir, \"edge-feat-1.pt\")\n",
"edge_feat_1 = torch.rand(num_edges, 5)\n", "edge_feat_1 = torch.rand(num_edges, 5)\n",
"print(f\"Part of edge feature [feat_1]: {edge_feat_1[:10, :]}\")\n", "print(f\"Part of edge feature [feat_1]: {edge_feat_1[:10, :]}\")\n",
"torch.save(edge_feat_1, edge_feat_1_path)\n", "torch.save(edge_feat_1, edge_feat_1_path)\n",
"print(f\"Edge feature [feat_1] is saved to {edge_feat_1_path}\")\n" "print(f\"Edge feature [feat_1] is saved to {edge_feat_1_path}\\n\")\n"
], ],
"metadata": { "metadata": {
"id": "_PVu1u5brBhF" "id": "_PVu1u5brBhF"
...@@ -215,37 +215,37 @@ ...@@ -215,37 +215,37 @@
"nc_train_ids = ids[:num_trains]\n", "nc_train_ids = ids[:num_trains]\n",
"print(f\"Part of train ids for node classification: {nc_train_ids[:10]}\")\n", "print(f\"Part of train ids for node classification: {nc_train_ids[:10]}\")\n",
"np.save(nc_train_ids_path, nc_train_ids)\n", "np.save(nc_train_ids_path, nc_train_ids)\n",
"print(f\"NC train ids are saved to {nc_train_ids_path}\")\n", "print(f\"NC train ids are saved to {nc_train_ids_path}\\n\")\n",
"\n", "\n",
"nc_train_labels_path = os.path.join(base_dir, \"nc-train-labels.pt\")\n", "nc_train_labels_path = os.path.join(base_dir, \"nc-train-labels.pt\")\n",
"nc_train_labels = torch.randint(0, 10, (num_trains,))\n", "nc_train_labels = torch.randint(0, 10, (num_trains,))\n",
"print(f\"Part of train labels for node classification: {nc_train_labels[:10]}\")\n", "print(f\"Part of train labels for node classification: {nc_train_labels[:10]}\")\n",
"torch.save(nc_train_labels, nc_train_labels_path)\n", "torch.save(nc_train_labels, nc_train_labels_path)\n",
"print(f\"NC train labels are saved to {nc_train_labels_path}\")\n", "print(f\"NC train labels are saved to {nc_train_labels_path}\\n\")\n",
"\n", "\n",
"nc_val_ids_path = os.path.join(base_dir, \"nc-val-ids.npy\")\n", "nc_val_ids_path = os.path.join(base_dir, \"nc-val-ids.npy\")\n",
"nc_val_ids = ids[num_trains:num_trains+num_vals]\n", "nc_val_ids = ids[num_trains:num_trains+num_vals]\n",
"print(f\"Part of val ids for node classification: {nc_val_ids[:10]}\")\n", "print(f\"Part of val ids for node classification: {nc_val_ids[:10]}\")\n",
"np.save(nc_val_ids_path, nc_val_ids)\n", "np.save(nc_val_ids_path, nc_val_ids)\n",
"print(f\"NC val ids are saved to {nc_val_ids_path}\")\n", "print(f\"NC val ids are saved to {nc_val_ids_path}\\n\")\n",
"\n", "\n",
"nc_val_labels_path = os.path.join(base_dir, \"nc-val-labels.pt\")\n", "nc_val_labels_path = os.path.join(base_dir, \"nc-val-labels.pt\")\n",
"nc_val_labels = torch.randint(0, 10, (num_vals,))\n", "nc_val_labels = torch.randint(0, 10, (num_vals,))\n",
"print(f\"Part of val labels for node classification: {nc_val_labels[:10]}\")\n", "print(f\"Part of val labels for node classification: {nc_val_labels[:10]}\")\n",
"torch.save(nc_val_labels, nc_val_labels_path)\n", "torch.save(nc_val_labels, nc_val_labels_path)\n",
"print(f\"NC val labels are saved to {nc_val_labels_path}\")\n", "print(f\"NC val labels are saved to {nc_val_labels_path}\\n\")\n",
"\n", "\n",
"nc_test_ids_path = os.path.join(base_dir, \"nc-test-ids.npy\")\n", "nc_test_ids_path = os.path.join(base_dir, \"nc-test-ids.npy\")\n",
"nc_test_ids = ids[-num_tests:]\n", "nc_test_ids = ids[-num_tests:]\n",
"print(f\"Part of test ids for node classification: {nc_test_ids[:10]}\")\n", "print(f\"Part of test ids for node classification: {nc_test_ids[:10]}\")\n",
"np.save(nc_test_ids_path, nc_test_ids)\n", "np.save(nc_test_ids_path, nc_test_ids)\n",
"print(f\"NC test ids are saved to {nc_test_ids_path}\")\n", "print(f\"NC test ids are saved to {nc_test_ids_path}\\n\")\n",
"\n", "\n",
"nc_test_labels_path = os.path.join(base_dir, \"nc-test-labels.pt\")\n", "nc_test_labels_path = os.path.join(base_dir, \"nc-test-labels.pt\")\n",
"nc_test_labels = torch.randint(0, 10, (num_tests,))\n", "nc_test_labels = torch.randint(0, 10, (num_tests,))\n",
"print(f\"Part of test labels for node classification: {nc_test_labels[:10]}\")\n", "print(f\"Part of test labels for node classification: {nc_test_labels[:10]}\")\n",
"torch.save(nc_test_labels, nc_test_labels_path)\n", "torch.save(nc_test_labels, nc_test_labels_path)\n",
"print(f\"NC test labels are saved to {nc_test_labels_path}\")" "print(f\"NC test labels are saved to {nc_test_labels_path}\\n\")"
], ],
"metadata": { "metadata": {
"id": "S5-fyBbHzTCO" "id": "S5-fyBbHzTCO"
...@@ -274,31 +274,31 @@ ...@@ -274,31 +274,31 @@
"lp_train_node_pairs = edges[:num_trains, :]\n", "lp_train_node_pairs = edges[:num_trains, :]\n",
"print(f\"Part of train node pairs for link prediction: {lp_train_node_pairs[:10]}\")\n", "print(f\"Part of train node pairs for link prediction: {lp_train_node_pairs[:10]}\")\n",
"np.save(lp_train_node_pairs_path, lp_train_node_pairs)\n", "np.save(lp_train_node_pairs_path, lp_train_node_pairs)\n",
"print(f\"LP train node pairs are saved to {lp_train_node_pairs_path}\")\n", "print(f\"LP train node pairs are saved to {lp_train_node_pairs_path}\\n\")\n",
"\n", "\n",
"lp_val_node_pairs_path = os.path.join(base_dir, \"lp-val-node-pairs.npy\")\n", "lp_val_node_pairs_path = os.path.join(base_dir, \"lp-val-node-pairs.npy\")\n",
"lp_val_node_pairs = edges[num_trains:num_trains+num_vals, :]\n", "lp_val_node_pairs = edges[num_trains:num_trains+num_vals, :]\n",
"print(f\"Part of val node pairs for link prediction: {lp_val_node_pairs[:10]}\")\n", "print(f\"Part of val node pairs for link prediction: {lp_val_node_pairs[:10]}\")\n",
"np.save(lp_val_node_pairs_path, lp_val_node_pairs)\n", "np.save(lp_val_node_pairs_path, lp_val_node_pairs)\n",
"print(f\"LP val node pairs are saved to {lp_val_node_pairs_path}\")\n", "print(f\"LP val node pairs are saved to {lp_val_node_pairs_path}\\n\")\n",
"\n", "\n",
"lp_val_neg_dsts_path = os.path.join(base_dir, \"lp-val-neg-dsts.pt\")\n", "lp_val_neg_dsts_path = os.path.join(base_dir, \"lp-val-neg-dsts.pt\")\n",
"lp_val_neg_dsts = torch.randint(0, num_nodes, (num_vals, 10))\n", "lp_val_neg_dsts = torch.randint(0, num_nodes, (num_vals, 10))\n",
"print(f\"Part of val negative dsts for link prediction: {lp_val_neg_dsts[:10]}\")\n", "print(f\"Part of val negative dsts for link prediction: {lp_val_neg_dsts[:10]}\")\n",
"torch.save(lp_val_neg_dsts, lp_val_neg_dsts_path)\n", "torch.save(lp_val_neg_dsts, lp_val_neg_dsts_path)\n",
"print(f\"LP val negative dsts are saved to {lp_val_neg_dsts_path}\")\n", "print(f\"LP val negative dsts are saved to {lp_val_neg_dsts_path}\\n\")\n",
"\n", "\n",
"lp_test_node_pairs_path = os.path.join(base_dir, \"lp-test-node-pairs.npy\")\n", "lp_test_node_pairs_path = os.path.join(base_dir, \"lp-test-node-pairs.npy\")\n",
"lp_test_node_pairs = edges[-num_tests, :]\n", "lp_test_node_pairs = edges[-num_tests, :]\n",
"print(f\"Part of test node pairs for link prediction: {lp_test_node_pairs[:10]}\")\n", "print(f\"Part of test node pairs for link prediction: {lp_test_node_pairs[:10]}\")\n",
"np.save(lp_test_node_pairs_path, lp_test_node_pairs)\n", "np.save(lp_test_node_pairs_path, lp_test_node_pairs)\n",
"print(f\"LP test node pairs are saved to {lp_test_node_pairs_path}\")\n", "print(f\"LP test node pairs are saved to {lp_test_node_pairs_path}\\n\")\n",
"\n", "\n",
"lp_test_neg_dsts_path = os.path.join(base_dir, \"lp-test-neg-dsts.pt\")\n", "lp_test_neg_dsts_path = os.path.join(base_dir, \"lp-test-neg-dsts.pt\")\n",
"lp_test_neg_dsts = torch.randint(0, num_nodes, (num_tests, 10))\n", "lp_test_neg_dsts = torch.randint(0, num_nodes, (num_tests, 10))\n",
"print(f\"Part of test negative dsts for link prediction: {lp_test_neg_dsts[:10]}\")\n", "print(f\"Part of test negative dsts for link prediction: {lp_test_neg_dsts[:10]}\")\n",
"torch.save(lp_test_neg_dsts, lp_test_neg_dsts_path)\n", "torch.save(lp_test_neg_dsts, lp_test_neg_dsts_path)\n",
"print(f\"LP test negative dsts are saved to {lp_test_neg_dsts_path}\")" "print(f\"LP test negative dsts are saved to {lp_test_neg_dsts_path}\\n\")"
], ],
"metadata": { "metadata": {
"id": "u0jCnXIcAQy4" "id": "u0jCnXIcAQy4"
...@@ -310,7 +310,14 @@ ...@@ -310,7 +310,14 @@
"cell_type": "markdown", "cell_type": "markdown",
"source": [ "source": [
"## Organize Data into YAML File\n", "## Organize Data into YAML File\n",
"Now we need to create a `metadata.yaml` file which contains the paths, dadta types of graph structure, feature data, training/validation/test sets. Please note that all path should be relative to `metadata.yaml`." "Now we need to create a `metadata.yaml` file which contains the paths, dadta types of graph structure, feature data, training/validation/test sets.\n",
"\n",
"Notes:\n",
"- all path should be relative to `metadata.yaml`.\n",
"- Below fields are optional and not specified in below example.\n",
" - `in_memory`: indicates whether to load dada into memory or `mmap`. Default is `True`.\n",
"\n",
"Please refer to [YAML specification](https://github.com/dmlc/dgl/blob/master/docs/source/stochastic_training/ondisk-dataset-specification.rst) for more details."
], ],
"metadata": { "metadata": {
"id": "wbk6-wxRK-6S" "id": "wbk6-wxRK-6S"
...@@ -331,22 +338,18 @@ ...@@ -331,22 +338,18 @@
" - domain: node\n", " - domain: node\n",
" name: feat_0\n", " name: feat_0\n",
" format: numpy\n", " format: numpy\n",
" in_memory: true\n",
" path: {os.path.basename(node_feat_0_path)}\n", " path: {os.path.basename(node_feat_0_path)}\n",
" - domain: node\n", " - domain: node\n",
" name: feat_1\n", " name: feat_1\n",
" format: torch\n", " format: torch\n",
" in_memory: true\n",
" path: {os.path.basename(node_feat_1_path)}\n", " path: {os.path.basename(node_feat_1_path)}\n",
" - domain: edge\n", " - domain: edge\n",
" name: feat_0\n", " name: feat_0\n",
" format: numpy\n", " format: numpy\n",
" in_memory: true\n",
" path: {os.path.basename(edge_feat_0_path)}\n", " path: {os.path.basename(edge_feat_0_path)}\n",
" - domain: edge\n", " - domain: edge\n",
" name: feat_1\n", " name: feat_1\n",
" format: torch\n", " format: torch\n",
" in_memory: true\n",
" path: {os.path.basename(edge_feat_1_path)}\n", " path: {os.path.basename(edge_feat_1_path)}\n",
" tasks:\n", " tasks:\n",
" - name: node_classification\n", " - name: node_classification\n",
...@@ -355,31 +358,25 @@ ...@@ -355,31 +358,25 @@
" - data:\n", " - data:\n",
" - name: seed_nodes\n", " - name: seed_nodes\n",
" format: numpy\n", " format: numpy\n",
" in_memory: true\n",
" path: {os.path.basename(nc_train_ids_path)}\n", " path: {os.path.basename(nc_train_ids_path)}\n",
" - name: labels\n", " - name: labels\n",
" format: torch\n", " format: torch\n",
" in_memory: true\n",
" path: {os.path.basename(nc_train_labels_path)}\n", " path: {os.path.basename(nc_train_labels_path)}\n",
" validation_set:\n", " validation_set:\n",
" - data:\n", " - data:\n",
" - name: seed_nodes\n", " - name: seed_nodes\n",
" format: numpy\n", " format: numpy\n",
" in_memory: true\n",
" path: {os.path.basename(nc_val_ids_path)}\n", " path: {os.path.basename(nc_val_ids_path)}\n",
" - name: labels\n", " - name: labels\n",
" format: torch\n", " format: torch\n",
" in_memory: true\n",
" path: {os.path.basename(nc_val_labels_path)}\n", " path: {os.path.basename(nc_val_labels_path)}\n",
" test_set:\n", " test_set:\n",
" - data:\n", " - data:\n",
" - name: seed_nodes\n", " - name: seed_nodes\n",
" format: numpy\n", " format: numpy\n",
" in_memory: true\n",
" path: {os.path.basename(nc_test_ids_path)}\n", " path: {os.path.basename(nc_test_ids_path)}\n",
" - name: labels\n", " - name: labels\n",
" format: torch\n", " format: torch\n",
" in_memory: true\n",
" path: {os.path.basename(nc_test_labels_path)}\n", " path: {os.path.basename(nc_test_labels_path)}\n",
" - name: link_prediction\n", " - name: link_prediction\n",
" num_classes: 10\n", " num_classes: 10\n",
...@@ -387,27 +384,22 @@ ...@@ -387,27 +384,22 @@
" - data:\n", " - data:\n",
" - name: node_pairs\n", " - name: node_pairs\n",
" format: numpy\n", " format: numpy\n",
" in_memory: true\n",
" path: {os.path.basename(lp_train_node_pairs_path)}\n", " path: {os.path.basename(lp_train_node_pairs_path)}\n",
" validation_set:\n", " validation_set:\n",
" - data:\n", " - data:\n",
" - name: node_pairs\n", " - name: node_pairs\n",
" format: numpy\n", " format: numpy\n",
" in_memory: true\n",
" path: {os.path.basename(lp_val_node_pairs_path)}\n", " path: {os.path.basename(lp_val_node_pairs_path)}\n",
" - name: negative_dsts\n", " - name: negative_dsts\n",
" format: torch\n", " format: torch\n",
" in_memory: true\n",
" path: {os.path.basename(lp_val_neg_dsts_path)}\n", " path: {os.path.basename(lp_val_neg_dsts_path)}\n",
" test_set:\n", " test_set:\n",
" - data:\n", " - data:\n",
" - name: node_pairs\n", " - name: node_pairs\n",
" format: numpy\n", " format: numpy\n",
" in_memory: true\n",
" path: {os.path.basename(lp_test_node_pairs_path)}\n", " path: {os.path.basename(lp_test_node_pairs_path)}\n",
" - name: negative_dsts\n", " - name: negative_dsts\n",
" format: torch\n", " format: torch\n",
" in_memory: true\n",
" path: {os.path.basename(lp_test_neg_dsts_path)}\n", " path: {os.path.basename(lp_test_neg_dsts_path)}\n",
"\"\"\"\n", "\"\"\"\n",
"metadata_path = os.path.join(base_dir, \"metadata.yaml\")\n", "metadata_path = os.path.join(base_dir, \"metadata.yaml\")\n",
...@@ -439,16 +431,16 @@ ...@@ -439,16 +431,16 @@
"source": [ "source": [
"dataset = gb.OnDiskDataset(base_dir).load()\n", "dataset = gb.OnDiskDataset(base_dir).load()\n",
"graph = dataset.graph\n", "graph = dataset.graph\n",
"print(f\"Loaded graph: {graph}\")\n", "print(f\"Loaded graph: {graph}\\n\")\n",
"\n", "\n",
"feature = dataset.feature\n", "feature = dataset.feature\n",
"print(f\"Loaded feature store: {feature}\")\n", "print(f\"Loaded feature store: {feature}\\n\")\n",
"\n", "\n",
"tasks = dataset.tasks\n", "tasks = dataset.tasks\n",
"nc_task = tasks[0]\n", "nc_task = tasks[0]\n",
"print(f\"Loaded node classification task: {nc_task}\")\n", "print(f\"Loaded node classification task: {nc_task}\\n\")\n",
"lp_task = tasks[1]\n", "lp_task = tasks[1]\n",
"print(f\"Loaded link prediction task: {lp_task}\")" "print(f\"Loaded link prediction task: {lp_task}\\n\")"
], ],
"metadata": { "metadata": {
"id": "W58CZoSzOiyo" "id": "W58CZoSzOiyo"
...@@ -457,4 +449,4 @@ ...@@ -457,4 +449,4 @@
"outputs": [] "outputs": []
} }
] ]
} }
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment