"torchvision/git@developer.sourcefind.cn:OpenDAS/vision.git" did not exist on "93c85bbcc31f8d5a052daf06f2f91f39697af1a4"
Unverified Commit d2ed57e9 authored by Rhett Ying's avatar Rhett Ying Committed by GitHub
Browse files

[DistDGL] add log when load graph and data (#5350)



* [DistDGL] add log when load graph and data

* fix lint

* refine log

---------
Co-authored-by: default avatarIsrat Nisa <neesha295@gmail.com>
parent e74b3d3d
...@@ -391,8 +391,13 @@ class DistGraphServer(KVServer): ...@@ -391,8 +391,13 @@ class DistGraphServer(KVServer):
self.client_g.edata[k], dtype self.client_g.edata[k], dtype
) )
# Create the graph formats specified the users. # Create the graph formats specified the users.
print(
"Start to create specified graph formats which may take "
"non-trivial time."
)
self.client_g = self.client_g.formats(graph_format) self.client_g = self.client_g.formats(graph_format)
self.client_g.create_formats_() self.client_g.create_formats_()
print("Finished creating specified graph formats.")
if not disable_shared_mem: if not disable_shared_mem:
self.client_g = _copy_graph_to_shared_mem( self.client_g = _copy_graph_to_shared_mem(
self.client_g, graph_name, graph_format self.client_g, graph_name, graph_format
......
...@@ -190,7 +190,14 @@ def load_partition(part_config, part_id, load_feats=True): ...@@ -190,7 +190,14 @@ def load_partition(part_config, part_id, load_feats=True):
assert ( assert (
"part_graph" in part_files "part_graph" in part_files
), "the partition does not contain graph structure." ), "the partition does not contain graph structure."
graph = load_graphs(relative_to_config(part_files["part_graph"]))[0][0] partition_path = relative_to_config(part_files["part_graph"])
print(
f"Start to load partition from {partition_path} which is "
f"{os.path.getsize(partition_path)} bytes. It may take non-trivial "
"time for large partition."
)
graph = load_graphs(partition_path)[0][0]
print("Finished loading partition.")
assert ( assert (
NID in graph.ndata NID in graph.ndata
...@@ -294,10 +301,22 @@ def load_partition_feats( ...@@ -294,10 +301,22 @@ def load_partition_feats(
), "the partition does not contain edge feature." ), "the partition does not contain edge feature."
node_feats = None node_feats = None
if load_nodes: if load_nodes:
node_feats = load_tensors(relative_to_config(part_files["node_feats"])) feat_path = relative_to_config(part_files["node_feats"])
print(
f"Start to load node data from {feat_path} which is "
f"{os.path.getsize(feat_path)} bytes."
)
node_feats = load_tensors(feat_path)
print("Finished loading node data.")
edge_feats = None edge_feats = None
if load_edges: if load_edges:
edge_feats = load_tensors(relative_to_config(part_files["edge_feats"])) feat_path = relative_to_config(part_files["edge_feats"])
print(
f"Start to load edge data from {feat_path} which is "
f"{os.path.getsize(feat_path)} bytes."
)
edge_feats = load_tensors(feat_path)
print("Finished loading edge data.")
# In the old format, the feature name doesn't contain node/edge type. # In the old format, the feature name doesn't contain node/edge type.
# For compatibility, let's add node/edge types to the feature names. # For compatibility, let's add node/edge types to the feature names.
if node_feats is not None: if node_feats is not None:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment