chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,35 @@
|
||||
# risky imports
|
||||
try:
|
||||
from easygraph.datasets.get_sample_graph import *
|
||||
from easygraph.datasets.gnn_benchmark import *
|
||||
from easygraph.datasets.hypergraph.coauthorship import *
|
||||
from easygraph.datasets.hypergraph.contact_primary_school import *
|
||||
from easygraph.datasets.hypergraph.cooking_200 import Cooking200
|
||||
from easygraph.datasets.hypergraph.House_Committees import House_Committees
|
||||
from easygraph.datasets.karate import KarateClubDataset
|
||||
from easygraph.datasets.mathoverflow_answers import mathoverflow_answers
|
||||
|
||||
from .ppi import LegacyPPIDataset
|
||||
from .ppi import PPIDataset
|
||||
except Exception as e:
|
||||
print(
|
||||
" Please install Pytorch before use graph-related datasets and"
|
||||
" hypergraph-related datasets."
|
||||
)
|
||||
|
||||
from .amazon_photo import AmazonPhotoDataset
|
||||
from .arxiv import ArxivHEPTHDataset
|
||||
from .citation_graph import CitationGraphDataset
|
||||
from .citation_graph import CiteseerGraphDataset
|
||||
from .citation_graph import CoraBinary
|
||||
from .citation_graph import CoraGraphDataset
|
||||
from .citation_graph import PubmedGraphDataset
|
||||
from .coauthor import CoauthorCSDataset
|
||||
from .facebook_ego import FacebookEgoNetDataset
|
||||
from .flickr import FlickrDataset
|
||||
from .github import GitHubUsersDataset
|
||||
from .reddit import RedditDataset
|
||||
from .roadnet import RoadNetCADataset
|
||||
from .twitter_ego import TwitterEgoDataset
|
||||
from .web_google import WebGoogleDataset
|
||||
from .wiki_topcats import WikiTopCatsDataset
|
||||
@@ -0,0 +1,110 @@
|
||||
import os
|
||||
|
||||
import easygraph as eg
|
||||
import numpy as np
|
||||
import scipy.sparse as sp
|
||||
|
||||
from easygraph.classes.graph import Graph
|
||||
|
||||
from .graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from .utils import data_type_dict
|
||||
from .utils import download
|
||||
from .utils import extract_archive
|
||||
from .utils import tensor
|
||||
|
||||
|
||||
class AmazonPhotoDataset(EasyGraphBuiltinDataset):
|
||||
r"""Amazon Electronics Photo co-purchase graph dataset.
|
||||
|
||||
Nodes represent products, and edges link products frequently co-purchased.
|
||||
Node features are bag-of-words of product reviews. The task is to classify
|
||||
the product category.
|
||||
|
||||
Statistics:
|
||||
|
||||
- Nodes: 7,650
|
||||
- Edges: 119,081
|
||||
- Number of Classes: 8
|
||||
- Features: 745
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_dir : str, optional
|
||||
Raw file directory to download/contains the input data directory. Default: None
|
||||
force_reload : bool, optional
|
||||
Whether to reload the dataset. Default: False
|
||||
verbose : bool, optional
|
||||
Whether to print out progress information. Default: True
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~easygraph.Graph` object and returns
|
||||
a transformed version. The :class:`~easygraph.Graph` object will be
|
||||
transformed before every access.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from easygraph.datasets import AmazonPhotoDataset
|
||||
>>> dataset = AmazonPhotoDataset()
|
||||
>>> g = dataset[0]
|
||||
>>> print(g.number_of_nodes())
|
||||
>>> print(g.number_of_edges())
|
||||
>>> print(g.nodes[0]['feat'].shape)
|
||||
>>> print(g.nodes[0]['label'])
|
||||
>>> print(dataset.num_classes)
|
||||
"""
|
||||
|
||||
def __init__(self, raw_dir=None, force_reload=False, verbose=True, transform=None):
|
||||
name = "amazon_photo"
|
||||
url = "https://data.dgl.ai/dataset/amazon_co_buy_photo.zip"
|
||||
super(AmazonPhotoDataset, self).__init__(
|
||||
name=name,
|
||||
url=url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def process(self):
|
||||
path = os.path.join(self.raw_path, "amazon_co_buy_photo.npz")
|
||||
data = np.load(path)
|
||||
|
||||
adj = sp.csr_matrix(
|
||||
(data["adj_data"], data["adj_indices"], data["adj_indptr"]),
|
||||
shape=data["adj_shape"],
|
||||
)
|
||||
|
||||
features = sp.csr_matrix(
|
||||
(data["attr_data"], data["attr_indices"], data["attr_indptr"]),
|
||||
shape=data["attr_shape"],
|
||||
).todense()
|
||||
|
||||
labels = data["labels"]
|
||||
|
||||
g = eg.Graph()
|
||||
g.add_edges_from(list(zip(*adj.nonzero())))
|
||||
|
||||
for i in range(features.shape[0]):
|
||||
g.add_node(i, feat=np.array(features[i]).squeeze(), label=int(labels[i]))
|
||||
|
||||
self._g = g
|
||||
self._num_classes = len(np.unique(labels))
|
||||
|
||||
if self.verbose:
|
||||
print("Finished loading AmazonPhoto dataset.")
|
||||
print(f" NumNodes: {g.number_of_nodes()}")
|
||||
print(f" NumEdges: {g.number_of_edges()}")
|
||||
print(f" NumFeats: {features.shape[1]}")
|
||||
print(f" NumClasses: {self._num_classes}")
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "AmazonPhotoDataset only contains one graph"
|
||||
if self._g is None:
|
||||
raise ValueError("Graph has not been loaded or processed correctly.")
|
||||
return self._g if self._transform is None else self._transform(self._g)
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
|
||||
@property
|
||||
def num_classes(self):
|
||||
return self._num_classes
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Arxiv HEP-TH Citation Network
|
||||
|
||||
This dataset represents the citation network of preprints from the High Energy Physics - Theory (HEP-TH) category on arXiv, covering the period from January 1993 to April 2003.
|
||||
|
||||
Each node corresponds to a paper, and a directed edge from paper A to paper B indicates that A cites B.
|
||||
|
||||
No features or labels are included in this dataset.
|
||||
|
||||
Statistics:
|
||||
- Nodes: 27,770
|
||||
- Edges: 352,807
|
||||
- Features: None
|
||||
- Labels: None
|
||||
|
||||
Reference:
|
||||
J. Leskovec, J. Kleinberg and C. Faloutsos, "Graphs over Time: Densification Laws, Shrinking Diameters and Possible Explanations,"
|
||||
in KDD 2005. Dataset: https://snap.stanford.edu/data/cit-HepTh.html
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import easygraph as eg
|
||||
|
||||
from easygraph.classes.graph import Graph
|
||||
|
||||
from .graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from .utils import download
|
||||
|
||||
|
||||
class ArxivHEPTHDataset(EasyGraphBuiltinDataset):
|
||||
r"""Arxiv HEP-TH citation network dataset.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_dir : str, optional
|
||||
Directory to store the raw downloaded files. Default: None
|
||||
force_reload : bool, optional
|
||||
Whether to re-download and process the dataset. Default: False
|
||||
verbose : bool, optional
|
||||
Whether to print detailed processing logs. Default: True
|
||||
transform : callable, optional
|
||||
Optional transform to apply on the graph.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from easygraph.datasets import ArxivHEPTHDataset
|
||||
>>> dataset = ArxivHEPTHDataset()
|
||||
>>> g = dataset[0]
|
||||
>>> print("Nodes:", g.number_of_nodes())
|
||||
>>> print("Edges:", g.number_of_edges())
|
||||
"""
|
||||
|
||||
def __init__(self, raw_dir=None, force_reload=False, verbose=True, transform=None):
|
||||
name = "cit-HepTh"
|
||||
url = "https://snap.stanford.edu/data/cit-HepTh.txt.gz"
|
||||
super(ArxivHEPTHDataset, self).__init__(
|
||||
name=name,
|
||||
url=url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def download(self):
|
||||
r"""Download and decompress the .txt.gz file."""
|
||||
compressed_path = os.path.join(self.raw_dir, self.name + ".txt.gz")
|
||||
extracted_path = os.path.join(self.raw_path, self.name + ".txt")
|
||||
|
||||
download(self.url, path=compressed_path)
|
||||
|
||||
if not os.path.exists(self.raw_path):
|
||||
os.makedirs(self.raw_path)
|
||||
|
||||
with gzip.open(compressed_path, "rb") as f_in:
|
||||
with open(extracted_path, "wb") as f_out:
|
||||
shutil.copyfileobj(f_in, f_out)
|
||||
|
||||
def process(self):
|
||||
graph = eg.DiGraph() # Citation network is directed
|
||||
edge_list_path = os.path.join(self.raw_path, self.name + ".txt")
|
||||
|
||||
with open(edge_list_path, "r") as f:
|
||||
for line in f:
|
||||
if line.startswith("#") or line.strip() == "":
|
||||
continue
|
||||
u, v = map(int, line.strip().split())
|
||||
graph.add_edge(u, v)
|
||||
|
||||
self._g = graph
|
||||
self._num_nodes = graph.number_of_nodes()
|
||||
self._num_edges = graph.number_of_edges()
|
||||
|
||||
if self.verbose:
|
||||
print("Finished loading Arxiv HEP-TH dataset.")
|
||||
print(f" NumNodes: {self._num_nodes}")
|
||||
print(f" NumEdges: {self._num_edges}")
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "ArxivHEPTHDataset only contains one graph"
|
||||
return self._g if self._transform is None else self._transform(self._g)
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
@@ -0,0 +1,875 @@
|
||||
"""Cora, citeseer, pubmed dataset."""
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import pickle as pkl
|
||||
import sys
|
||||
|
||||
import easygraph as eg
|
||||
import numpy as np
|
||||
import scipy.sparse as sp
|
||||
|
||||
from easygraph.classes.graph import Graph
|
||||
|
||||
from .graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from .utils import _get_dgl_url
|
||||
from .utils import data_type_dict
|
||||
from .utils import deprecate_property
|
||||
from .utils import generate_mask_tensor
|
||||
from .utils import nonzero_1d
|
||||
from .utils import tensor
|
||||
|
||||
|
||||
def _pickle_load(pkl_file):
|
||||
if sys.version_info > (3, 0):
|
||||
return pkl.load(pkl_file, encoding="latin1")
|
||||
else:
|
||||
return pkl.load(pkl_file)
|
||||
|
||||
|
||||
class CitationGraphDataset(EasyGraphBuiltinDataset):
|
||||
r"""The citation graph dataset, including Cora, CiteSeer and PubMed.
|
||||
Nodes mean authors and edges mean citation relationships.
|
||||
|
||||
Parameters
|
||||
-----------
|
||||
name: str
|
||||
name can be 'Cora', 'CiteSeer' or 'PubMed'.
|
||||
raw_dir : str
|
||||
Raw file directory to download/contains the input data directory.
|
||||
Default: ~/.dgl/
|
||||
force_reload : bool
|
||||
Whether to reload the dataset. Default: False
|
||||
verbose : bool
|
||||
Whether to print out progress information. Default: True.
|
||||
reverse_edge : bool
|
||||
Whether to add reverse edges in graph. Default: True.
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~eg.Graph` object and returns
|
||||
a transformed version. The :class:`~eg.Graph` object will be
|
||||
transformed before every access.
|
||||
reorder : bool
|
||||
Whether to reorder the graph using :func:`~eg.reorder_graph`. Default: False.
|
||||
"""
|
||||
|
||||
_urls = {
|
||||
"cora_v2": "dataset/cora_v2.zip",
|
||||
"citeseer": "dataset/citeseer.zip",
|
||||
"pubmed": "dataset/pubmed.zip",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name,
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=True,
|
||||
reverse_edge=True,
|
||||
transform=None,
|
||||
reorder=False,
|
||||
):
|
||||
assert name.lower() in ["cora", "citeseer", "pubmed"]
|
||||
|
||||
# Previously we use the pre-processing in pygcn (https://github.com/tkipf/pygcn)
|
||||
# for Cora, which is slightly different from the one used in the GCN paper
|
||||
if name.lower() == "cora":
|
||||
name = "cora_v2"
|
||||
|
||||
url = _get_dgl_url(self._urls[name])
|
||||
self._reverse_edge = reverse_edge
|
||||
self._reorder = reorder
|
||||
|
||||
super(CitationGraphDataset, self).__init__(
|
||||
name,
|
||||
url=url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def process(self):
|
||||
"""Loads input data from data directory and reorder graph for better locality
|
||||
|
||||
ind.name.x => the feature vectors of the training instances as scipy.sparse.csr.csr_matrix object;
|
||||
ind.name.tx => the feature vectors of the test instances as scipy.sparse.csr.csr_matrix object;
|
||||
ind.name.allx => the feature vectors of both labeled and unlabeled training instances
|
||||
(a superset of ind.name.x) as scipy.sparse.csr.csr_matrix object;
|
||||
ind.name.y => the one-hot labels of the labeled training instances as numpy.ndarray object;
|
||||
ind.name.ty => the one-hot labels of the test instances as numpy.ndarray object;
|
||||
ind.name.ally => the labels for instances in ind.name.allx as numpy.ndarray object;
|
||||
ind.name.graph => a dict in the format {index: [index_of_neighbor_nodes]} as collections.defaultdict
|
||||
object;
|
||||
ind.name.test.index => the indices of test instances in graph, for the inductive setting as list object.
|
||||
"""
|
||||
root = self.raw_path
|
||||
objnames = ["x", "y", "tx", "ty", "allx", "ally", "graph"]
|
||||
objects = []
|
||||
for i in range(len(objnames)):
|
||||
with open("{}/ind.{}.{}".format(root, self.name, objnames[i]), "rb") as f:
|
||||
objects.append(_pickle_load(f))
|
||||
|
||||
x, y, tx, ty, allx, ally, graph = tuple(objects)
|
||||
test_idx_reorder = _parse_index_file(
|
||||
"{}/ind.{}.test.index".format(root, self.name)
|
||||
)
|
||||
test_idx_range = np.sort(test_idx_reorder)
|
||||
|
||||
if self.name == "citeseer":
|
||||
# Fix CiteSeer dataset (there are some isolated nodes in the graph)
|
||||
# Find isolated nodes, add them as zero-vecs into the right position
|
||||
test_idx_range_full = range(
|
||||
min(test_idx_reorder), max(test_idx_reorder) + 1
|
||||
)
|
||||
tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1]))
|
||||
tx_extended[test_idx_range - min(test_idx_range), :] = tx
|
||||
tx = tx_extended
|
||||
ty_extended = np.zeros((len(test_idx_range_full), y.shape[1]))
|
||||
ty_extended[test_idx_range - min(test_idx_range), :] = ty
|
||||
ty = ty_extended
|
||||
|
||||
features = sp.vstack((allx, tx)).tolil()
|
||||
features[test_idx_reorder, :] = features[test_idx_range, :]
|
||||
|
||||
if self.reverse_edge:
|
||||
g = eg.DiGraph(eg.from_dict_of_lists(graph))
|
||||
# g = from_networkx(graph)
|
||||
else:
|
||||
graph = eg.Graph(eg.from_dict_of_lists(graph))
|
||||
# edges = list(graph.edges())
|
||||
# u, v = map(list, zip(*edges))
|
||||
# g = dgl_graph((u, v))
|
||||
|
||||
onehot_labels = np.vstack((ally, ty))
|
||||
onehot_labels[test_idx_reorder, :] = onehot_labels[test_idx_range, :]
|
||||
labels = np.argmax(onehot_labels, 1)
|
||||
|
||||
idx_test = test_idx_range.tolist()
|
||||
idx_train = range(len(y))
|
||||
idx_val = range(len(y), len(y) + 500)
|
||||
|
||||
train_mask = generate_mask_tensor(_sample_mask(idx_train, labels.shape[0]))
|
||||
val_mask = generate_mask_tensor(_sample_mask(idx_val, labels.shape[0]))
|
||||
test_mask = generate_mask_tensor(_sample_mask(idx_test, labels.shape[0]))
|
||||
|
||||
g.ndata["train_mask"] = train_mask
|
||||
g.ndata["val_mask"] = val_mask
|
||||
g.ndata["test_mask"] = test_mask
|
||||
g.ndata["label"] = tensor(labels)
|
||||
g.ndata["feat"] = tensor(
|
||||
_preprocess_features(features), dtype=data_type_dict()["float32"]
|
||||
)
|
||||
self._num_classes = onehot_labels.shape[1]
|
||||
self._labels = labels
|
||||
# if self._reorder:
|
||||
# self._g = reorder_graph(
|
||||
# g, node_permute_algo='rcmk', edge_permute_algo='dst', store_ids=False)
|
||||
# else:
|
||||
self._g = g
|
||||
|
||||
if self.verbose:
|
||||
print("Finished data loading and preprocessing.")
|
||||
print(" NumNodes: {}".format(self._g.number_of_nodes()))
|
||||
print(" NumEdges: {}".format(self._g.number_of_edges()))
|
||||
print(" NumFeats: {}".format(self._g.ndata["feat"].shape[1]))
|
||||
print(" NumClasses: {}".format(self.num_classes))
|
||||
print(
|
||||
" NumTrainingSamples: {}".format(
|
||||
nonzero_1d(self._g.ndata["train_mask"]).shape[0]
|
||||
)
|
||||
)
|
||||
print(
|
||||
" NumValidationSamples: {}".format(
|
||||
nonzero_1d(self._g.ndata["val_mask"]).shape[0]
|
||||
)
|
||||
)
|
||||
print(
|
||||
" NumTestSamples: {}".format(
|
||||
nonzero_1d(self._g.ndata["test_mask"]).shape[0]
|
||||
)
|
||||
)
|
||||
|
||||
def has_cache(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".bin")
|
||||
info_path = os.path.join(self.save_path, self.save_name + ".pkl")
|
||||
if os.path.exists(graph_path) and os.path.exists(info_path):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
# def save(self):
|
||||
# """save the graph list and the labels"""
|
||||
# graph_path = os.path.join(self.save_path,
|
||||
# self.save_name + '.bin')
|
||||
# info_path = os.path.join(self.save_path,
|
||||
# self.save_name + '.pkl')
|
||||
# save_graphs(str(graph_path), self._g)
|
||||
# save_info(str(info_path), {'num_classes': self.num_classes})
|
||||
#
|
||||
# def load(self):
|
||||
# graph_path = os.path.join(self.save_path,
|
||||
# self.save_name + '.bin')
|
||||
# info_path = os.path.join(self.save_path,
|
||||
# self.save_name + '.pkl')
|
||||
# graphs, _ = load_graphs(str(graph_path))
|
||||
#
|
||||
# info = load_info(str(info_path))
|
||||
# graph = graphs[0]
|
||||
# self._g = graph
|
||||
# # for compatibility
|
||||
# graph = graph.clone()
|
||||
# graph.ndata.pop('train_mask')
|
||||
# graph.ndata.pop('val_mask')
|
||||
# graph.ndata.pop('test_mask')
|
||||
# graph.ndata.pop('feat')
|
||||
# graph.ndata.pop('label')
|
||||
# graph = to_networkx(graph)
|
||||
#
|
||||
# self._num_classes = info['num_classes']
|
||||
# self._g.ndata['train_mask'] = generate_mask_tensor(F.asnumpy(self._g.ndata['train_mask']))
|
||||
# self._g.ndata['val_mask'] = generate_mask_tensor(F.asnumpy(self._g.ndata['val_mask']))
|
||||
# self._g.ndata['test_mask'] = generate_mask_tensor(F.asnumpy(self._g.ndata['test_mask']))
|
||||
# # hack for mxnet compatibility
|
||||
#
|
||||
# if self.verbose:
|
||||
# print(' NumNodes: {}'.format(self._g.number_of_nodes()))
|
||||
# print(' NumEdges: {}'.format(self._g.number_of_edges()))
|
||||
# print(' NumFeats: {}'.format(self._g.ndata['feat'].shape[1]))
|
||||
# print(' NumClasses: {}'.format(self.num_classes))
|
||||
# print(' NumTrainingSamples: {}'.format(
|
||||
# F.nonzero_1d(self._g.ndata['train_mask']).shape[0]))
|
||||
# print(' NumValidationSamples: {}'.format(
|
||||
# F.nonzero_1d(self._g.ndata['val_mask']).shape[0]))
|
||||
# print(' NumTestSamples: {}'.format(
|
||||
# F.nonzero_1d(self._g.ndata['test_mask']).shape[0]))
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "This dataset has only one graph"
|
||||
if self._transform is None:
|
||||
return self._g
|
||||
else:
|
||||
return self._transform(self._g)
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
|
||||
@property
|
||||
def save_name(self):
|
||||
return self.name + "_dgl_graph"
|
||||
|
||||
@property
|
||||
def num_labels(self):
|
||||
deprecate_property("dataset.num_labels", "dataset.num_classes")
|
||||
return self.num_classes
|
||||
|
||||
@property
|
||||
def num_classes(self):
|
||||
return self._num_classes
|
||||
|
||||
""" Citation graph is used in many examples
|
||||
We preserve these properties for compatibility.
|
||||
"""
|
||||
|
||||
@property
|
||||
def reverse_edge(self):
|
||||
return self._reverse_edge
|
||||
|
||||
|
||||
def _preprocess_features(features):
|
||||
"""Row-normalize feature matrix and convert to tuple representation"""
|
||||
rowsum = np.asarray(features.sum(1))
|
||||
r_inv = np.power(rowsum, -1).flatten()
|
||||
r_inv[np.isinf(r_inv)] = 0.0
|
||||
r_mat_inv = sp.diags(r_inv)
|
||||
features = r_mat_inv.dot(features)
|
||||
return np.asarray(features.todense())
|
||||
|
||||
|
||||
def _parse_index_file(filename):
|
||||
"""Parse index file."""
|
||||
index = []
|
||||
for line in open(filename):
|
||||
index.append(int(line.strip()))
|
||||
return index
|
||||
|
||||
|
||||
def _sample_mask(idx, l):
|
||||
"""Create mask."""
|
||||
mask = np.zeros(l)
|
||||
mask[idx] = 1
|
||||
return mask
|
||||
|
||||
|
||||
class CoraGraphDataset(CitationGraphDataset):
|
||||
r"""Cora citation network dataset.
|
||||
|
||||
Nodes mean paper and edges mean citation
|
||||
relationships. Each node has a predefined
|
||||
feature with 1433 dimensions. The dataset is
|
||||
designed for the node classification task.
|
||||
The task is to predict the category of
|
||||
certain paper.
|
||||
|
||||
Statistics:
|
||||
|
||||
- Nodes: 2708
|
||||
- Edges: 10556
|
||||
- Number of Classes: 7
|
||||
- Label split:
|
||||
|
||||
- Train: 140
|
||||
- Valid: 500
|
||||
- Test: 1000
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_dir : str
|
||||
Raw file directory to download/contains the input data directory.
|
||||
Default: ~/.dgl/
|
||||
force_reload : bool
|
||||
Whether to reload the dataset. Default: False
|
||||
verbose : bool
|
||||
Whether to print out progress information. Default: True.
|
||||
reverse_edge : bool
|
||||
Whether to add reverse edges in graph. Default: True.
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~dgl.DGLGraph` object and returns
|
||||
a transformed version. The :class:`~dgl.DGLGraph` object will be
|
||||
transformed before every access.
|
||||
reorder : bool
|
||||
Whether to reorder the graph using :func:`~dgl.reorder_graph`. Default: False.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
num_classes: int
|
||||
Number of label classes
|
||||
|
||||
Notes
|
||||
-----
|
||||
The node feature is row-normalized.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> dataset = CoraGraphDataset()
|
||||
>>> g = dataset[0]
|
||||
>>> num_class = dataset.num_classes
|
||||
>>>
|
||||
>>> # get node feature
|
||||
>>> feat = g.ndata['feat']
|
||||
>>>
|
||||
>>> # get data split
|
||||
>>> train_mask = g.ndata['train_mask']
|
||||
>>> val_mask = g.ndata['val_mask']
|
||||
>>> test_mask = g.ndata['test_mask']
|
||||
>>>
|
||||
>>> # get labels
|
||||
>>> label = g.ndata['label']
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=True,
|
||||
reverse_edge=True,
|
||||
transform=None,
|
||||
reorder=False,
|
||||
):
|
||||
name = "cora"
|
||||
|
||||
super(CoraGraphDataset, self).__init__(
|
||||
name, raw_dir, force_reload, verbose, reverse_edge, transform, reorder
|
||||
)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
r"""Gets the graph object
|
||||
|
||||
Parameters
|
||||
-----------
|
||||
idx: int
|
||||
Item index, CoraGraphDataset has only one graph object
|
||||
|
||||
Return
|
||||
------
|
||||
:class:`dgl.DGLGraph`
|
||||
|
||||
graph structure, node features and labels.
|
||||
|
||||
- ``ndata['train_mask']``: mask for training node set
|
||||
- ``ndata['val_mask']``: mask for validation node set
|
||||
- ``ndata['test_mask']``: mask for test node set
|
||||
- ``ndata['feat']``: node feature
|
||||
- ``ndata['label']``: ground truth labels
|
||||
"""
|
||||
return super(CoraGraphDataset, self).__getitem__(idx)
|
||||
|
||||
def __len__(self):
|
||||
r"""The number of graphs in the dataset."""
|
||||
return super(CoraGraphDataset, self).__len__()
|
||||
|
||||
|
||||
class CiteseerGraphDataset(CitationGraphDataset):
|
||||
r"""Citeseer citation network dataset.
|
||||
|
||||
Nodes mean scientific publications and edges
|
||||
mean citation relationships. Each node has a
|
||||
predefined feature with 3703 dimensions. The
|
||||
dataset is designed for the node classification
|
||||
task. The task is to predict the category of
|
||||
certain publication.
|
||||
|
||||
Statistics:
|
||||
|
||||
- Nodes: 3327
|
||||
- Edges: 9228
|
||||
- Number of Classes: 6
|
||||
- Label Split:
|
||||
|
||||
- Train: 120
|
||||
- Valid: 500
|
||||
- Test: 1000
|
||||
|
||||
Parameters
|
||||
-----------
|
||||
raw_dir : str
|
||||
Raw file directory to download/contains the input data directory.
|
||||
Default: ~/.dgl/
|
||||
force_reload : bool
|
||||
Whether to reload the dataset. Default: False
|
||||
verbose : bool
|
||||
Whether to print out progress information. Default: True.
|
||||
reverse_edge : bool
|
||||
Whether to add reverse edges in graph. Default: True.
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~dgl.DGLGraph` object and returns
|
||||
a transformed version. The :class:`~dgl.DGLGraph` object will be
|
||||
transformed before every access.
|
||||
reorder : bool
|
||||
Whether to reorder the graph using :func:`~dgl.reorder_graph`. Default: False.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
num_classes: int
|
||||
Number of label classes
|
||||
|
||||
Notes
|
||||
-----
|
||||
The node feature is row-normalized.
|
||||
|
||||
In citeseer dataset, there are some isolated nodes in the graph.
|
||||
These isolated nodes are added as zero-vecs into the right position.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> dataset = CiteseerGraphDataset()
|
||||
>>> g = dataset[0]
|
||||
>>> num_class = dataset.num_classes
|
||||
>>>
|
||||
>>> # get node feature
|
||||
>>> feat = g.ndata['feat']
|
||||
>>>
|
||||
>>> # get data split
|
||||
>>> train_mask = g.ndata['train_mask']
|
||||
>>> val_mask = g.ndata['val_mask']
|
||||
>>> test_mask = g.ndata['test_mask']
|
||||
>>>
|
||||
>>> # get labels
|
||||
>>> label = g.ndata['label']
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=True,
|
||||
reverse_edge=True,
|
||||
transform=None,
|
||||
reorder=False,
|
||||
):
|
||||
name = "citeseer"
|
||||
|
||||
super(CiteseerGraphDataset, self).__init__(
|
||||
name, raw_dir, force_reload, verbose, reverse_edge, transform, reorder
|
||||
)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
r"""Gets the graph object
|
||||
|
||||
Parameters
|
||||
-----------
|
||||
idx: int
|
||||
Item index, CiteseerGraphDataset has only one graph object
|
||||
|
||||
Return
|
||||
------
|
||||
:class:`dgl.DGLGraph`
|
||||
|
||||
graph structure, node features and labels.
|
||||
|
||||
- ``ndata['train_mask']``: mask for training node set
|
||||
- ``ndata['val_mask']``: mask for validation node set
|
||||
- ``ndata['test_mask']``: mask for test node set
|
||||
- ``ndata['feat']``: node feature
|
||||
- ``ndata['label']``: ground truth labels
|
||||
"""
|
||||
return super(CiteseerGraphDataset, self).__getitem__(idx)
|
||||
|
||||
def __len__(self):
|
||||
r"""The number of graphs in the dataset."""
|
||||
return super(CiteseerGraphDataset, self).__len__()
|
||||
|
||||
|
||||
class PubmedGraphDataset(CitationGraphDataset):
|
||||
r"""Pubmed citation network dataset.
|
||||
|
||||
Nodes mean scientific publications and edges
|
||||
mean citation relationships. Each node has a
|
||||
predefined feature with 500 dimensions. The
|
||||
dataset is designed for the node classification
|
||||
task. The task is to predict the category of
|
||||
certain publication.
|
||||
|
||||
Statistics:
|
||||
|
||||
- Nodes: 19717
|
||||
- Edges: 88651
|
||||
- Number of Classes: 3
|
||||
- Label Split:
|
||||
|
||||
- Train: 60
|
||||
- Valid: 500
|
||||
- Test: 1000
|
||||
|
||||
Parameters
|
||||
-----------
|
||||
raw_dir : str
|
||||
Raw file directory to download/contains the input data directory.
|
||||
Default: ~/.dgl/
|
||||
force_reload : bool
|
||||
Whether to reload the dataset. Default: False
|
||||
verbose : bool
|
||||
Whether to print out progress information. Default: True.
|
||||
reverse_edge : bool
|
||||
Whether to add reverse edges in graph. Default: True.
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~dgl.DGLGraph` object and returns
|
||||
a transformed version. The :class:`~dgl.DGLGraph` object will be
|
||||
transformed before every access.
|
||||
reorder : bool
|
||||
Whether to reorder the graph using :func:`~dgl.reorder_graph`. Default: False.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
num_classes: int
|
||||
Number of label classes
|
||||
|
||||
Notes
|
||||
-----
|
||||
The node feature is row-normalized.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> dataset = PubmedGraphDataset()
|
||||
>>> g = dataset[0]
|
||||
>>> num_class = dataset.num_of_class
|
||||
>>>
|
||||
>>> # get node feature
|
||||
>>> feat = g.ndata['feat']
|
||||
>>>
|
||||
>>> # get data split
|
||||
>>> train_mask = g.ndata['train_mask']
|
||||
>>> val_mask = g.ndata['val_mask']
|
||||
>>> test_mask = g.ndata['test_mask']
|
||||
>>>
|
||||
>>> # get labels
|
||||
>>> label = g.ndata['label']
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=True,
|
||||
reverse_edge=True,
|
||||
transform=None,
|
||||
reorder=False,
|
||||
):
|
||||
name = "pubmed"
|
||||
|
||||
super(PubmedGraphDataset, self).__init__(
|
||||
name, raw_dir, force_reload, verbose, reverse_edge, transform, reorder
|
||||
)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
r"""Gets the graph object
|
||||
|
||||
Parameters
|
||||
-----------
|
||||
idx: int
|
||||
Item index, PubmedGraphDataset has only one graph object
|
||||
|
||||
Return
|
||||
------
|
||||
:class:`dgl.DGLGraph`
|
||||
|
||||
graph structure, node features and labels.
|
||||
|
||||
- ``ndata['train_mask']``: mask for training node set
|
||||
- ``ndata['val_mask']``: mask for validation node set
|
||||
- ``ndata['test_mask']``: mask for test node set
|
||||
- ``ndata['feat']``: node feature
|
||||
- ``ndata['label']``: ground truth labels
|
||||
"""
|
||||
return super(PubmedGraphDataset, self).__getitem__(idx)
|
||||
|
||||
def __len__(self):
|
||||
r"""The number of graphs in the dataset."""
|
||||
return super(PubmedGraphDataset, self).__len__()
|
||||
|
||||
|
||||
def load_cora(
|
||||
raw_dir=None, force_reload=False, verbose=True, reverse_edge=True, transform=None
|
||||
):
|
||||
"""Get CoraGraphDataset
|
||||
|
||||
Parameters
|
||||
-----------
|
||||
raw_dir : str
|
||||
Raw file directory to download/contains the input data directory.
|
||||
Default: ~/.dgl/
|
||||
force_reload : bool
|
||||
Whether to reload the dataset. Default: False
|
||||
verbose : bool
|
||||
Whether to print out progress information. Default: True.
|
||||
reverse_edge : bool
|
||||
Whether to add reverse edges in graph. Default: True.
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~dgl.DGLGraph` object and returns
|
||||
a transformed version. The :class:`~dgl.DGLGraph` object will be
|
||||
transformed before every access.
|
||||
|
||||
Return
|
||||
-------
|
||||
CoraGraphDataset
|
||||
"""
|
||||
data = CoraGraphDataset(raw_dir, force_reload, verbose, reverse_edge, transform)
|
||||
return data
|
||||
|
||||
|
||||
def load_citeseer(
|
||||
raw_dir=None, force_reload=False, verbose=True, reverse_edge=True, transform=None
|
||||
):
|
||||
"""Get CiteseerGraphDataset
|
||||
|
||||
Parameters
|
||||
-----------
|
||||
raw_dir : str
|
||||
Raw file directory to download/contains the input data directory.
|
||||
Default: ~/.dgl/
|
||||
force_reload : bool
|
||||
Whether to reload the dataset. Default: False
|
||||
verbose : bool
|
||||
Whether to print out progress information. Default: True.
|
||||
reverse_edge : bool
|
||||
Whether to add reverse edges in graph. Default: True.
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~dgl.DGLGraph` object and returns
|
||||
a transformed version. The :class:`~dgl.DGLGraph` object will be
|
||||
transformed before every access.
|
||||
|
||||
Return
|
||||
-------
|
||||
CiteseerGraphDataset
|
||||
"""
|
||||
data = CiteseerGraphDataset(raw_dir, force_reload, verbose, reverse_edge, transform)
|
||||
return data
|
||||
|
||||
|
||||
def load_pubmed(
|
||||
raw_dir=None, force_reload=False, verbose=True, reverse_edge=True, transform=None
|
||||
):
|
||||
"""Get PubmedGraphDataset
|
||||
|
||||
Parameters
|
||||
-----------
|
||||
raw_dir : str
|
||||
Raw file directory to download/contains the input data directory.
|
||||
Default: ~/.dgl/
|
||||
force_reload : bool
|
||||
Whether to reload the dataset. Default: False
|
||||
verbose : bool
|
||||
Whether to print out progress information. Default: True.
|
||||
reverse_edge : bool
|
||||
Whether to add reverse edges in graph. Default: True.
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~dgl.DGLGraph` object and returns
|
||||
a transformed version. The :class:`~dgl.DGLGraph` object will be
|
||||
transformed before every access.
|
||||
|
||||
Return
|
||||
-------
|
||||
PubmedGraphDataset
|
||||
"""
|
||||
data = PubmedGraphDataset(raw_dir, force_reload, verbose, reverse_edge, transform)
|
||||
return data
|
||||
|
||||
|
||||
class CoraBinary(EasyGraphBuiltinDataset):
|
||||
"""A mini-dataset for binary classification task using Cora.
|
||||
|
||||
After loaded, it has following members:
|
||||
|
||||
graphs : list of :class:`~dgl.DGLGraph`
|
||||
pmpds : list of :class:`scipy.sparse.coo_matrix`
|
||||
labels : list of :class:`numpy.ndarray`
|
||||
|
||||
Parameters
|
||||
-----------
|
||||
raw_dir : str
|
||||
Raw file directory to download/contains the input data directory.
|
||||
Default: ~/.dgl/
|
||||
force_reload : bool
|
||||
Whether to reload the dataset. Default: False
|
||||
verbose: bool
|
||||
Whether to print out progress information. Default: True.
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~dgl.DGLGraph` object and returns
|
||||
a transformed version. The :class:`~dgl.DGLGraph` object will be
|
||||
transformed before every access.
|
||||
"""
|
||||
|
||||
def __init__(self, raw_dir=None, force_reload=False, verbose=True, transform=None):
|
||||
name = "cora_binary"
|
||||
url = _get_dgl_url("dataset/cora_binary.zip")
|
||||
super(CoraBinary, self).__init__(
|
||||
name,
|
||||
url=url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def process(self):
|
||||
root = self.raw_path
|
||||
# load graphs
|
||||
self.graphs = []
|
||||
with open("{}/graphs.txt".format(root), "r") as f:
|
||||
elist = []
|
||||
for line in f.readlines():
|
||||
if line.startswith("graph"):
|
||||
if len(elist) != 0:
|
||||
self.graphs.append(Graph(elist))
|
||||
elist = []
|
||||
else:
|
||||
u, v = line.strip().split(" ")
|
||||
elist.append((int(u), int(v)))
|
||||
if len(elist) != 0:
|
||||
self.graphs.append(Graph(tuple(zip(*elist))))
|
||||
with open("{}/pmpds.pkl".format(root), "rb") as f:
|
||||
self.pmpds = _pickle_load(f)
|
||||
self.labels = []
|
||||
with open("{}/labels.txt".format(root), "r") as f:
|
||||
cur = []
|
||||
for line in f.readlines():
|
||||
if line.startswith("graph"):
|
||||
if len(cur) != 0:
|
||||
self.labels.append(np.asarray(cur))
|
||||
cur = []
|
||||
else:
|
||||
cur.append(int(line.strip()))
|
||||
if len(cur) != 0:
|
||||
self.labels.append(np.asarray(cur))
|
||||
# sanity check
|
||||
assert len(self.graphs) == len(self.pmpds)
|
||||
assert len(self.graphs) == len(self.labels)
|
||||
|
||||
def has_cache(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".bin")
|
||||
if os.path.exists(graph_path):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
# def save(self):
|
||||
# """save the graph list and the labels"""
|
||||
# graph_path = os.path.join(self.save_path,
|
||||
# self.save_name + '.bin')
|
||||
# labels = {}
|
||||
# for i, label in enumerate(self.labels):
|
||||
# labels['{}'.format(i)] = F.tensor(label)
|
||||
# save_graphs(str(graph_path), self.graphs, labels)
|
||||
# if self.verbose:
|
||||
# print('Done saving data into cached files.')
|
||||
#
|
||||
# def load(self):
|
||||
# graph_path = os.path.join(self.save_path,
|
||||
# self.save_name + '.bin')
|
||||
# self.graphs, labels = load_graphs(str(graph_path))
|
||||
#
|
||||
# self.labels = []
|
||||
# for i in range(len(labels)):
|
||||
# self.labels.append(F.asnumpy(labels['{}'.format(i)]))
|
||||
# # load pmpds under self.raw_path
|
||||
# with open("{}/pmpds.pkl".format(self.raw_path), 'rb') as f:
|
||||
# self.pmpds = _pickle_load(f)
|
||||
# if self.verbose:
|
||||
# print('Done loading data into cached files.')
|
||||
# # sanity check
|
||||
# assert len(self.graphs) == len(self.pmpds)
|
||||
# assert len(self.graphs) == len(self.labels)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.graphs)
|
||||
|
||||
def __getitem__(self, i):
|
||||
r"""Gets the idx-th sample.
|
||||
|
||||
Parameters
|
||||
-----------
|
||||
idx : int
|
||||
The sample index.
|
||||
|
||||
Returns
|
||||
-------
|
||||
(dgl.DGLGraph, scipy.sparse.coo_matrix, int)
|
||||
The graph, scipy sparse coo_matrix and its label.
|
||||
"""
|
||||
if self._transform is None:
|
||||
g = self.graphs[i]
|
||||
else:
|
||||
g = self._transform(self.graphs[i])
|
||||
return (g, self.pmpds[i], self.labels[i])
|
||||
|
||||
@property
|
||||
def save_name(self):
|
||||
return self.name + "_dgl_graph"
|
||||
|
||||
# @staticmethod
|
||||
# def collate_fn(cur):
|
||||
# graphs, pmpds, labels = zip(*cur)
|
||||
# batched_graphs = batch.batch(graphs)
|
||||
# batched_pmpds = sp.block_diag(pmpds)
|
||||
# batched_labels = np.concatenate(labels, axis=0)
|
||||
# return batched_graphs, batched_pmpds, batched_labels
|
||||
|
||||
|
||||
def _normalize(mx):
|
||||
"""Row-normalize sparse matrix"""
|
||||
rowsum = np.asarray(mx.sum(1))
|
||||
r_inv = np.power(rowsum, -1).flatten()
|
||||
r_inv[np.isinf(r_inv)] = 0.0
|
||||
r_mat_inv = sp.diags(r_inv)
|
||||
mx = r_mat_inv.dot(mx)
|
||||
return mx
|
||||
|
||||
|
||||
def _encode_onehot(labels):
|
||||
classes = list(sorted(set(labels)))
|
||||
classes_dict = {c: np.identity(len(classes))[i, :] for i, c in enumerate(classes)}
|
||||
labels_onehot = np.asarray(list(map(classes_dict.get, labels)), dtype=np.int32)
|
||||
return labels_onehot
|
||||
@@ -0,0 +1,118 @@
|
||||
"""CoauthorCS Dataset
|
||||
|
||||
This dataset contains a co-authorship network of authors who submitted papers to CS category.
|
||||
Each node represents an author and edges represent co-authorships.
|
||||
Node features are bag-of-words representations of keywords in the author's papers.
|
||||
The task is node classification, with labels indicating the primary field of study.
|
||||
|
||||
Statistics:
|
||||
- Nodes: 18333
|
||||
- Edges: 81894
|
||||
- Feature Dim: 6805
|
||||
- Classes: 15
|
||||
|
||||
Source: https://github.com/dmlc/dgl/tree/master/examples/pytorch/cluster_gcn
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import easygraph as eg
|
||||
import numpy as np
|
||||
import scipy.sparse as sp
|
||||
|
||||
from easygraph.classes.graph import Graph
|
||||
|
||||
from .graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from .utils import data_type_dict
|
||||
from .utils import download
|
||||
from .utils import extract_archive
|
||||
from .utils import tensor
|
||||
|
||||
|
||||
class CoauthorCSDataset(EasyGraphBuiltinDataset):
|
||||
r"""CoauthorCS citation network dataset.
|
||||
|
||||
Nodes are authors, and edges indicate co-authorship relationships. Each node
|
||||
has a bag-of-words feature vector and a label denoting the primary research field.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_dir : str, optional
|
||||
Directory to store the raw downloaded files. Default: None
|
||||
force_reload : bool, optional
|
||||
Whether to re-download and process the dataset. Default: False
|
||||
verbose : bool, optional
|
||||
Whether to print detailed processing logs. Default: True
|
||||
transform : callable, optional
|
||||
Transform to apply to the graph on access.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from easygraph.datasets import CoauthorCSDataset
|
||||
>>> dataset = CoauthorCSDataset()
|
||||
>>> g = dataset[0]
|
||||
>>> print("Nodes:", g.number_of_nodes())
|
||||
>>> print("Edges:", g.number_of_edges())
|
||||
>>> print("Feature shape:", g.nodes[0]['feat'].shape)
|
||||
>>> print("Label:", g.nodes[0]['label'])
|
||||
>>> print("Number of classes:", dataset.num_classes)
|
||||
"""
|
||||
|
||||
def __init__(self, raw_dir=None, force_reload=False, verbose=True, transform=None):
|
||||
name = "coauthor_cs"
|
||||
url = "https://data.dgl.ai/dataset/coauthor_cs.zip"
|
||||
super(CoauthorCSDataset, self).__init__(
|
||||
name=name,
|
||||
url=url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def process(self):
|
||||
path = os.path.join(self.raw_path, "coauthor_cs.npz")
|
||||
data = np.load(path)
|
||||
|
||||
# Reconstruct adjacency matrix
|
||||
adj = sp.csr_matrix(
|
||||
(data["adj_data"], data["adj_indices"], data["adj_indptr"]),
|
||||
shape=data["adj_shape"],
|
||||
)
|
||||
|
||||
# Reconstruct feature matrix
|
||||
features = sp.csr_matrix(
|
||||
(data["attr_data"], data["attr_indices"], data["attr_indptr"]),
|
||||
shape=data["attr_shape"],
|
||||
).todense()
|
||||
|
||||
labels = data["labels"]
|
||||
|
||||
g = eg.Graph()
|
||||
g.add_edges_from(list(zip(*adj.nonzero())))
|
||||
|
||||
for i in range(features.shape[0]):
|
||||
g.add_node(i, feat=np.array(features[i]).squeeze(), label=int(labels[i]))
|
||||
|
||||
self._g = g
|
||||
self._num_classes = len(np.unique(labels))
|
||||
|
||||
if self.verbose:
|
||||
print("Finished loading CoauthorCS dataset.")
|
||||
print(f" NumNodes: {g.number_of_nodes()}")
|
||||
print(f" NumEdges: {g.number_of_edges()}")
|
||||
print(f" NumFeats: {features.shape[1]}")
|
||||
print(f" NumClasses: {self._num_classes}")
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "CoauthorCSDataset only contains one graph"
|
||||
if self._g is None:
|
||||
raise ValueError("Graph has not been loaded or processed correctly.")
|
||||
return self._g if self._transform is None else self._transform(self._g)
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
|
||||
@property
|
||||
def num_classes(self):
|
||||
return self._num_classes
|
||||
@@ -0,0 +1,4 @@
|
||||
from .email_enron import *
|
||||
from .email_eu import *
|
||||
from .hospital_lyon import *
|
||||
from .load_dataset import *
|
||||
@@ -0,0 +1,86 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
from easygraph.convert import dict_to_hypergraph
|
||||
from easygraph.datasets.dynamic.load_dataset import request_json_from_url
|
||||
from easygraph.datasets.graph_dataset_base import EasyGraphDataset
|
||||
from easygraph.datasets.utils import _get_eg_url
|
||||
from easygraph.datasets.utils import tensor
|
||||
|
||||
|
||||
class Email_Enron(EasyGraphDataset):
|
||||
_urls = {
|
||||
"email-enron": (
|
||||
"easygraph-data-email-enron/-/raw/main/email-enron.json?inline=false"
|
||||
),
|
||||
"email-eu": "easygraph-data-email-eu/-/raw/main/email-eu.json?inline=false",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=True,
|
||||
transform=None,
|
||||
save_dir="./",
|
||||
):
|
||||
name = "email-enron"
|
||||
self.url = _get_eg_url(self._urls[name])
|
||||
super(Email_Enron, self).__init__(
|
||||
name=name,
|
||||
url=self.url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
save_dir=save_dir,
|
||||
)
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return self._url
|
||||
|
||||
@property
|
||||
def save_name(self):
|
||||
return self.name
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "This dataset has only one graph"
|
||||
if self._transform is None:
|
||||
return self._g
|
||||
else:
|
||||
return self._transform(self._g)
|
||||
|
||||
def load(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".json")
|
||||
with open(graph_path, "r") as f:
|
||||
self.load_data = json.load(f)
|
||||
|
||||
def has_cache(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".json")
|
||||
if os.path.exists(graph_path):
|
||||
return True
|
||||
return False
|
||||
|
||||
def download(self):
|
||||
if self.has_cache():
|
||||
self.load()
|
||||
else:
|
||||
root = self.raw_dir
|
||||
data = request_json_from_url(self.url)
|
||||
with open(os.path.join(root, self.save_name + ".json"), "w") as f:
|
||||
json.dump(data, f)
|
||||
self.load_data = data
|
||||
|
||||
def process(self):
|
||||
"""Loads input data from data directory and transfer to target graph for better analysis"""
|
||||
|
||||
self._g, edge_feature_list = dict_to_hypergraph(self.load_data, is_dynamic=True)
|
||||
|
||||
self._g.ndata["hyperedge_feature"] = tensor(
|
||||
range(1, len(edge_feature_list) + 1)
|
||||
)
|
||||
|
||||
@url.setter
|
||||
def url(self, value):
|
||||
self._url = value
|
||||
@@ -0,0 +1,81 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
from easygraph.convert import dict_to_hypergraph
|
||||
from easygraph.datasets.dynamic.load_dataset import request_json_from_url
|
||||
from easygraph.datasets.graph_dataset_base import EasyGraphDataset
|
||||
from easygraph.datasets.utils import _get_eg_url
|
||||
from easygraph.datasets.utils import tensor
|
||||
|
||||
|
||||
class Email_Eu(EasyGraphDataset):
|
||||
_urls = {
|
||||
"email-eu": "easygraph-data-email-eu/-/raw/main/email-eu.json?inline=false",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=True,
|
||||
transform=None,
|
||||
save_dir="./",
|
||||
):
|
||||
name = "email-eu"
|
||||
self.url = _get_eg_url(self._urls[name])
|
||||
super(Email_Eu, self).__init__(
|
||||
name=name,
|
||||
url=self.url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
save_dir=save_dir,
|
||||
)
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return self._url
|
||||
|
||||
@property
|
||||
def save_name(self):
|
||||
return self.name
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "This dataset has only one graph"
|
||||
if self._transform is None:
|
||||
return self._g
|
||||
else:
|
||||
return self._transform(self._g)
|
||||
|
||||
def load(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".json")
|
||||
with open(graph_path, "r") as f:
|
||||
self.load_data = json.load(f)
|
||||
|
||||
def has_cache(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".json")
|
||||
if os.path.exists(graph_path):
|
||||
return True
|
||||
return False
|
||||
|
||||
def download(self):
|
||||
if self.has_cache():
|
||||
self.load()
|
||||
else:
|
||||
root = self.raw_dir
|
||||
data = request_json_from_url(self.url)
|
||||
with open(os.path.join(root, self.save_name + ".json"), "w") as f:
|
||||
json.dump(data, f)
|
||||
self.load_data = data
|
||||
|
||||
def process(self):
|
||||
"""Loads input data from data directory and transfer to target graph for better analysis"""
|
||||
self._g, edge_feature_list = dict_to_hypergraph(self.load_data, is_dynamic=True)
|
||||
self._g.ndata["hyperedge_feature"] = tensor(
|
||||
range(1, len(edge_feature_list) + 1)
|
||||
)
|
||||
|
||||
@url.setter
|
||||
def url(self, value):
|
||||
self._url = value
|
||||
@@ -0,0 +1,133 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
from easygraph.classes.hypergraph import Hypergraph
|
||||
from easygraph.datasets.dynamic.load_dataset import request_json_from_url
|
||||
from easygraph.datasets.graph_dataset_base import EasyGraphDataset
|
||||
from easygraph.datasets.utils import _get_eg_url
|
||||
from easygraph.datasets.utils import tensor
|
||||
|
||||
|
||||
class Hospital_Lyon(EasyGraphDataset):
|
||||
_urls = {
|
||||
"hospital_lyon": (
|
||||
"easygraph-data-hospital-lyon/-/raw/main/hospital-lyon.json?ref_type=heads&inline=false"
|
||||
),
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=True,
|
||||
transform=None,
|
||||
save_dir="./",
|
||||
):
|
||||
name = "hospital_lyon"
|
||||
self.url = _get_eg_url(self._urls[name])
|
||||
super(Hospital_Lyon, self).__init__(
|
||||
name=name,
|
||||
url=self.url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
save_dir=save_dir,
|
||||
)
|
||||
|
||||
def preprocess(self, data, max_order=None, is_dynamic=True):
|
||||
# The index of the nodes in this dataset are not continuous and therefore require special processing
|
||||
timestamp_lst = list()
|
||||
node_data = data["node-data"]
|
||||
node_num = len(node_data)
|
||||
G = Hypergraph(num_v=node_num)
|
||||
id = 0
|
||||
name_dict = {}
|
||||
for k, v in data["node-data"].items():
|
||||
name_dict[k] = id
|
||||
v["name"] = k
|
||||
G.v_property[id] = v
|
||||
id = id + 1
|
||||
e_property_dict = data["edge-data"]
|
||||
rows = []
|
||||
cols = []
|
||||
edge_flag_dict = {}
|
||||
edge_id = 0
|
||||
for id, edge in data["edge-dict"].items():
|
||||
if max_order and len(edge) > max_order + 1:
|
||||
continue
|
||||
|
||||
try:
|
||||
id = int(id)
|
||||
except ValueError as e:
|
||||
raise TypeError(
|
||||
f"Failed to convert the edge with ID {id} to type int."
|
||||
) from e
|
||||
|
||||
try:
|
||||
edge = [name_dict[n] for n in edge]
|
||||
rows.extend(edge)
|
||||
cols.extend(len(edge) * [edge_id])
|
||||
edge_id += 1
|
||||
except ValueError as e:
|
||||
raise TypeError(f"Failed to convert nodes to type int.") from e
|
||||
if is_dynamic:
|
||||
G.add_hyperedges(
|
||||
e_list=edge,
|
||||
e_property=e_property_dict[str(id)],
|
||||
group_name=e_property_dict[str(id)]["timestamp"],
|
||||
)
|
||||
timestamp_lst.append(e_property_dict[str(id)]["timestamp"])
|
||||
else:
|
||||
G.add_hyperedges(e_list=edge, e_property=e_property_dict[str(id)])
|
||||
G._rows = rows
|
||||
G._cols = cols
|
||||
return G, timestamp_lst
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return self._url
|
||||
|
||||
@property
|
||||
def save_name(self):
|
||||
return self.name
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "This dataset has only one graph"
|
||||
if self._transform is None:
|
||||
return self._g
|
||||
else:
|
||||
return self._transform(self._g)
|
||||
|
||||
def load(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".json")
|
||||
with open(graph_path, "r") as f:
|
||||
self.load_data = json.load(f)
|
||||
|
||||
def has_cache(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".json")
|
||||
if os.path.exists(graph_path):
|
||||
return True
|
||||
return False
|
||||
|
||||
def download(self):
|
||||
if self.has_cache():
|
||||
self.load()
|
||||
else:
|
||||
root = self.raw_dir
|
||||
data = request_json_from_url(self.url)
|
||||
with open(os.path.join(root, self.save_name + ".json"), "w") as f:
|
||||
json.dump(data, f)
|
||||
self.load_data = data
|
||||
|
||||
def process(self):
|
||||
"""Loads input data from data directory and transfer to target graph for better analysis"""
|
||||
|
||||
self._g, edge_feature_list = self.preprocess(self.load_data, is_dynamic=True)
|
||||
self._g.ndata["hyperedge_feature"] = tensor(
|
||||
range(1, len(edge_feature_list) + 1)
|
||||
)
|
||||
|
||||
@url.setter
|
||||
def url(self, value):
|
||||
self._url = value
|
||||
@@ -0,0 +1,94 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
from warnings import warn
|
||||
|
||||
import requests
|
||||
|
||||
from easygraph.convert import dict_to_hypergraph
|
||||
from easygraph.utils.exception import EasyGraphError
|
||||
|
||||
|
||||
__all__ = [
|
||||
"load_dynamic_hypergraph_dataset",
|
||||
]
|
||||
|
||||
dataset_index_url = "https://gitlab.com/easy-graph/easygraph-data/-/raw/main/dataset_index.json?inline=false"
|
||||
|
||||
|
||||
def request_json_from_url(url):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
except requests.ConnectionError:
|
||||
raise EasyGraphError("Connection Error!")
|
||||
|
||||
if r.ok:
|
||||
return r.json()
|
||||
else:
|
||||
raise EasyGraphError(f"Error: HTTP response {r.status_code}")
|
||||
|
||||
|
||||
def _request_from_eg_data(dataset=None, cache=True):
|
||||
"""Request a dataset from eg-data.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
dataset : str, optional
|
||||
Dataset name. Valid options are the top-level tags of the
|
||||
index.json file in the xgi-data repository. If None, prints
|
||||
the list of available datasets.
|
||||
cache : bool, optional
|
||||
Whether or not to cache the output
|
||||
|
||||
Returns
|
||||
-------
|
||||
Data
|
||||
The requested data loaded from a json file.
|
||||
|
||||
Raises
|
||||
------
|
||||
EasyGraphError
|
||||
If the HTTP request is not successful or the dataset does not exist.
|
||||
|
||||
|
||||
"""
|
||||
|
||||
index_data = request_json_from_url(dataset_index_url)
|
||||
|
||||
key = dataset.lower()
|
||||
if key not in index_data:
|
||||
print("Valid dataset names:")
|
||||
print(*index_data, sep="\n")
|
||||
raise EasyGraphError("Must choose a valid dataset name!")
|
||||
|
||||
return request_json_from_url(index_data[key]["url"])
|
||||
|
||||
|
||||
def load_dynamic_hypergraph_dataset(
|
||||
dataset=None,
|
||||
local_read=False,
|
||||
path="",
|
||||
max_order=None,
|
||||
):
|
||||
index_datasets = request_json_from_url(dataset_index_url)
|
||||
if dataset is None:
|
||||
print("Please refer to available list")
|
||||
|
||||
print(*index_datasets, sep="\n")
|
||||
return
|
||||
|
||||
if local_read:
|
||||
cfp = os.path.join(path, dataset + ".json")
|
||||
if os.path.exists(cfp):
|
||||
data = json.load(open(cfp, "r"))
|
||||
return dict_to_hypergraph(data, max_order=max_order)
|
||||
else:
|
||||
warn(
|
||||
f"No local copy was found at {cfp}. The data is requested "
|
||||
"from the xgi-data repository instead. To download a local "
|
||||
"copy, use `download_xgi_data`."
|
||||
)
|
||||
data = _request_from_eg_data(dataset)
|
||||
return dict_to_hypergraph(
|
||||
data, max_order=max_order, is_dynamic=index_datasets[dataset]["is_dynamic"]
|
||||
)
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Facebook Ego-Net Dataset
|
||||
|
||||
This dataset contains a subset of Facebook’s social network collected from
|
||||
survey participants in the SNAP EgoNet project. Nodes represent users, and
|
||||
edges indicate friendship links between them.
|
||||
|
||||
Each ego network is centered on a user and includes their friend connections
|
||||
and friend-to-friend connections. The `.circles` files contain labeled groups
|
||||
(i.e., communities) of friends identified by the ego user.
|
||||
|
||||
This version processes all ego-nets as a single undirected graph. Node features
|
||||
are not provided. Labels (circles) are optional and not included by default.
|
||||
|
||||
Statistics (based on merged graph):
|
||||
- Nodes: ~4,000+
|
||||
- Edges: ~88,000+
|
||||
- Features: None
|
||||
- Classes: None
|
||||
|
||||
Reference:
|
||||
J. McAuley and J. Leskovec, “Learning to Discover Social Circles in Ego Networks,”
|
||||
in NIPS, 2012. [https://snap.stanford.edu/data/egonets-Facebook.html]
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import easygraph as eg
|
||||
|
||||
from easygraph.classes.graph import Graph
|
||||
|
||||
from .graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from .utils import download
|
||||
from .utils import extract_archive
|
||||
|
||||
|
||||
class FacebookEgoNetDataset(EasyGraphBuiltinDataset):
|
||||
r"""Facebook Ego-Net social network dataset.
|
||||
|
||||
Each node is a user, and edges represent friendship. The dataset
|
||||
includes 10 ego networks centered on different users.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_dir : str, optional
|
||||
Directory to store the raw downloaded files. Default: None
|
||||
force_reload : bool, optional
|
||||
Whether to re-download and process the dataset. Default: False
|
||||
verbose : bool, optional
|
||||
Whether to print detailed processing logs. Default: True
|
||||
transform : callable, optional
|
||||
Optional transform to apply on the graph.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from easygraph.datasets import FacebookEgoNetDataset
|
||||
>>> dataset = FacebookEgoNetDataset()
|
||||
>>> g = dataset[0]
|
||||
>>> print("Nodes:", g.number_of_nodes())
|
||||
>>> print("Edges:", g.number_of_edges())
|
||||
"""
|
||||
|
||||
def __init__(self, raw_dir=None, force_reload=False, verbose=True, transform=None):
|
||||
name = "facebook"
|
||||
url = "https://snap.stanford.edu/data/facebook.tar.gz"
|
||||
super(FacebookEgoNetDataset, self).__init__(
|
||||
name=name,
|
||||
url=url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def process(self):
|
||||
parent_dir = os.path.join(self.raw_path, "facebook")
|
||||
g = eg.Graph()
|
||||
|
||||
# Iterate over all .edges files in the subdirectory
|
||||
for filename in os.listdir(parent_dir):
|
||||
if filename.endswith(".edges"):
|
||||
edge_file = os.path.join(parent_dir, filename)
|
||||
|
||||
with open(edge_file, "r") as f:
|
||||
for line in f:
|
||||
u, v = map(int, line.strip().split())
|
||||
g.add_edge(u, v)
|
||||
|
||||
self._g = g
|
||||
self._num_nodes = g.number_of_nodes()
|
||||
self._num_edges = g.number_of_edges()
|
||||
|
||||
if self.verbose:
|
||||
print("Finished loading Facebook Ego-Net dataset.")
|
||||
print(f" NumNodes: {self._num_nodes}")
|
||||
print(f" NumEdges: {self._num_edges}")
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "FacebookEgoNetDataset only contains one merged graph"
|
||||
return self._g if self._transform is None else self._transform(self._g)
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
|
||||
def download(self):
|
||||
r"""Automatically download data and extract it."""
|
||||
if self.url is not None:
|
||||
archive_path = os.path.join(self.raw_dir, self.name + ".tar.gz")
|
||||
download(self.url, path=archive_path)
|
||||
extract_archive(archive_path, self.raw_path)
|
||||
@@ -0,0 +1,129 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
import easygraph as eg
|
||||
import numpy as np
|
||||
import scipy.sparse as sp
|
||||
|
||||
from easygraph.classes.graph import Graph
|
||||
|
||||
from .graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from .utils import data_type_dict
|
||||
from .utils import tensor
|
||||
|
||||
|
||||
class FlickrDataset(EasyGraphBuiltinDataset):
|
||||
r"""Flickr dataset for node classification.
|
||||
|
||||
Nodes are images and edges represent social tags co-occurrence.
|
||||
Node features are precomputed image embeddings. Labels indicate image categories.
|
||||
|
||||
Statistics:
|
||||
- Nodes: 89,250
|
||||
- Edges: 899,756
|
||||
- Classes: 7
|
||||
- Feature dim: 500
|
||||
|
||||
Source: GraphSAINT (https://arxiv.org/abs/1907.04931)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_dir : str, optional
|
||||
Custom directory to download the dataset. Default: None (uses standard cache dir).
|
||||
force_reload : bool, optional
|
||||
Whether to re-download and reprocess. Default: False.
|
||||
verbose : bool, optional
|
||||
Whether to print loading progress. Default: False.
|
||||
transform : callable, optional
|
||||
A transform applied to the graph on access.
|
||||
reorder : bool, optional
|
||||
Whether to apply graph reordering for locality (requires torch). Default: False.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from easygraph.datasets import FlickrDataset
|
||||
>>> ds = FlickrDataset(verbose=True)
|
||||
>>> g = ds[0]
|
||||
>>> print(g.number_of_nodes(), g.number_of_edges(), ds.num_classes)
|
||||
>>> print(g.nodes[0]['feat'].shape, g.nodes[0]['label'])
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=False,
|
||||
transform=None,
|
||||
reorder=False,
|
||||
):
|
||||
name = "flickr"
|
||||
url = self._get_dgl_url("dataset/flickr.zip")
|
||||
self._reorder = reorder
|
||||
super(FlickrDataset, self).__init__(
|
||||
name=name,
|
||||
url=url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def process(self):
|
||||
# Load adjacency
|
||||
coo = sp.load_npz(os.path.join(self.raw_path, "adj_full.npz"))
|
||||
g = eg.Graph()
|
||||
g.add_edges_from(list(zip(*coo.nonzero())))
|
||||
|
||||
# Load features
|
||||
feats = np.load(os.path.join(self.raw_path, "feats.npy"))
|
||||
# Load labels
|
||||
with open(os.path.join(self.raw_path, "class_map.json")) as f:
|
||||
class_map = json.load(f)
|
||||
labels = np.array([class_map[str(i)] for i in range(feats.shape[0])])
|
||||
|
||||
# Load train/val/test splits
|
||||
with open(os.path.join(self.raw_path, "role.json")) as f:
|
||||
role = json.load(f)
|
||||
train_mask = np.zeros(feats.shape[0], dtype=bool)
|
||||
train_mask[role["tr"]] = True
|
||||
val_mask = np.zeros(feats.shape[0], dtype=bool)
|
||||
val_mask[role["va"]] = True
|
||||
test_mask = np.zeros(feats.shape[0], dtype=bool)
|
||||
test_mask[role["te"]] = True
|
||||
|
||||
# Attach node data
|
||||
for i in range(feats.shape[0]):
|
||||
g.add_node(i, feat=feats[i].astype(np.float32), label=int(labels[i]))
|
||||
g.graph["train_mask"] = train_mask
|
||||
g.graph["val_mask"] = val_mask
|
||||
g.graph["test_mask"] = test_mask
|
||||
|
||||
self._g = g
|
||||
self._num_classes = int(labels.max() + 1)
|
||||
if self.verbose:
|
||||
print("Loaded Flickr dataset")
|
||||
print(
|
||||
f" Nodes: {g.number_of_nodes()}, Edges: {g.number_of_edges()}, Features: {feats.shape[1]}, Classes: {self._num_classes}"
|
||||
)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "FlickrDataset contains only one graph"
|
||||
g = self._g
|
||||
# transfer mask info
|
||||
g.graph["train_mask"] = g.graph.pop("train_mask")
|
||||
g.graph["val_mask"] = g.graph.pop("val_mask")
|
||||
g.graph["test_mask"] = g.graph.pop("test_mask")
|
||||
return self._transform(g) if self._transform else g
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
|
||||
@property
|
||||
def num_classes(self):
|
||||
return self._num_classes
|
||||
|
||||
@staticmethod
|
||||
def _get_dgl_url(path):
|
||||
from .utils import _get_dgl_url
|
||||
|
||||
return _get_dgl_url(path)
|
||||
@@ -0,0 +1,210 @@
|
||||
import easygraph as eg
|
||||
|
||||
|
||||
# import progressbar
|
||||
|
||||
|
||||
__all__ = [
|
||||
"get_graph_karateclub",
|
||||
"get_graph_blogcatalog",
|
||||
"get_graph_youtube",
|
||||
"get_graph_flickr",
|
||||
]
|
||||
|
||||
|
||||
def get_graph_karateclub():
|
||||
"""Returns the undirected graph of Karate Club.
|
||||
|
||||
Returns
|
||||
-------
|
||||
get_graph_karateclub : easygraph.Graph
|
||||
The undirected graph instance of karate club from dataset:
|
||||
http://vlado.fmf.uni-lj.si/pub/networks/data/Ucinet/UciData.htm
|
||||
|
||||
References
|
||||
----------
|
||||
.. [1] http://vlado.fmf.uni-lj.si/pub/networks/data/Ucinet/UciData.htm
|
||||
|
||||
"""
|
||||
all_members = set(range(34))
|
||||
club1 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 16, 17, 19, 21}
|
||||
# club2 = all_members - club1
|
||||
|
||||
G = eg.Graph(name="Zachary's Karate Club")
|
||||
for node in all_members:
|
||||
G.add_node(node + 1)
|
||||
|
||||
zacharydat = """\
|
||||
0 1 1 1 1 1 1 1 1 0 1 1 1 1 0 0 0 1 0 1 0 1 0 0 0 0 0 0 0 0 0 1 0 0
|
||||
1 0 1 1 0 0 0 1 0 0 0 0 0 1 0 0 0 1 0 1 0 1 0 0 0 0 0 0 0 0 1 0 0 0
|
||||
1 1 0 1 0 0 0 1 1 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 1 0
|
||||
1 1 1 0 0 0 0 1 0 0 0 0 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
1 0 0 0 1 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 1
|
||||
0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1
|
||||
1 0 0 0 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1
|
||||
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1
|
||||
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1
|
||||
0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1
|
||||
1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1
|
||||
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1
|
||||
1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1
|
||||
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 1 0 0 1 1
|
||||
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 1 0 0
|
||||
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 1 0 0
|
||||
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1
|
||||
0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1
|
||||
0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1
|
||||
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 1 1
|
||||
0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1
|
||||
1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 1 0 0 0 1 1
|
||||
0 0 1 0 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 0 1 0 1 1 0 0 0 0 0 1 1 1 0 1
|
||||
0 0 0 0 0 0 0 0 1 1 0 0 0 1 1 1 0 0 1 1 1 0 1 1 0 0 1 1 1 1 1 1 1 0"""
|
||||
|
||||
for row, line in enumerate(zacharydat.split("\n")):
|
||||
thisrow = [int(b) for b in line.split()]
|
||||
for col, entry in enumerate(thisrow):
|
||||
if entry == 1:
|
||||
G.add_edge(row + 1, col + 1)
|
||||
|
||||
# Add the name of each member's club as a node attribute.
|
||||
for v in G:
|
||||
G.nodes[v]["club"] = "Mr. Hi" if v in club1 else "Officer"
|
||||
return G
|
||||
|
||||
|
||||
def get_graph_blogcatalog():
|
||||
"""Returns the undirected graph of blogcatalog.
|
||||
|
||||
Returns
|
||||
-------
|
||||
get_graph_blogcatalog : easygraph.Graph
|
||||
The undirected graph instance of blogcatalog from dataset:
|
||||
https://github.com/phanein/deepwalk/blob/master/example_graphs/blogcatalog.mat
|
||||
|
||||
References
|
||||
----------
|
||||
.. [1] https://github.com/phanein/deepwalk/blob/master/example_graphs/blogcatalog.mat
|
||||
|
||||
"""
|
||||
from scipy.io import loadmat
|
||||
|
||||
def sparse2graph(x):
|
||||
from collections import defaultdict
|
||||
|
||||
G = defaultdict(lambda: set())
|
||||
cx = x.tocoo()
|
||||
for i, j, v in zip(cx.row, cx.col, cx.data):
|
||||
G[i].add(j)
|
||||
return {str(k): [str(x) for x in v] for k, v in G.items()}
|
||||
|
||||
mat = loadmat("./samples/blogcatalog.mat")
|
||||
A = mat["network"]
|
||||
data = sparse2graph(A)
|
||||
|
||||
G = eg.Graph()
|
||||
for u in data:
|
||||
for v in data[u]:
|
||||
G.add_edge(u, v)
|
||||
|
||||
return G
|
||||
|
||||
|
||||
def get_graph_youtube():
|
||||
"""Returns the undirected graph of Youtube dataset.
|
||||
|
||||
Returns
|
||||
-------
|
||||
get_graph_youtube : easygraph.Graph
|
||||
The undirected graph instance of Youtube from dataset:
|
||||
http://socialnetworks.mpi-sws.mpg.de/data/youtube-links.txt.gz
|
||||
|
||||
References
|
||||
----------
|
||||
.. [1] http://socialnetworks.mpi-sws.mpg.de/data/youtube-links.txt.gz
|
||||
|
||||
"""
|
||||
import gzip
|
||||
|
||||
from urllib import request
|
||||
|
||||
url = "http://socialnetworks.mpi-sws.mpg.de/data/youtube-links.txt.gz"
|
||||
zipped_data_path = "./samples/youtube-links.txt.gz"
|
||||
unzipped_data_path = "./samples/youtube-links.txt"
|
||||
|
||||
# Download .gz file
|
||||
print("Downloading Youtube dataset...")
|
||||
request.urlretrieve(url, zipped_data_path, _show_progress)
|
||||
|
||||
# Unzip
|
||||
unzipped_data = gzip.GzipFile(zipped_data_path)
|
||||
open(unzipped_data_path, "wb+").write(unzipped_data.read())
|
||||
unzipped_data.close()
|
||||
|
||||
# Returns graph
|
||||
G = eg.Graph()
|
||||
G.add_edges_from_file(file=unzipped_data_path)
|
||||
return G
|
||||
|
||||
|
||||
def get_graph_flickr():
|
||||
"""Returns the undirected graph of Flickr dataset.
|
||||
|
||||
Returns
|
||||
-------
|
||||
get_graph_flickr : easygraph.Graph
|
||||
The undirected graph instance of Flickr from dataset:
|
||||
http://socialnetworks.mpi-sws.mpg.de/data/flickr-links.txt.gz
|
||||
|
||||
References
|
||||
----------
|
||||
.. [1] http://socialnetworks.mpi-sws.mpg.de/data/flickr-links.txt.gz
|
||||
|
||||
"""
|
||||
import gzip
|
||||
|
||||
from urllib import request
|
||||
|
||||
url = "http://socialnetworks.mpi-sws.mpg.de/data/flickr-links.txt.gz"
|
||||
zipped_data_path = "./samples/flickr-links.txt.gz"
|
||||
unzipped_data_path = "./samples/flickr-links.txt"
|
||||
|
||||
# Download .gz file
|
||||
print("Downloading Flickr dataset...")
|
||||
request.urlretrieve(url, zipped_data_path, _show_progress)
|
||||
|
||||
# Unzip
|
||||
unzipped_data = gzip.GzipFile(zipped_data_path)
|
||||
open(unzipped_data_path, "wb+").write(unzipped_data.read())
|
||||
unzipped_data.close()
|
||||
|
||||
# Returns graph
|
||||
G = eg.Graph()
|
||||
G.add_edges_from_file(file=unzipped_data_path)
|
||||
return G
|
||||
|
||||
|
||||
_pbar = None
|
||||
|
||||
|
||||
def _show_progress(block_num, block_size, total_size):
|
||||
global _pbar
|
||||
if _pbar is None:
|
||||
_pbar = progressbar.ProgressBar(maxval=total_size)
|
||||
_pbar.start()
|
||||
|
||||
downloaded = block_num * block_size
|
||||
if downloaded < total_size:
|
||||
_pbar.update(downloaded)
|
||||
else:
|
||||
_pbar.finish()
|
||||
_pbar = None
|
||||
@@ -0,0 +1,125 @@
|
||||
"""GitHub Users Social Network Dataset (musae_git)
|
||||
|
||||
This dataset represents a directed social network of GitHub users collected in 2019.
|
||||
Nodes represent GitHub developers, and a directed edge from user A to user B indicates that A follows B.
|
||||
|
||||
Each node also includes:
|
||||
- Features: User profile and activity-based features.
|
||||
- Labels: Developer's project area (e.g., machine learning, web dev, etc.)
|
||||
|
||||
Statistics:
|
||||
- Nodes: 37,700
|
||||
- Edges: 289,003
|
||||
- Feature dim: 5,575
|
||||
- Classes: 2
|
||||
|
||||
Reference:
|
||||
J. Leskovec et al. "SNAP Datasets: Stanford Large Network Dataset Collection",
|
||||
https://snap.stanford.edu/data/github-social.html
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
|
||||
import easygraph as eg
|
||||
import numpy as np
|
||||
|
||||
from easygraph.classes.graph import Graph
|
||||
|
||||
from .graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from .utils import download
|
||||
from .utils import extract_archive
|
||||
|
||||
|
||||
class GitHubUsersDataset(EasyGraphBuiltinDataset):
|
||||
r"""GitHub developers social graph (musae_git).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_dir : str, optional
|
||||
Directory to store raw data. Default: None
|
||||
force_reload : bool, optional
|
||||
Force re-download and processing. Default: False
|
||||
verbose : bool, optional
|
||||
Print processing information. Default: True
|
||||
transform : callable, optional
|
||||
Transform to apply to the graph on load.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from easygraph.datasets import GitHubUsersDataset
|
||||
>>> dataset = GitHubUsersDataset()
|
||||
>>> g = dataset[0]
|
||||
>>> print("Nodes:", g.number_of_nodes())
|
||||
>>> print("Edges:", g.number_of_edges())
|
||||
>>> print("Feature shape:", g.nodes[0]['feat'].shape)
|
||||
>>> print("Label:", g.nodes[0]['label'])
|
||||
"""
|
||||
|
||||
def __init__(self, raw_dir=None, force_reload=False, verbose=True, transform=None):
|
||||
name = "musae_git"
|
||||
url = "https://snap.stanford.edu/data/git_web_ml.zip"
|
||||
super(GitHubUsersDataset, self).__init__(
|
||||
name=name,
|
||||
url=url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def download(self):
|
||||
archive = os.path.join(self.raw_dir, self.name + ".zip")
|
||||
download(self.url, path=archive)
|
||||
extract_archive(archive, self.raw_path)
|
||||
|
||||
def process(self):
|
||||
g = eg.DiGraph()
|
||||
base_path = os.path.join(self.raw_path, "git_web_ml")
|
||||
|
||||
# Load node features
|
||||
with open(os.path.join(base_path, "musae_git_features.json"), "r") as f:
|
||||
features = json.load(f)
|
||||
|
||||
# Load labels
|
||||
labels = {}
|
||||
with open(os.path.join(base_path, "musae_git_target.csv"), "r") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
node_id = int(row["id"])
|
||||
labels[node_id] = int(row["ml_target"])
|
||||
|
||||
# Load edges
|
||||
with open(os.path.join(base_path, "musae_git_edges.csv"), "r") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
u, v = int(row["id_1"]), int(row["id_2"])
|
||||
g.add_edge(u, v)
|
||||
|
||||
# Add node attributes
|
||||
for node_id in g.nodes:
|
||||
feat = np.array(features[str(node_id)], dtype=np.float32)
|
||||
label = labels.get(node_id, -1)
|
||||
g.add_node(node_id, feat=feat, label=label)
|
||||
|
||||
self._g = g
|
||||
self._num_classes = len(set(labels.values()))
|
||||
|
||||
if self.verbose:
|
||||
print("Finished loading GitHub Users dataset.")
|
||||
print(f" NumNodes: {g.number_of_nodes()}")
|
||||
print(f" NumEdges: {g.number_of_edges()}")
|
||||
print(f" Feature dim: {feat.shape[0]}")
|
||||
print(f" NumClasses: {self._num_classes}")
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "GitHubUsersDataset only contains one graph"
|
||||
return self._g if self._transform is None else self._transform(self._g)
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
|
||||
@property
|
||||
def num_classes(self):
|
||||
return self._num_classes
|
||||
@@ -0,0 +1,216 @@
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import scipy.sparse as sp
|
||||
|
||||
from easygraph.classes.graph import Graph
|
||||
|
||||
from .graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from .utils import _get_dgl_url
|
||||
from .utils import _set_labels
|
||||
from .utils import data_type_dict
|
||||
from .utils import tensor
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AmazonCoBuyComputerDataset",
|
||||
]
|
||||
|
||||
|
||||
class GNNBenchmarkDataset(EasyGraphBuiltinDataset):
|
||||
r"""Base Class for GNN Benchmark dataset
|
||||
|
||||
Reference: https://github.com/shchur/gnn-benchmark#datasets
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, name, raw_dir=None, force_reload=False, verbose=True, transform=None
|
||||
):
|
||||
_url = _get_dgl_url("dataset/" + name + ".zip")
|
||||
super(GNNBenchmarkDataset, self).__init__(
|
||||
name=name,
|
||||
url=_url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def process(self):
|
||||
npz_path = os.path.join(self.raw_path, self.name + ".npz")
|
||||
g = self._load_npz(npz_path)
|
||||
# g = transforms.reorder_graph(
|
||||
# g, node_permute_algo='rcmk', edge_permute_algo='dst', store_ids=False)
|
||||
self._graph = g
|
||||
self._data = [g]
|
||||
self._print_info()
|
||||
|
||||
def has_cache(self):
|
||||
graph_path = os.path.join(self.save_path, "dgl_graph_v1.bin")
|
||||
if os.path.exists(graph_path):
|
||||
return True
|
||||
return False
|
||||
|
||||
# def save(self):
|
||||
# graph_path = os.path.join(self.save_path, 'dgl_graph_v1.bin')
|
||||
# save_graphs(graph_path, self._graph)
|
||||
#
|
||||
# def load(self):
|
||||
# graph_path = os.path.join(self.save_path, 'dgl_graph_v1.bin')
|
||||
# graphs, _ = load_graphs(graph_path)
|
||||
# self._graph = graphs[0]
|
||||
# self._data = [graphs[0]]
|
||||
# self._print_info()
|
||||
|
||||
def _print_info(self):
|
||||
if self.verbose:
|
||||
print(" NumNodes: {}".format(self._graph.number_of_nodes()))
|
||||
print(" NumEdges: {}".format(2 * self._graph.number_of_edges()))
|
||||
print(" NumFeats: {}".format(self._graph.ndata["feat"].shape[-1]))
|
||||
print(" NumbClasses: {}".format(self.num_classes))
|
||||
|
||||
def _load_npz(self, file_name):
|
||||
with np.load(file_name, allow_pickle=True) as loader:
|
||||
loader = dict(loader)
|
||||
num_nodes = loader["adj_shape"][0]
|
||||
adj_matrix = sp.csr_matrix(
|
||||
(loader["adj_data"], loader["adj_indices"], loader["adj_indptr"]),
|
||||
shape=loader["adj_shape"],
|
||||
).tocoo()
|
||||
|
||||
if "attr_data" in loader:
|
||||
# Attributes are stored as a sparse CSR matrix
|
||||
attr_matrix = sp.csr_matrix(
|
||||
(
|
||||
loader["attr_data"],
|
||||
loader["attr_indices"],
|
||||
loader["attr_indptr"],
|
||||
),
|
||||
shape=loader["attr_shape"],
|
||||
).todense()
|
||||
elif "attr_matrix" in loader:
|
||||
# Attributes are stored as a (dense) np.ndarray
|
||||
attr_matrix = loader["attr_matrix"]
|
||||
else:
|
||||
attr_matrix = None
|
||||
|
||||
if "labels_data" in loader:
|
||||
# Labels are stored as a CSR matrix
|
||||
labels = sp.csr_matrix(
|
||||
(
|
||||
loader["labels_data"],
|
||||
loader["labels_indices"],
|
||||
loader["labels_indptr"],
|
||||
),
|
||||
shape=loader["labels_shape"],
|
||||
).todense()
|
||||
elif "labels" in loader:
|
||||
# Labels are stored as a numpy array
|
||||
labels = loader["labels"]
|
||||
else:
|
||||
labels = None
|
||||
if hasattr(adj_matrix, "format"):
|
||||
print("can be generate eg!")
|
||||
g = Graph(incoming_graph_data=adj_matrix)
|
||||
# g = transforms.to_bidirected(g)
|
||||
g = _set_labels(g, labels)
|
||||
g.ndata["feat"] = tensor(attr_matrix, data_type_dict()["float32"])
|
||||
g.ndata["label"] = tensor(labels, data_type_dict()["int64"])
|
||||
return g
|
||||
|
||||
@property
|
||||
def num_classes(self):
|
||||
"""Number of classes."""
|
||||
raise NotImplementedError
|
||||
|
||||
def __getitem__(self, idx):
|
||||
r"""Get graph by index
|
||||
|
||||
Parameters
|
||||
----------
|
||||
idx : int
|
||||
Item index
|
||||
|
||||
Returns
|
||||
-------
|
||||
:class:`dgl.DGLGraph`
|
||||
|
||||
The graph contains:
|
||||
|
||||
- ``ndata['feat']``: node features
|
||||
- ``ndata['label']``: node labels
|
||||
"""
|
||||
assert idx == 0, "This dataset has only one graph"
|
||||
if self._transform is None:
|
||||
return self._graph
|
||||
else:
|
||||
return self._transform(self._graph)
|
||||
|
||||
def __len__(self):
|
||||
r"""Number of graphs in the dataset"""
|
||||
return 1
|
||||
|
||||
|
||||
class AmazonCoBuyComputerDataset(GNNBenchmarkDataset):
|
||||
r"""'Computer' part of the AmazonCoBuy dataset for node classification task.
|
||||
|
||||
Amazon Computers and Amazon Photo are segments of the Amazon co-purchase graph [McAuley et al., 2015],
|
||||
where nodes represent goods, edges indicate that two goods are frequently bought together, node
|
||||
features are bag-of-words encoded product reviews, and class labels are given by the product category.
|
||||
|
||||
Reference: `<https://github.com/shchur/gnn-benchmark#datasets>`_
|
||||
|
||||
Statistics:
|
||||
|
||||
- Nodes: 13,752
|
||||
- Edges: 491,722 (note that the original dataset has 245,778 edges but DGL adds
|
||||
the reverse edges and remove the duplicates, hence with a different number)
|
||||
- Number of classes: 10
|
||||
- Node feature size: 767
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_dir : str
|
||||
Raw file directory to download/contains the input data directory.
|
||||
Default: ~/.dgl/
|
||||
force_reload : bool
|
||||
Whether to reload the dataset. Default: False
|
||||
verbose : bool
|
||||
Whether to print out progress information. Default: True.
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~dgl.DGLGraph` object and returns
|
||||
a transformed version. The :class:`~dgl.DGLGraph` object will be
|
||||
transformed before every access.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
num_classes : int
|
||||
Number of classes for each node.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> data = AmazonCoBuyComputerDataset()
|
||||
>>> g = data[0]
|
||||
>>> num_class = data.num_classes
|
||||
>>> feat = g.ndata['feat'] # get node feature
|
||||
>>> label = g.ndata['label'] # get node labels
|
||||
"""
|
||||
|
||||
def __init__(self, raw_dir=None, force_reload=False, verbose=True, transform=None):
|
||||
super(AmazonCoBuyComputerDataset, self).__init__(
|
||||
name="amazon_co_buy_computer",
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
@property
|
||||
def num_classes(self):
|
||||
"""Number of classes.
|
||||
|
||||
Return
|
||||
-------
|
||||
int
|
||||
"""
|
||||
return 10
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Basic EasyGraph Dataset"""
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import abc
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from ..utils import retry_method_with_fix
|
||||
from .utils import download
|
||||
from .utils import extract_archive
|
||||
from .utils import get_download_dir
|
||||
from .utils import makedirs
|
||||
|
||||
|
||||
class EasyGraphDataset(object):
|
||||
r"""The basic EasyGraph dataset for creating graph datasets.
|
||||
This class defines a basic template class for EasyGraph Dataset.
|
||||
The following steps will be executed automatically:
|
||||
|
||||
1. Check whether there is a dataset cache on disk
|
||||
(already processed and stored on the disk) by
|
||||
invoking ``has_cache()``. If true, goto 5.
|
||||
2. Call ``download()`` to download the data if ``url`` is not None.
|
||||
3. Call ``process()`` to process the data.
|
||||
4. Call ``save()`` to save the processed dataset on disk and goto 6.
|
||||
5. Call ``load()`` to load the processed dataset from disk.
|
||||
6. Done.
|
||||
|
||||
Users can overwrite these functions with their
|
||||
own data processing logic.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
Name of the dataset
|
||||
url : str
|
||||
Url to download the raw dataset. Default: None
|
||||
raw_dir : str
|
||||
Specifying the directory that will store the
|
||||
downloaded data or the directory that
|
||||
already stores the input data.
|
||||
Default: ~/.EasyGraphData/
|
||||
save_dir : str
|
||||
Directory to save the processed dataset.
|
||||
Default: same as raw_dir
|
||||
hash_key : tuple
|
||||
A tuple of values as the input for the hash function.
|
||||
Users can distinguish instances (and their caches on the disk)
|
||||
from the same dataset class by comparing the hash values.
|
||||
Default: (), the corresponding hash value is ``'f9065fa7'``.
|
||||
force_reload : bool
|
||||
Whether to reload the dataset. Default: False
|
||||
verbose : bool
|
||||
Whether to print out progress information
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~dgl.DGLGraph` object and returns
|
||||
a transformed version. The :class:`~dgl.DGLGraph` object will be
|
||||
transformed before every access.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name,
|
||||
url=None,
|
||||
raw_dir=None,
|
||||
save_dir=None,
|
||||
hash_key=(),
|
||||
force_reload=False,
|
||||
verbose=False,
|
||||
transform=None,
|
||||
):
|
||||
self._name = name
|
||||
self._url = url
|
||||
self._force_reload = force_reload
|
||||
self._verbose = verbose
|
||||
self._hash_key = hash_key
|
||||
self._hash = self._get_hash()
|
||||
self._transform = transform
|
||||
|
||||
# if no dir is provided, the default EasyGraph download dir is used.
|
||||
if raw_dir is None:
|
||||
self._raw_dir = get_download_dir()
|
||||
else:
|
||||
self._raw_dir = raw_dir
|
||||
|
||||
if save_dir is None:
|
||||
self._save_dir = self._raw_dir
|
||||
else:
|
||||
self._save_dir = save_dir
|
||||
self._load()
|
||||
|
||||
def download(self):
|
||||
r"""Overwrite to realize your own logic of downloading data.
|
||||
|
||||
It is recommended to download the to the :obj:`self.raw_dir`
|
||||
folder. Can be ignored if the dataset is
|
||||
already in :obj:`self.raw_dir`.
|
||||
"""
|
||||
pass
|
||||
|
||||
def save(self):
|
||||
r"""Overwrite to realize your own logic of
|
||||
saving the processed dataset into files.
|
||||
|
||||
It is recommended to use ``dgl.data.utils.save_graphs``
|
||||
to save dgl graph into files and use
|
||||
``dgl.data.utils.save_info`` to save extra
|
||||
information into files.
|
||||
"""
|
||||
pass
|
||||
|
||||
def load(self):
|
||||
r"""Overwrite to realize your own logic of
|
||||
loading the saved dataset from files.
|
||||
|
||||
It is recommended to use ``dgl.data.utils.load_graphs``
|
||||
to load dgl graph from files and use
|
||||
``dgl.data.utils.load_info`` to load extra information
|
||||
into python dict object.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def process(self):
|
||||
r"""Overwrite to realize your own logic of processing the input data."""
|
||||
pass
|
||||
|
||||
def has_cache(self):
|
||||
r"""Overwrite to realize your own logic of
|
||||
deciding whether there exists a cached dataset.
|
||||
|
||||
By default False.
|
||||
"""
|
||||
return False
|
||||
|
||||
@retry_method_with_fix(download)
|
||||
def _download(self):
|
||||
"""Download dataset by calling ``self.download()``
|
||||
if the dataset does not exists under ``self.raw_path``.
|
||||
|
||||
By default ``self.raw_path = os.path.join(self.raw_dir, self.name)``
|
||||
One can overwrite ``raw_path()`` function to change the path.
|
||||
"""
|
||||
|
||||
if os.path.exists(self.raw_path): # pragma: no cover
|
||||
return
|
||||
|
||||
makedirs(self.raw_dir)
|
||||
self.download()
|
||||
|
||||
def _load(self):
|
||||
"""Entry point from __init__ to load the dataset.
|
||||
|
||||
If cache exists:
|
||||
|
||||
- Load the dataset from saved dgl graph and information files.
|
||||
- If loading process fails, re-download and process the dataset.
|
||||
|
||||
else:
|
||||
|
||||
- Download the dataset if needed.
|
||||
- Process the dataset and build the dgl graph.
|
||||
- Save the processed dataset into files.
|
||||
"""
|
||||
|
||||
load_flag = not self._force_reload and self.has_cache()
|
||||
if load_flag:
|
||||
try:
|
||||
self.load()
|
||||
self.process()
|
||||
if self.verbose:
|
||||
print("Done loading data from cached files.")
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except:
|
||||
load_flag = False
|
||||
if self.verbose:
|
||||
print(traceback.format_exc())
|
||||
print("Loading from cache failed, re-processing.")
|
||||
|
||||
if not load_flag:
|
||||
self._download()
|
||||
self.process()
|
||||
self.save()
|
||||
if self.verbose:
|
||||
print("Done saving data into cached files.")
|
||||
|
||||
def _get_hash(self):
|
||||
"""Compute the hash of the input tuple
|
||||
|
||||
Example
|
||||
-------
|
||||
Assume `self._hash_key = (10, False, True)`
|
||||
|
||||
>>> hash_value = self._get_hash()
|
||||
>>> hash_value
|
||||
'a770b222'
|
||||
"""
|
||||
hash_func = hashlib.sha1()
|
||||
hash_func.update(str(self._hash_key).encode("utf-8"))
|
||||
return hash_func.hexdigest()[:8]
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
r"""Get url to download the raw dataset."""
|
||||
return self._url
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
r"""Name of the dataset."""
|
||||
return self._name
|
||||
|
||||
@property
|
||||
def raw_dir(self):
|
||||
r"""Raw file directory contains the input data folder."""
|
||||
return self._raw_dir
|
||||
|
||||
@property
|
||||
def raw_path(self):
|
||||
r"""Directory contains the input data files.
|
||||
By default raw_path = os.path.join(self.raw_dir, self.name)
|
||||
"""
|
||||
return os.path.join(self.raw_dir, self.name)
|
||||
|
||||
@property
|
||||
def save_dir(self):
|
||||
r"""Directory to save the processed dataset."""
|
||||
return self._save_dir
|
||||
|
||||
@property
|
||||
def save_path(self):
|
||||
r"""Path to save the processed dataset."""
|
||||
return os.path.join(self._save_dir)
|
||||
|
||||
@property
|
||||
def verbose(self):
|
||||
r"""Whether to print information."""
|
||||
return self._verbose
|
||||
|
||||
@property
|
||||
def hash(self):
|
||||
r"""Hash value for the dataset and the setting."""
|
||||
return self._hash
|
||||
|
||||
@abc.abstractmethod
|
||||
def __getitem__(self, idx):
|
||||
r"""Gets the data object at index."""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def __len__(self):
|
||||
r"""The number of examples in the dataset."""
|
||||
pass
|
||||
|
||||
def __repr__(self):
|
||||
return f'Dataset("{self.name}"' + f" save_path={self.save_path})"
|
||||
|
||||
|
||||
class EasyGraphBuiltinDataset(EasyGraphDataset):
|
||||
r"""The Basic EasyGraph Builtin Dataset.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
Name of the dataset.
|
||||
url : str
|
||||
Url to download the raw dataset.
|
||||
raw_dir : str
|
||||
Specifying the directory that will store the
|
||||
downloaded data or the directory that
|
||||
already stores the input data.
|
||||
Default: ~/.dgl/
|
||||
hash_key : tuple
|
||||
A tuple of values as the input for the hash function.
|
||||
Users can distinguish instances (and their caches on the disk)
|
||||
from the same dataset class by comparing the hash values.
|
||||
force_reload : bool
|
||||
Whether to reload the dataset. Default: False
|
||||
verbose : bool
|
||||
Whether to print out progress information. Default: False
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~dgl.DGLGraph` object and returns
|
||||
a transformed version. The :class:`~dgl.DGLGraph` object will be
|
||||
transformed before every access.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name,
|
||||
url,
|
||||
raw_dir=None,
|
||||
hash_key=(),
|
||||
force_reload=False,
|
||||
verbose=True,
|
||||
transform=None,
|
||||
save_dir=None,
|
||||
):
|
||||
super(EasyGraphBuiltinDataset, self).__init__(
|
||||
name,
|
||||
url=url,
|
||||
raw_dir=raw_dir,
|
||||
save_dir=save_dir,
|
||||
hash_key=hash_key,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def download(self):
|
||||
r"""Automatically download data and extract it."""
|
||||
if self.url is not None:
|
||||
zip_file_path = os.path.join(self.raw_dir, self.name + ".zip")
|
||||
download(self.url, path=zip_file_path)
|
||||
extract_archive(zip_file_path, self.raw_path)
|
||||
@@ -0,0 +1,216 @@
|
||||
import requests
|
||||
|
||||
from easygraph.utils.exception import EasyGraphError
|
||||
|
||||
|
||||
def request_text_from_url(url):
|
||||
"""
|
||||
Requests text data from the specified URL.
|
||||
|
||||
Args:
|
||||
url (str): The URL from which to request the text data.
|
||||
|
||||
Returns:
|
||||
str: The text content of the response if the request is successful.
|
||||
|
||||
Raises:
|
||||
EasyGraphError: If a connection error occurs during the request or if the HTTP response status code
|
||||
indicates a failure.
|
||||
"""
|
||||
try:
|
||||
r = requests.get(url)
|
||||
except requests.ConnectionError:
|
||||
raise EasyGraphError("Connection Error!")
|
||||
|
||||
if r.ok:
|
||||
return r.text
|
||||
else:
|
||||
raise EasyGraphError(f"Error: HTTP response {r.status_code}")
|
||||
|
||||
|
||||
class House_Committees:
|
||||
"""
|
||||
A class for loading and processing the House Committees hypergraph dataset.
|
||||
|
||||
This class fetches hyperedge, node label, node name, and label name data from predefined URLs,
|
||||
processes the data, and generates a hypergraph representation. It also provides access to various
|
||||
dataset attributes through properties and indexing.
|
||||
|
||||
Attributes:
|
||||
data_root (str): The root URL for the data. If `data_root` is provided during initialization,
|
||||
it is set to "https://"; otherwise, it is `None`.
|
||||
hyperedges_path (str): The URL of the file containing hyperedge information.
|
||||
node_labels_path (str): The URL of the file containing node label information.
|
||||
node_names_path (str): The URL of the file containing node name information.
|
||||
label_names_path (str): The URL of the file containing label name information.
|
||||
_hyperedges (list): A list of tuples representing hyperedges.
|
||||
_node_labels (list): A list of node labels.
|
||||
_label_names (list): A list of label names.
|
||||
_node_names (list): A list of node names.
|
||||
_content (dict): A dictionary containing dataset statistics and data, including the number of
|
||||
classes, vertices, edges, the edge list, and node labels.
|
||||
"""
|
||||
|
||||
def __init__(self, data_root=None):
|
||||
"""
|
||||
Initializes a new instance of the `House_Committees` class.
|
||||
|
||||
Args:
|
||||
data_root (str, optional): The root URL for the data. If provided, it is set to "https://";
|
||||
otherwise, it is `None`. Defaults to `None`.
|
||||
"""
|
||||
self.data_root = "https://" if data_root is not None else data_root
|
||||
self.hyperedges_path = "https://gitlab.com/easy-graph/easygraph-data-house-committees/-/raw/main/hyperedges-house-committees.txt?inline=false"
|
||||
self.node_labels_path = "https://gitlab.com/easy-graph/easygraph-data-house-committees/-/raw/main/node-labels-house-committees.txt?ref_type=heads&inline=false"
|
||||
self.node_names_path = "https://gitlab.com/easy-graph/easygraph-data-house-committees/-/raw/main/node-names-house-committees.txt?ref_type=heads&inline=false"
|
||||
self.label_names_path = "https://gitlab.com/easy-graph/easygraph-data-house-committees/-/raw/main/label-names-house-committees.txt?ref_type=heads&inline=false"
|
||||
self._hyperedges = []
|
||||
self._node_labels = []
|
||||
self._label_names = []
|
||||
self._node_names = []
|
||||
self.generate_hypergraph(
|
||||
hyperedges_path=self.hyperedges_path,
|
||||
node_labels_path=self.node_labels_path,
|
||||
node_names_path=self.node_names_path,
|
||||
label_names_path=self.label_names_path,
|
||||
)
|
||||
|
||||
self._content = {
|
||||
"num_classes": len(self._label_names),
|
||||
"num_vertices": len(self._node_labels),
|
||||
"num_edges": len(self._hyperedges),
|
||||
"edge_list": self._hyperedges,
|
||||
"labels": self._node_labels,
|
||||
}
|
||||
|
||||
def process_label_txt(self, data_str, delimiter="\n", transform_fun=str):
|
||||
"""
|
||||
Processes a string containing label data into a list of transformed values.
|
||||
|
||||
Args:
|
||||
data_str (str): The input string containing label data.
|
||||
delimiter (str, optional): The delimiter used to split the input string. Defaults to "\n".
|
||||
transform_fun (callable, optional): A function used to transform each label value.
|
||||
Defaults to the `str` function.
|
||||
|
||||
Returns:
|
||||
list: A list of transformed label values.
|
||||
"""
|
||||
data_str = data_str.strip()
|
||||
data_lst = data_str.split(delimiter)
|
||||
final_lst = []
|
||||
for data in data_lst:
|
||||
data = data.strip()
|
||||
data = transform_fun(data)
|
||||
final_lst.append(data)
|
||||
return final_lst
|
||||
|
||||
def __getitem__(self, key: str):
|
||||
"""
|
||||
Retrieves a value from the `_content` dictionary using the specified key.
|
||||
|
||||
Args:
|
||||
key (str): The key used to access the `_content` dictionary.
|
||||
|
||||
Returns:
|
||||
Any: The value corresponding to the key in the `_content` dictionary.
|
||||
"""
|
||||
return self._content[key]
|
||||
|
||||
@property
|
||||
def node_labels(self):
|
||||
"""
|
||||
Gets the list of node labels.
|
||||
|
||||
Returns:
|
||||
list: A list of node labels.
|
||||
"""
|
||||
return self._node_labels
|
||||
|
||||
@property
|
||||
def node_names(self):
|
||||
"""
|
||||
Gets the list of node names.
|
||||
|
||||
Returns:
|
||||
list: A list of node names.
|
||||
"""
|
||||
return self._node_names
|
||||
|
||||
@property
|
||||
def label_names(self):
|
||||
"""
|
||||
Gets the list of label names.
|
||||
|
||||
Returns:
|
||||
list: A list of label names.
|
||||
"""
|
||||
return self._label_names
|
||||
|
||||
@property
|
||||
def hyperedges(self):
|
||||
"""
|
||||
Gets the list of hyperedges.
|
||||
|
||||
Returns:
|
||||
list: A list of tuples representing hyperedges.
|
||||
"""
|
||||
return self._hyperedges
|
||||
|
||||
def generate_hypergraph(
|
||||
self,
|
||||
hyperedges_path=None,
|
||||
node_labels_path=None,
|
||||
node_names_path=None,
|
||||
label_names_path=None,
|
||||
):
|
||||
"""
|
||||
Generates a hypergraph by fetching and processing data from the specified URLs.
|
||||
|
||||
Args:
|
||||
hyperedges_path (str, optional): The URL of the file containing hyperedge information.
|
||||
Defaults to `None`.
|
||||
node_labels_path (str, optional): The URL of the file containing node label information.
|
||||
Defaults to `None`.
|
||||
node_names_path (str, optional): The URL of the file containing node name information.
|
||||
Defaults to `None`.
|
||||
label_names_path (str, optional): The URL of the file containing label name information.
|
||||
Defaults to `None`.
|
||||
"""
|
||||
|
||||
def fun(data):
|
||||
"""
|
||||
Converts a string to an integer and subtracts 1.
|
||||
|
||||
Args:
|
||||
data (str): The input string to be converted.
|
||||
|
||||
Returns:
|
||||
int: The converted integer value minus 1.
|
||||
"""
|
||||
data = int(data) - 1
|
||||
return data
|
||||
|
||||
hyperedges_info = request_text_from_url(hyperedges_path)
|
||||
hyperedges_info = hyperedges_info.strip()
|
||||
hyperedges_lst = hyperedges_info.split("\n")
|
||||
for hyperedge in hyperedges_lst:
|
||||
hyperedge = hyperedge.strip()
|
||||
hyperedge = [int(i) - 1 for i in hyperedge.split(",")]
|
||||
self._hyperedges.append(tuple(hyperedge))
|
||||
# print(self.hyperedges)
|
||||
|
||||
node_labels_info = request_text_from_url(node_labels_path)
|
||||
|
||||
process_node_labels_info = self.process_label_txt(
|
||||
node_labels_info, transform_fun=fun
|
||||
)
|
||||
self._node_labels = process_node_labels_info
|
||||
# print("process_node_labels_info:", process_node_labels_info)
|
||||
node_names_info = request_text_from_url(node_names_path)
|
||||
process_node_names_info = self.process_label_txt(node_names_info)
|
||||
self._node_names = process_node_names_info
|
||||
# print("process_node_names_info:", process_node_names_info)
|
||||
label_names_info = request_text_from_url(label_names_path)
|
||||
process_label_names_info = self.process_label_txt(label_names_info)
|
||||
self._label_names = process_label_names_info
|
||||
@@ -0,0 +1,82 @@
|
||||
from typing import Optional
|
||||
|
||||
from easygraph.datapipe import load_from_pickle
|
||||
from easygraph.datapipe import to_long_tensor
|
||||
from easygraph.datapipe import to_tensor
|
||||
from easygraph.datasets.hypergraph.hypergraph_dataset_base import BaseData
|
||||
|
||||
|
||||
class YelpRestaurant(BaseData):
|
||||
r"""The Yelp-Restaurant dataset is a restaurant-review network dataset for node classification task.
|
||||
|
||||
More details see the DHG or `YOU ARE ALLSET: A MULTISET LEARNING FRAMEWORK FOR HYPERGRAPH NEURAL NETWORKS <https://openreview.net/pdf?id=hpBTIv2uy_E>`_ paper.
|
||||
|
||||
The content of the Yelp-Restaurant dataset includes the following:
|
||||
|
||||
- ``num_classes``: The number of classes: :math:`11`.
|
||||
- ``num_vertices``: The number of vertices: :math:`50,758`.
|
||||
- ``num_edges``: The number of edges: :math:`679,302`.
|
||||
- ``dim_features``: The dimension of features: :math:`1,862`.
|
||||
- ``features``: The vertex feature matrix. ``torch.Tensor`` with size :math:`(50,758 \times 1,862)`.
|
||||
- ``edge_list``: The edge list. ``List`` with length :math:`679,302`.
|
||||
- ``labels``: The label list. ``torch.LongTensor`` with size :math:`(50,758, )`.
|
||||
- ``state``: The state list. ``torch.LongTensor`` with size :math:`(50,758, )`.
|
||||
- ``city``: The city list. ``torch.LongTensor`` with size :math:`(50,758, )`.
|
||||
|
||||
Args:
|
||||
``data_root`` (``str``, optional): The ``data_root`` has stored the data. If set to ``None``, this function will auto-download from server and save into the default direction ``~/.dhg/datasets/``. Defaults to ``None``.
|
||||
"""
|
||||
|
||||
def __init__(self, data_root: Optional[str] = None) -> None:
|
||||
super().__init__("yelp_restaurant", data_root)
|
||||
self._content = {
|
||||
"num_classes": 11,
|
||||
"num_vertices": 50758,
|
||||
"num_edges": 679302,
|
||||
"dim_features": 1862,
|
||||
"features": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "features.pkl",
|
||||
"md5": "cedc4443884477c2e626025411c44cd7",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [
|
||||
to_tensor,
|
||||
],
|
||||
},
|
||||
"edge_list": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "edge_list.pkl",
|
||||
"md5": "4b26eecaa22305dd10edcd6372eb49da",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
},
|
||||
"labels": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "labels.pkl",
|
||||
"md5": "1cdc1ed9fb1f57b2accaa42db214d4ef",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_long_tensor],
|
||||
},
|
||||
"state": {
|
||||
"upon": [
|
||||
{"filename": "state.pkl", "md5": "eef3b835fad37409f29ad36539296b57"}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_long_tensor],
|
||||
},
|
||||
"city": {
|
||||
"upon": [
|
||||
{"filename": "city.pkl", "md5": "8302b167262b23067698e865cacd0b17"}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_long_tensor],
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
from .cat_edge_Cooking import *
|
||||
from .coauthorship import *
|
||||
from .cocitation import *
|
||||
from .contact_primary_school import *
|
||||
from .House_Committees import *
|
||||
from .mathoverflow_answers import *
|
||||
from .senate_committees import *
|
||||
from .trivago_clicks import *
|
||||
from .walmart_trips import *
|
||||
from .Yelp import *
|
||||
@@ -0,0 +1,14 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_eg_cache_root():
|
||||
root = Path.home() / Path(".easygraph/")
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
return root
|
||||
|
||||
|
||||
CACHE_ROOT = get_eg_cache_root()
|
||||
DATASETS_ROOT = CACHE_ROOT / "datasets"
|
||||
|
||||
REMOTE_ROOT = "https://download.moon-lab.tech:28501/"
|
||||
REMOTE_DATASETS_ROOT = REMOTE_ROOT + "datasets/"
|
||||
@@ -0,0 +1,104 @@
|
||||
import requests
|
||||
|
||||
from easygraph.utils.exception import EasyGraphError
|
||||
|
||||
|
||||
def request_text_from_url(url):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
except requests.ConnectionError:
|
||||
raise EasyGraphError("Connection Error!")
|
||||
|
||||
if r.ok:
|
||||
return r.text
|
||||
else:
|
||||
raise EasyGraphError(f"Error: HTTP response {r.status_code}")
|
||||
|
||||
|
||||
class cat_edge_Cooking:
|
||||
def __init__(self, data_root=None):
|
||||
self.data_root = "https://" if data_root is not None else data_root
|
||||
self.hyperedges_path = "https://gitlab.com/easy-graph/easygraph-data-cat-edge-cooking/-/raw/main/hyperedges.txt?inline=false"
|
||||
self.edge_labels_path = "https://gitlab.com/easy-graph/easygraph-data-cat-edge-cooking/-/raw/main/hyperedge-labels.txt?ref_type=heads&inline=false"
|
||||
self.node_names_path = "https://gitlab.com/easy-graph/easygraph-data-cat-edge-cooking/-/raw/main/main/node-labels.txt?ref_type=heads&inline=false"
|
||||
self.label_names_path = "https://gitlab.com/easy-graph/easygraph-data-cat-edge-cooking/-/raw/main/hyperedge-label-identities.txt?ref_type=heads&inline=false"
|
||||
# self.hyperedges_path = []
|
||||
# self.edge_labels_path = []
|
||||
# self.node_names_path = []
|
||||
# self.label_names_path = []
|
||||
self.generate_hypergraph(
|
||||
hyperedges_path=self.hyperedges_path,
|
||||
edge_labels_path=self.edge_labels_path,
|
||||
node_names_path=self.node_names_path,
|
||||
label_names_path=self.label_names_path,
|
||||
)
|
||||
self._content = {
|
||||
"num_classes": len(self._label_names),
|
||||
"num_vertices": len(self._node_labels),
|
||||
"num_edges": len(self._hyperedges),
|
||||
"edge_list": self._hyperedges,
|
||||
"labels": self._node_labels,
|
||||
}
|
||||
|
||||
def __getitem__(self, key: str):
|
||||
return self._content[key]
|
||||
|
||||
def process_label_txt(self, data_str, delimiter="\n", transform_fun=str):
|
||||
data_str = data_str.strip()
|
||||
data_lst = data_str.split(delimiter)
|
||||
final_lst = []
|
||||
for data in data_lst:
|
||||
data = data.strip()
|
||||
data = transform_fun(data)
|
||||
final_lst.append(data)
|
||||
return final_lst
|
||||
|
||||
@property
|
||||
def edge_labels(self):
|
||||
return self._edge_labels
|
||||
|
||||
@property
|
||||
def node_names(self):
|
||||
return self._node_names
|
||||
|
||||
@property
|
||||
def label_names(self):
|
||||
return self._label_names
|
||||
|
||||
@property
|
||||
def hyperedges(self):
|
||||
return self._hyperedges
|
||||
|
||||
def generate_hypergraph(
|
||||
self,
|
||||
hyperedges_path=None,
|
||||
node_labels_path=None,
|
||||
node_names_path=None,
|
||||
label_names_path=None,
|
||||
):
|
||||
def fun(data):
|
||||
data = int(data) - 1
|
||||
return data
|
||||
|
||||
hyperedges_info = request_text_from_url(hyperedges_path)
|
||||
hyperedges_info = hyperedges_info.strip()
|
||||
hyperedges_lst = hyperedges_info.split("\n")
|
||||
for hyperedge in hyperedges_lst:
|
||||
hyperedge = hyperedge.strip()
|
||||
hyperedge = [int(i) - 1 for i in hyperedge.split(" ")]
|
||||
self._hyperedges.append(tuple(hyperedge))
|
||||
# print(self.hyperedges)
|
||||
|
||||
edge_labels_info = request_text_from_url(self.edge_labels_path)
|
||||
process_node_labels_info = self.process_label_txt(
|
||||
node_labels_info, transform_fun=fun
|
||||
)
|
||||
self._edge_labels = process_edge_labels_info()
|
||||
|
||||
node_names_info = request_text_from_url(node_names_path)
|
||||
process_node_names_info = self.process_label_txt(node_names_info)
|
||||
self._node_names = process_node_names_info
|
||||
|
||||
label_names_info = request_text_from_url(label_names_path)
|
||||
process_label_names_info = self.process_label_txt(label_names_info)
|
||||
self._label_names = process_label_names_info
|
||||
@@ -0,0 +1,192 @@
|
||||
from functools import partial
|
||||
from typing import Optional
|
||||
|
||||
from easygraph.datapipe import load_from_pickle
|
||||
from easygraph.datapipe import norm_ft
|
||||
from easygraph.datapipe import to_bool_tensor
|
||||
from easygraph.datapipe import to_long_tensor
|
||||
from easygraph.datapipe import to_tensor
|
||||
from easygraph.datasets.hypergraph.hypergraph_dataset_base import BaseData
|
||||
|
||||
|
||||
__all__ = ["CoauthorshipCora", "CoauthorshipDBLP"]
|
||||
|
||||
|
||||
class CoauthorshipCora(BaseData):
|
||||
r"""The Co-authorship Cora dataset is a citation network dataset for vertex classification task.
|
||||
More details see the `HyperGCN <https://papers.nips.cc/paper/2019/file/1efa39bcaec6f3900149160693694536-Paper.pdf>`_ paper.
|
||||
|
||||
The content of the Co-authorship Cora dataset includes the following:
|
||||
|
||||
- ``num_classes``: The number of classes: :math:`7`.
|
||||
- ``num_vertices``: The number of vertices: :math:`2,708`.
|
||||
- ``num_edges``: The number of edges: :math:`1,072`.
|
||||
- ``dim_features``: The dimension of features: :math:`1,433`.
|
||||
- ``features``: The vertex feature matrix. ``torch.Tensor`` with size :math:`(2,708 \times 1,433)`.
|
||||
- ``edge_list``: The edge list. ``List`` with length :math:`1,072`.
|
||||
- ``labels``: The label list. ``torch.LongTensor`` with size :math:`(2,708, )`.
|
||||
- ``train_mask``: The train mask. ``torch.BoolTensor`` with size :math:`(2,708, )`.
|
||||
- ``val_mask``: The validation mask. ``torch.BoolTensor`` with size :math:`(2,708, )`.
|
||||
- ``test_mask``: The test mask. ``torch.BoolTensor`` with size :math:`(2,708, )`.
|
||||
|
||||
Args:
|
||||
``data_root`` (``str``, optional): The ``data_root`` has stored the data. If set to ``None``, this function will auto-download from server and save into the default direction ``~/.dhg/datasets/``. Defaults to ``None``.
|
||||
"""
|
||||
|
||||
def __init__(self, data_root: Optional[str] = None) -> None:
|
||||
super().__init__("coauthorship_cora", data_root)
|
||||
self._content = {
|
||||
"num_classes": 7,
|
||||
"num_vertices": 2708,
|
||||
"num_edges": 1072,
|
||||
"dim_features": 1433,
|
||||
"features": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "features.pkl",
|
||||
"md5": "14257c0e24b4eb741b469a351e524785",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_tensor, partial(norm_ft, ord=1)],
|
||||
},
|
||||
"edge_list": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "edge_list.pkl",
|
||||
"md5": "a17ff337f1b9099f5a9d4d670674e146",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
},
|
||||
"labels": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "labels.pkl",
|
||||
"md5": "c8d11c452e0be69f79a47dd839279117",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_long_tensor],
|
||||
},
|
||||
"train_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "train_mask.pkl",
|
||||
"md5": "111db6c6f986be2908378df7bdca7a9b",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
"val_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "val_mask.pkl",
|
||||
"md5": "ffab1055193ffb2fe74822bb575d332a",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
"test_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "test_mask.pkl",
|
||||
"md5": "ffab1055193ffb2fe74822bb575d332a",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class CoauthorshipDBLP(BaseData):
|
||||
r"""The Co-authorship DBLP dataset is a citation network dataset for vertex classification task.
|
||||
More details see the `HyperGCN <https://papers.nips.cc/paper/2019/file/1efa39bcaec6f3900149160693694536-Paper.pdf>`_ paper.
|
||||
|
||||
The content of the Co-authorship DBLP dataset includes the following:
|
||||
|
||||
- ``num_classes``: The number of classes: :math:`6`.
|
||||
- ``num_vertices``: The number of vertices: :math:`41,302`.
|
||||
- ``num_edges``: The number of edges: :math:`22,363`.
|
||||
- ``dim_features``: The dimension of features: :math:`1,425`.
|
||||
- ``features``: The vertex feature matrix. ``torch.Tensor`` with size :math:`(41,302 \times 1,425)`.
|
||||
- ``edge_list``: The edge list. ``List`` with length :math:`22,363`.
|
||||
- ``labels``: The label list. ``torch.LongTensor`` with size :math:`(41,302, )`.
|
||||
- ``train_mask``: The train mask. ``torch.BoolTensor`` with size :math:`(41,302, )`.
|
||||
- ``val_mask``: The validation mask. ``torch.BoolTensor`` with size :math:`(41,302, )`.
|
||||
- ``test_mask``: The test mask. ``torch.BoolTensor`` with size :math:`(41,302, )`.
|
||||
|
||||
Args:
|
||||
``data_root`` (``str``, optional): The ``data_root`` has stored the data. If set to ``None``, this function will auto-download from server and save into the default direction ``~/.dhg/datasets/``. Defaults to None.
|
||||
"""
|
||||
|
||||
def __init__(self, data_root: Optional[str] = None) -> None:
|
||||
super().__init__("coauthorship_dblp", data_root)
|
||||
self._content = {
|
||||
"num_classes": 6,
|
||||
"num_vertices": 41302,
|
||||
"num_edges": 22363,
|
||||
"dim_features": 1425,
|
||||
"features": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "features.pkl",
|
||||
"md5": "b78fd31b2586d1e19a40b3f6cd9cc2e7",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_tensor, partial(norm_ft, ord=1)],
|
||||
},
|
||||
"edge_list": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "edge_list.pkl",
|
||||
"md5": "c6bf5f9f3b9683bcc9b7bcc9eb8707d8",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
},
|
||||
"labels": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "labels.pkl",
|
||||
"md5": "2e7a792ea018028d582af8f02f2058ca",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_long_tensor],
|
||||
},
|
||||
"train_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "train_mask.pkl",
|
||||
"md5": "a842b795c7cac4c2f98a56cf599bc1de",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
"val_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "val_mask.pkl",
|
||||
"md5": "2ec4b7df7c5e6b355067a22c391ad578",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
"test_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "test_mask.pkl",
|
||||
"md5": "2ec4b7df7c5e6b355067a22c391ad578",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
from functools import partial
|
||||
from typing import Optional
|
||||
|
||||
from easygraph.datapipe import load_from_pickle
|
||||
from easygraph.datapipe import norm_ft
|
||||
from easygraph.datapipe import to_bool_tensor
|
||||
from easygraph.datapipe import to_long_tensor
|
||||
from easygraph.datapipe import to_tensor
|
||||
from easygraph.datasets.hypergraph.hypergraph_dataset_base import BaseData
|
||||
|
||||
|
||||
class CocitationCora(BaseData):
|
||||
r"""The Co-citation Cora dataset is a citation network dataset for vertex classification task.
|
||||
More details see the `HyperGCN <https://papers.nips.cc/paper/2019/file/1efa39bcaec6f3900149160693694536-Paper.pdf>`_ paper.
|
||||
|
||||
The content of the Co-citation Cora dataset includes the following:
|
||||
|
||||
- ``num_classes``: The number of classes: :math:`7`.
|
||||
- ``num_vertices``: The number of vertices: :math:`2,708`.
|
||||
- ``num_edges``: The number of edges: :math:`1,579`.
|
||||
- ``dim_features``: The dimension of features: :math:`1,433`.
|
||||
- ``features``: The vertex feature matrix. ``torch.Tensor`` with size :math:`(2,708 \times 1,433)`.
|
||||
- ``edge_list``: The edge list. ``List`` with length :math:`1,579`.
|
||||
- ``labels``: The label list. ``torch.LongTensor`` with size :math:`(2,708, )`.
|
||||
- ``train_mask``: The train mask. ``torch.BoolTensor`` with size :math:`(2,708, )`.
|
||||
- ``val_mask``: The validation mask. ``torch.BoolTensor`` with size :math:`(2,708, )`.
|
||||
- ``test_mask``: The test mask. ``torch.BoolTensor`` with size :math:`(2,708, )`.
|
||||
|
||||
Args:
|
||||
``data_root`` (``str``, optional): The ``data_root`` has stored the data. If set to ``None``, this function will auto-download from server and save into the default direction ``~/.dhg/datasets/``. Defaults to ``None``.
|
||||
"""
|
||||
|
||||
def __init__(self, data_root: Optional[str] = None) -> None:
|
||||
super().__init__("cocitation_cora", data_root)
|
||||
self._content = {
|
||||
"num_classes": 7,
|
||||
"num_vertices": 2708,
|
||||
"num_edges": 1579,
|
||||
"dim_features": 1433,
|
||||
"features": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "features.pkl",
|
||||
"md5": "14257c0e24b4eb741b469a351e524785",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_tensor, partial(norm_ft, ord=1)],
|
||||
},
|
||||
"edge_list": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "edge_list.pkl",
|
||||
"md5": "e43d1321880c8ecb2260d8fb7effd9ea",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
},
|
||||
"labels": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "labels.pkl",
|
||||
"md5": "c8d11c452e0be69f79a47dd839279117",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_long_tensor],
|
||||
},
|
||||
"train_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "train_mask.pkl",
|
||||
"md5": "111db6c6f986be2908378df7bdca7a9b",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
"val_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "val_mask.pkl",
|
||||
"md5": "ffab1055193ffb2fe74822bb575d332a",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
"test_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "test_mask.pkl",
|
||||
"md5": "ffab1055193ffb2fe74822bb575d332a",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class CocitationCiteseer(BaseData):
|
||||
r"""The Co-citation Citeseer dataset is a citation network dataset for vertex classification task.
|
||||
More details see the `HyperGCN <https://papers.nips.cc/paper/2019/file/1efa39bcaec6f3900149160693694536-Paper.pdf>`_ paper.
|
||||
|
||||
The content of the Co-citation Citaseer dataset includes the following:
|
||||
|
||||
- ``num_classes``: The number of classes: :math:`6`.
|
||||
- ``num_vertices``: The number of vertices: :math:`3,312`.
|
||||
- ``num_edges``: The number of edges: :math:`1,079`.
|
||||
- ``dim_features``: The dimension of features: :math:`3,703`.
|
||||
- ``features``: The vertex feature matrix. ``torch.Tensor`` with size :math:`(3,312 \times 3,703)`.
|
||||
- ``edge_list``: The edge list. ``List`` with length :math:`1,079`.
|
||||
- ``labels``: The label list. ``torch.LongTensor`` with size :math:`(3,312, )`.
|
||||
- ``train_mask``: The train mask. ``torch.BoolTensor`` with size :math:`(3,312, )`.
|
||||
- ``val_mask``: The validation mask. ``torch.BoolTensor`` with size :math:`(3,312, )`.
|
||||
- ``test_mask``: The test mask. ``torch.BoolTensor`` with size :math:`(3,312, )`.
|
||||
|
||||
Args:
|
||||
``data_root`` (``str``, optional): The ``data_root`` has stored the data. If set to ``None``, this function will auto-download from server and save into the default direction ``~/.dhg/datasets/``. Defaults to ``None``.
|
||||
"""
|
||||
|
||||
def __init__(self, data_root: Optional[str] = None) -> None:
|
||||
super().__init__("cocitation_citeseer", data_root)
|
||||
self._content = {
|
||||
"num_classes": 6,
|
||||
"num_vertices": 3312,
|
||||
"num_edges": 1079,
|
||||
"dim_features": 3703,
|
||||
"features": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "features.pkl",
|
||||
"md5": "1ee0dc89e0d5f5ac9187b55a407683e8",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_tensor, partial(norm_ft, ord=1)],
|
||||
},
|
||||
"edge_list": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "edge_list.pkl",
|
||||
"md5": "6687b2e96159c534a424253f536b49ae",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
},
|
||||
"labels": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "labels.pkl",
|
||||
"md5": "71069f78e83fa85dd6a4b9b6570447c2",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_long_tensor],
|
||||
},
|
||||
"train_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "train_mask.pkl",
|
||||
"md5": "3b831318fc3d3e588bead5ba469fe38f",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
"val_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "val_mask.pkl",
|
||||
"md5": "c22eb5b7493908042c7e039c8bb5a82e",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
"test_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "test_mask.pkl",
|
||||
"md5": "c22eb5b7493908042c7e039c8bb5a82e",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class CocitationPubmed(BaseData):
|
||||
r"""The Co-citation PubMed dataset is a citation network dataset for vertex classification task.
|
||||
More details see the `HyperGCN <https://papers.nips.cc/paper/2019/file/1efa39bcaec6f3900149160693694536-Paper.pdf>`_ paper.
|
||||
|
||||
The content of the Co-citation PubMed dataset includes the following:
|
||||
|
||||
- ``num_classes``: The number of classes: :math:`3`.
|
||||
- ``num_vertices``: The number of vertices: :math:`19,717`.
|
||||
- ``num_edges``: The number of edges: :math:`7,963`.
|
||||
- ``dim_features``: The dimension of features: :math:`500`.
|
||||
- ``features``: The vertex feature matrix. ``torch.Tensor`` with size :math:`(19,717 \times 500)`.
|
||||
- ``edge_list``: The edge list. ``List`` with length :math:`7,963`.
|
||||
- ``labels``: The label list. ``torch.LongTensor`` with size :math:`(19,717, )`.
|
||||
- ``train_mask``: The train mask. ``torch.BoolTensor`` with size :math:`(19,717, )`.
|
||||
- ``val_mask``: The validation mask. ``torch.BoolTensor`` with size :math:`(19,717, )`.
|
||||
- ``test_mask``: The test mask. ``torch.BoolTensor`` with size :math:`(19,717, )`.
|
||||
|
||||
Args:
|
||||
``data_root`` (``str``, optional): The ``data_root`` has stored the data. If set to ``None``, this function will auto-download from server and save into the default direction ``~/.dhg/datasets/``. Defaults to ``None``.
|
||||
"""
|
||||
|
||||
def __init__(self, data_root: Optional[str] = None) -> None:
|
||||
super().__init__("cocitation_pubmed", data_root)
|
||||
self._content = {
|
||||
"num_classes": 3,
|
||||
"num_vertices": 19717,
|
||||
"num_edges": 7963,
|
||||
"dim_features": 500,
|
||||
"features": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "features.pkl",
|
||||
"md5": "f89502c432ca451156a8235c5efc034e",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_tensor, partial(norm_ft, ord=1)],
|
||||
},
|
||||
"edge_list": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "edge_list.pkl",
|
||||
"md5": "c5fbedf63e5be527f200e8c4e0391b00",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
},
|
||||
"labels": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "labels.pkl",
|
||||
"md5": "c039f778409a15f9b2ceefacad9c2202",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_long_tensor],
|
||||
},
|
||||
"train_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "train_mask.pkl",
|
||||
"md5": "81b422937f3adccd89a334d7093b67d7",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
"val_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "val_mask.pkl",
|
||||
"md5": "10717940ddbfa3e4f6c0b148bb394f79",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
"test_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "test_mask.pkl",
|
||||
"md5": "10717940ddbfa3e4f6c0b148bb394f79",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
import requests
|
||||
|
||||
from easygraph.utils.exception import EasyGraphError
|
||||
|
||||
|
||||
def request_text_from_url(url):
|
||||
"""Requests text data from the specified URL.
|
||||
|
||||
Args:
|
||||
url (str): The URL from which to request data.
|
||||
|
||||
Returns:
|
||||
str: The text content of the response if the request is successful.
|
||||
|
||||
Raises:
|
||||
EasyGraphError: If a connection error occurs or the HTTP response status code indicates failure.
|
||||
"""
|
||||
try:
|
||||
r = requests.get(url)
|
||||
except requests.ConnectionError:
|
||||
raise EasyGraphError("Connection Error!")
|
||||
|
||||
if r.ok:
|
||||
return r.text
|
||||
else:
|
||||
raise EasyGraphError(f"Error: HTTP response {r.status_code}")
|
||||
|
||||
|
||||
class contact_primary_school:
|
||||
"""A class for loading and processing the primary school contact network hypergraph dataset.
|
||||
|
||||
This class loads hyperedge, node label, and label name data from specified URLs and generates a hypergraph.
|
||||
|
||||
Attributes:
|
||||
data_root (str): The root URL for the data. If not provided, it is set to None.
|
||||
hyperedges_path (str): The URL for the hyperedge data.
|
||||
node_labels_path (str): The URL for the node label data.
|
||||
label_names_path (str): The URL for the label name data.
|
||||
_hyperedges (list): A list storing hyperedges.
|
||||
_node_labels (list): A list storing node labels.
|
||||
_label_names (list): A list storing label names.
|
||||
_node_names (list): A list storing node names (currently unused).
|
||||
_content (dict): A dictionary containing dataset statistics and data.
|
||||
"""
|
||||
|
||||
def __init__(self, data_root=None):
|
||||
"""Initializes an instance of the contact_primary_school class.
|
||||
|
||||
Args:
|
||||
data_root (str, optional): The root URL for the data. Defaults to None.
|
||||
"""
|
||||
self.data_root = "https://" if data_root is not None else data_root
|
||||
self.hyperedges_path = "https://gitlab.com/easy-graph/easygraph-data-contact-primary-school/-/raw/main/hyperedges-contact-primary-school.txt?inline=false"
|
||||
self.node_labels_path = "https://gitlab.com/easy-graph/easygraph-data-contact-primary-school/-/raw/main/node-labels-contact-primary-school.txt?ref_type=heads&inline=false"
|
||||
# self.node_names_path = "https://gitlab.com/easy-graph/easygraph-data-house-committees/-/raw/main/node-names-house-committees.txt?ref_type=heads&inline=false"
|
||||
self.label_names_path = "https://gitlab.com/easy-graph/easygraph-data-contact-primary-school/-/raw/main/label-names-contact-primary-school.txt?ref_type=heads&inline=false"
|
||||
self._hyperedges = []
|
||||
self._node_labels = []
|
||||
self._label_names = []
|
||||
self._node_names = []
|
||||
self.generate_hypergraph(
|
||||
hyperedges_path=self.hyperedges_path,
|
||||
node_labels_path=self.node_labels_path,
|
||||
# node_names_path=self.node_names_path,
|
||||
label_names_path=self.label_names_path,
|
||||
)
|
||||
self._content = {
|
||||
"num_classes": len(self._label_names),
|
||||
"num_vertices": len(self._node_labels),
|
||||
"num_edges": len(self._hyperedges),
|
||||
"edge_list": self._hyperedges,
|
||||
"labels": self._node_labels,
|
||||
}
|
||||
|
||||
def __getitem__(self, key: str):
|
||||
"""Accesses data in the _content dictionary by key.
|
||||
|
||||
Args:
|
||||
key (str): The key of the data to access.
|
||||
|
||||
Returns:
|
||||
Any: The value corresponding to the key in the _content dictionary.
|
||||
"""
|
||||
return self._content[key]
|
||||
|
||||
def process_label_txt(self, data_str, delimiter="\n", transform_fun=str):
|
||||
"""Processes label data read from a text file.
|
||||
|
||||
Args:
|
||||
data_str (str): A string containing label data.
|
||||
delimiter (str, optional): The delimiter used to split the string. Defaults to "\n".
|
||||
transform_fun (callable, optional): A function used to transform each label. Defaults to str.
|
||||
|
||||
Returns:
|
||||
list: A list of processed labels.
|
||||
"""
|
||||
data_str = data_str.strip()
|
||||
data_lst = data_str.split(delimiter)
|
||||
final_lst = []
|
||||
for data in data_lst:
|
||||
data = data.strip()
|
||||
data = transform_fun(data)
|
||||
final_lst.append(data)
|
||||
return final_lst
|
||||
|
||||
@property
|
||||
def node_labels(self):
|
||||
"""Gets the list of node labels.
|
||||
|
||||
Returns:
|
||||
list: A list of node labels.
|
||||
"""
|
||||
return self._node_labels
|
||||
|
||||
"""
|
||||
@property
|
||||
def node_names(self):
|
||||
return self._node_names
|
||||
"""
|
||||
|
||||
@property
|
||||
def label_names(self):
|
||||
"""Gets the list of label names.
|
||||
|
||||
Returns:
|
||||
list: A list of label names.
|
||||
"""
|
||||
return self._label_names
|
||||
|
||||
@property
|
||||
def hyperedges(self):
|
||||
"""Gets the list of hyperedges.
|
||||
|
||||
Returns:
|
||||
list: A list of hyperedges.
|
||||
"""
|
||||
return self._hyperedges
|
||||
|
||||
def generate_hypergraph(
|
||||
self,
|
||||
hyperedges_path=None,
|
||||
node_labels_path=None,
|
||||
# node_names_path=None,
|
||||
label_names_path=None,
|
||||
):
|
||||
"""Generates hypergraph data from specified URLs.
|
||||
|
||||
Args:
|
||||
hyperedges_path (str, optional): The URL for the hyperedge data. Defaults to None.
|
||||
node_labels_path (str, optional): The URL for the node label data. Defaults to None.
|
||||
label_names_path (str, optional): The URL for the label name data. Defaults to None.
|
||||
"""
|
||||
|
||||
def fun(data):
|
||||
"""Converts the input data to an integer and subtracts 1.
|
||||
|
||||
Args:
|
||||
data (str): The input string data.
|
||||
|
||||
Returns:
|
||||
int: The converted integer data.
|
||||
"""
|
||||
data = int(data) - 1
|
||||
return data
|
||||
|
||||
hyperedges_info = request_text_from_url(hyperedges_path)
|
||||
hyperedges_info = hyperedges_info.strip()
|
||||
hyperedges_lst = hyperedges_info.split("\n")
|
||||
for hyperedge in hyperedges_lst:
|
||||
hyperedge = hyperedge.strip()
|
||||
hyperedge = [int(i) - 1 for i in hyperedge.split(",")]
|
||||
self._hyperedges.append(tuple(hyperedge))
|
||||
# print(self.hyperedges)
|
||||
|
||||
node_labels_info = request_text_from_url(node_labels_path)
|
||||
|
||||
process_node_labels_info = self.process_label_txt(
|
||||
node_labels_info, transform_fun=fun
|
||||
)
|
||||
self._node_labels = process_node_labels_info
|
||||
label_names_info = request_text_from_url(label_names_path)
|
||||
process_label_names_info = self.process_label_txt(label_names_info)
|
||||
self._label_names = process_label_names_info
|
||||
@@ -0,0 +1,85 @@
|
||||
from typing import Optional
|
||||
|
||||
from easygraph.datapipe import load_from_pickle
|
||||
from easygraph.datapipe import to_bool_tensor
|
||||
from easygraph.datapipe import to_long_tensor
|
||||
from easygraph.datasets.hypergraph.hypergraph_dataset_base import BaseData
|
||||
|
||||
|
||||
class Cooking200(BaseData):
|
||||
r"""The Cooking 200 dataset is collected from `Yummly.com <https://www.yummly.com/>`_ for vertex classification task.
|
||||
It is a hypergraph dataset, in which vertex denotes the dish and hyperedge denotes
|
||||
the ingredient. Each dish is also associated with category information, which indicates the dish's cuisine like
|
||||
Chinese, Japanese, French, and Russian.
|
||||
|
||||
The content of the Cooking200 dataset includes the following:
|
||||
|
||||
- ``num_classes``: The number of classes: :math:`20`.
|
||||
- ``num_vertices``: The number of vertices: :math:`7,403`.
|
||||
- ``num_edges``: The number of edges: :math:`2,755`.
|
||||
- ``edge_list``: The edge list. ``List`` with length :math:`(2,755)`.
|
||||
- ``labels``: The label list. ``torch.LongTensor`` with size :math:`(7,403)`.
|
||||
- ``train_mask``: The train mask. ``torch.BoolTensor`` with size :math:`(7,403)`.
|
||||
- ``val_mask``: The validation mask. ``torch.BoolTensor`` with size :math:`(7,403)`.
|
||||
- ``test_mask``: The test mask. ``torch.BoolTensor`` with size :math:`(7,403)`.
|
||||
|
||||
Args:
|
||||
``data_root`` (``str``, optional): The ``data_root`` has stored the data. If set to ``None``, this function will auto-download from server and save into the default direction ``~/.dhg/datasets/``. Defaults to ``None``.
|
||||
"""
|
||||
|
||||
def __init__(self, data_root: Optional[str] = None) -> None:
|
||||
super().__init__("cooking_200", data_root)
|
||||
self._content = {
|
||||
"num_classes": 20,
|
||||
"num_vertices": 7403,
|
||||
"num_edges": 2755,
|
||||
"edge_list": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "edge_list.pkl",
|
||||
"md5": "2cd32e13dd4e33576c43936542975220",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
},
|
||||
"labels": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "labels.pkl",
|
||||
"md5": "f1f3c0399c9c28547088f44e0bfd5c81",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_long_tensor],
|
||||
},
|
||||
"train_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "train_mask.pkl",
|
||||
"md5": "66ea36bae024aaaed289e1998fe894bd",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
"val_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "val_mask.pkl",
|
||||
"md5": "6c0d3d8b752e3955c64788cc65dcd018",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
"test_mask": {
|
||||
"upon": [
|
||||
{
|
||||
"filename": "test_mask.pkl",
|
||||
"md5": "0e1564904551ba493e1f8a09d103461e",
|
||||
}
|
||||
],
|
||||
"loader": load_from_pickle,
|
||||
"preprocess": [to_bool_tensor],
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
|
||||
from easygraph.datapipe import compose_pipes
|
||||
from easygraph.datasets.hypergraph._global import DATASETS_ROOT
|
||||
from easygraph.datasets.hypergraph._global import REMOTE_DATASETS_ROOT
|
||||
from easygraph.datasets.utils import download_and_check
|
||||
|
||||
|
||||
class BaseData:
|
||||
r"""The Base Class of all datasets.
|
||||
|
||||
::
|
||||
|
||||
self._content = {
|
||||
'item': {
|
||||
'upon': [
|
||||
{'filename': 'part1.pkl', 'md5': 'xxxxx',},
|
||||
{'filename': 'part2.pkl', 'md5': 'xxxxx',},
|
||||
],
|
||||
'loader': loader_function,
|
||||
'preprocess': [datapipe1, datapipe2],
|
||||
},
|
||||
...
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, data_root=None):
|
||||
# configure the data local/remote root
|
||||
self.name = name
|
||||
if data_root is None:
|
||||
self.data_root = DATASETS_ROOT / name
|
||||
else:
|
||||
self.data_root = Path(data_root) / name
|
||||
self.remote_root = REMOTE_DATASETS_ROOT + name + "/"
|
||||
# init
|
||||
self._content = {}
|
||||
self._raw = {}
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"This is {self.name} dataset:\n"
|
||||
+ "\n".join(f" -> {k}" for k in self.content)
|
||||
+ "\nPlease try `data['name']` to get the specified data."
|
||||
)
|
||||
|
||||
@property
|
||||
def content(self):
|
||||
r"""Return the content of the dataset."""
|
||||
return list(self._content.keys())
|
||||
|
||||
def needs_to_load(self, item_name: str) -> bool:
|
||||
r"""Return whether the ``item_name`` of the dataset needs to be loaded.
|
||||
|
||||
Args:
|
||||
``item_name`` (``str``): The name of the item in the dataset.
|
||||
"""
|
||||
assert item_name in self.content, f"{item_name} is not provided in the Data"
|
||||
return (
|
||||
isinstance(self._content[item_name], dict)
|
||||
and "upon" in self._content[item_name]
|
||||
and "loader" in self._content[item_name]
|
||||
)
|
||||
|
||||
def __getitem__(self, key: str) -> Any:
|
||||
if self.needs_to_load(key):
|
||||
cur_cfg = self._content[key]
|
||||
if cur_cfg.get("cache", None) is None:
|
||||
# get raw data
|
||||
item = self.raw(key)
|
||||
# preprocess and cache
|
||||
pipes = cur_cfg.get("preprocess", None)
|
||||
if pipes is not None:
|
||||
cur_cfg["cache"] = compose_pipes(*pipes)(item)
|
||||
else:
|
||||
cur_cfg["cache"] = item
|
||||
return cur_cfg["cache"]
|
||||
else:
|
||||
return self._content[key]
|
||||
|
||||
def raw(self, key: str) -> Any:
|
||||
r"""Return the ``key`` of the dataset with un-preprocessed format."""
|
||||
if self.needs_to_load(key):
|
||||
cur_cfg = self._content[key]
|
||||
if self._raw.get(key, None) is None:
|
||||
upon = cur_cfg["upon"]
|
||||
if len(upon) == 0:
|
||||
return None
|
||||
self.fetch_files(cur_cfg["upon"])
|
||||
file_path_list = [
|
||||
self.data_root / u["filename"] for u in cur_cfg["upon"]
|
||||
]
|
||||
if len(file_path_list) == 1:
|
||||
self._raw[key] = cur_cfg["loader"](file_path_list[0])
|
||||
else:
|
||||
# here, you should implement a multi-file loader
|
||||
self._raw[key] = cur_cfg["loader"](file_path_list)
|
||||
return self._raw[key]
|
||||
else:
|
||||
return self._content[key]
|
||||
|
||||
def fetch_files(self, files: List[Dict[str, str]]):
|
||||
r"""Download and check the files if they are not exist.
|
||||
|
||||
Args:
|
||||
``files`` (``List[Dict[str, str]]``): The files to download, each element
|
||||
in the list is a dict with at lease two keys: ``filename`` and ``md5``.
|
||||
If extra key ``bk_url`` is provided, it will be used to download the
|
||||
file from the backup url.
|
||||
"""
|
||||
for file in files:
|
||||
cur_filename = file["filename"]
|
||||
cur_url = file.get("bk_url", None)
|
||||
if cur_url is None:
|
||||
cur_url = self.remote_root + cur_filename
|
||||
download_and_check(cur_url, self.data_root / cur_filename, file["md5"])
|
||||
@@ -0,0 +1,78 @@
|
||||
import os.path as osp
|
||||
|
||||
import numpy as np
|
||||
import scipy.sparse as sp
|
||||
import torch
|
||||
|
||||
from torch_geometric.data import Data
|
||||
from torch_sparse import coalesce
|
||||
|
||||
|
||||
__all__ = ["load_line_expansion_dataset"]
|
||||
|
||||
|
||||
def load_line_expansion_dataset(
|
||||
path=None, dataset="cocitation-cora", train_percent=0.5
|
||||
):
|
||||
# load edges, features, and labels.
|
||||
print("Loading {} dataset...".format(dataset))
|
||||
|
||||
file_name = f"{dataset}.content"
|
||||
p2idx_features_labels = osp.join(path, dataset, file_name)
|
||||
idx_features_labels = np.genfromtxt(p2idx_features_labels, dtype=np.dtype(str))
|
||||
# features = np.array(idx_features_labels[:, 1:-1])
|
||||
features = sp.csr_matrix(idx_features_labels[:, 1:-1], dtype=np.float32)
|
||||
# labels = encode_onehot(idx_features_labels[:, -1])
|
||||
labels = torch.LongTensor(idx_features_labels[:, -1].astype(float))
|
||||
|
||||
print("load features")
|
||||
|
||||
# build graph
|
||||
idx = np.array(idx_features_labels[:, 0], dtype=np.int32)
|
||||
idx_map = {j: i for i, j in enumerate(idx)}
|
||||
|
||||
file_name = f"{dataset}.edges"
|
||||
p2edges_unordered = osp.join(path, dataset, file_name)
|
||||
edges_unordered = np.genfromtxt(p2edges_unordered, dtype=np.int32)
|
||||
|
||||
edges = np.array(
|
||||
list(map(idx_map.get, edges_unordered.flatten())), dtype=np.int32
|
||||
).reshape(edges_unordered.shape)
|
||||
|
||||
print("load edges")
|
||||
|
||||
# From adjacency matrix to edge_list
|
||||
edge_index = edges.T
|
||||
# ipdb.set_trace()
|
||||
assert edge_index[0].max() == edge_index[1].min() - 1
|
||||
|
||||
# check if values in edge_index is consecutive. i.e. no missing value for node_id/he_id.
|
||||
assert len(np.unique(edge_index)) == edge_index.max() + 1
|
||||
|
||||
num_nodes = edge_index[0].max() + 1
|
||||
num_he = edge_index[1].max() - num_nodes + 1
|
||||
edge_index = np.hstack((edge_index, edge_index[::-1, :]))
|
||||
|
||||
# build torch data class
|
||||
data = Data(
|
||||
x=torch.FloatTensor(np.array(features[:num_nodes].todense())),
|
||||
edge_index=torch.LongTensor(edge_index),
|
||||
y=labels[:num_nodes],
|
||||
)
|
||||
|
||||
# used user function to override the default function.
|
||||
# the following will also sort the edge_index and remove duplicates.
|
||||
total_num_node_id_he_id = len(np.unique(edge_index))
|
||||
data.edge_index, data.edge_attr = coalesce(
|
||||
data.edge_index, None, total_num_node_id_he_id, total_num_node_id_he_id
|
||||
)
|
||||
n_x = num_nodes
|
||||
# n_x = n_expanded
|
||||
num_class = len(np.unique(labels[:num_nodes].numpy()))
|
||||
data.n_x = n_x
|
||||
# add parameters to attribute
|
||||
|
||||
data.train_percent = train_percent
|
||||
data.num_hyperedges = num_he
|
||||
|
||||
return data
|
||||
@@ -0,0 +1,113 @@
|
||||
import requests
|
||||
|
||||
from easygraph.utils.exception import EasyGraphError
|
||||
|
||||
|
||||
def request_text_from_url(url):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
except requests.ConnectionError:
|
||||
raise EasyGraphError("Connection Error!")
|
||||
|
||||
if r.ok:
|
||||
return r.text
|
||||
else:
|
||||
raise EasyGraphError(f"Error: HTTP response {r.status_code}")
|
||||
|
||||
|
||||
class mathoverflow_answers:
|
||||
def __init__(self, data_root=None):
|
||||
self.data_root = "https://" if data_root is not None else data_root
|
||||
self.hyperedges_path = "https://gitlab.com/easy-graph/easygraph-data-mathoverflow-answers/-/raw/main/hyperedges-mathoverflow-answers.txt?inline=false"
|
||||
self.node_labels_path = "https://gitlab.com/easy-graph/easygraph-data-mathoverflow-answers/-/raw/main/node-labels-mathoverflow-answers.txt?ref_type=heads&inline=false"
|
||||
# self.node_names_path = "https://gitlab.com/easy-graph/easygraph-data-house-committees/-/raw/main/node-names-house-committees.txt?ref_type=heads&inline=false"
|
||||
self.label_names_path = "https://gitlab.com/easy-graph/easygraph-data-mathoverflow-answers/-/raw/main/label-names-mathoverflow-answers.txt?ref_type=heads&inline=false"
|
||||
self._hyperedges = []
|
||||
self._node_labels = []
|
||||
self._label_names = []
|
||||
self._node_names = []
|
||||
self.generate_hypergraph(
|
||||
hyperedges_path=self.hyperedges_path,
|
||||
node_labels_path=self.node_labels_path,
|
||||
# node_names_path=self.node_names_path,
|
||||
label_names_path=self.label_names_path,
|
||||
)
|
||||
self._content = {
|
||||
"num_classes": len(self._label_names),
|
||||
"num_vertices": len(self._node_labels),
|
||||
"num_edges": len(self._hyperedges),
|
||||
"edge_list": self._hyperedges,
|
||||
"labels": self._node_labels,
|
||||
}
|
||||
|
||||
def __getitem__(self, key: str):
|
||||
return self._content[key]
|
||||
|
||||
def process_label_txt(self, data_str, delimiter="\n", transform_fun=str):
|
||||
data_str = data_str.strip()
|
||||
data_lst = data_str.split(delimiter)
|
||||
final_lst = []
|
||||
for data in data_lst:
|
||||
data = data.strip()
|
||||
data = transform_fun(data)
|
||||
final_lst.append(data)
|
||||
return final_lst
|
||||
|
||||
@property
|
||||
def node_labels(self):
|
||||
return self._node_labels
|
||||
|
||||
"""
|
||||
@property
|
||||
def node_names(self):
|
||||
return self._node_names
|
||||
"""
|
||||
|
||||
@property
|
||||
def label_names(self):
|
||||
return self._label_names
|
||||
|
||||
@property
|
||||
def hyperedges(self):
|
||||
return self._hyperedges
|
||||
|
||||
def generate_hypergraph(
|
||||
self,
|
||||
hyperedges_path=None,
|
||||
node_labels_path=None,
|
||||
# node_names_path=None,
|
||||
label_names_path=None,
|
||||
):
|
||||
def fun(data):
|
||||
data = int(data) - 1
|
||||
return data
|
||||
|
||||
hyperedges_info = request_text_from_url(hyperedges_path)
|
||||
hyperedges_info = hyperedges_info.strip()
|
||||
hyperedges_lst = hyperedges_info.split("\n")
|
||||
for hyperedge in hyperedges_lst:
|
||||
hyperedge = hyperedge.strip()
|
||||
hyperedge = [int(i) - 1 for i in hyperedge.split(",")]
|
||||
self._hyperedges.append(tuple(hyperedge))
|
||||
# print(self.hyperedges)
|
||||
"""
|
||||
node_labels_info = request_text_from_url(node_labels_path)
|
||||
|
||||
process_node_labels_info = self.process_label_txt(
|
||||
node_labels_info, transform_fun=fun
|
||||
)
|
||||
self._node_labels = process_node_labels_info
|
||||
"""
|
||||
node_labels_info = request_text_from_url(node_labels_path)
|
||||
node_labels_info = node_labels_info.strip()
|
||||
node_labels_lst = node_labels_info.split("\n")
|
||||
for node_label in node_labels_lst:
|
||||
node_label = node_label.strip()
|
||||
node_label = [int(i) - 1 for i in node_label.split(",")]
|
||||
self._node_labels.append(tuple(node_label))
|
||||
# print("process_node_labels_info:", process_node_labels_info)
|
||||
# print("process_node_names_info:", process_node_names_info)
|
||||
label_names_info = request_text_from_url(label_names_path)
|
||||
process_label_names_info = self.process_label_txt(label_names_info)
|
||||
self._label_names = process_label_names_info
|
||||
# print("process_label_names_info:", process_label_names_info)
|
||||
@@ -0,0 +1,106 @@
|
||||
import requests
|
||||
|
||||
from easygraph.utils.exception import EasyGraphError
|
||||
|
||||
|
||||
def request_text_from_url(url):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
except requests.ConnectionError:
|
||||
raise EasyGraphError("Connection Error!")
|
||||
|
||||
if r.ok:
|
||||
return r.text
|
||||
else:
|
||||
raise EasyGraphError(f"Error: HTTP response {r.status_code}")
|
||||
|
||||
|
||||
class senate_committees:
|
||||
def __init__(self, data_root=None):
|
||||
self.data_root = "https://" if data_root is not None else data_root
|
||||
self.hyperedges_path = "https://gitlab.com/easy-graph/easygraph-data-senate-committees/-/raw/main/hyperedges-senate-committees.txt?inline=false"
|
||||
self.node_labels_path = "https://gitlab.com/easy-graph/easygraph-data-senate-committees/-/raw/main/node-labels-senate-committees.txt?ref_type=heads&inline=false"
|
||||
self.node_names_path = "https://gitlab.com/easy-graph/easygraph-data-senate-committees/-/raw/main/node-names-senate-committees.txt?ref_type=heads&inline=false"
|
||||
self.label_names_path = "https://gitlab.com/easy-graph/easygraph-data-senate-committees/-/raw/main/label-names-senate-committees.txt?ref_type=heads&inline=false"
|
||||
self._hyperedges = []
|
||||
self._node_labels = []
|
||||
self._label_names = []
|
||||
self._node_names = []
|
||||
self.generate_hypergraph(
|
||||
hyperedges_path=self.hyperedges_path,
|
||||
node_labels_path=self.node_labels_path,
|
||||
node_names_path=self.node_names_path,
|
||||
label_names_path=self.label_names_path,
|
||||
)
|
||||
self._content = {
|
||||
"num_classes": len(self._label_names),
|
||||
"num_vertices": len(self._node_labels),
|
||||
"num_edges": len(self._hyperedges),
|
||||
"edge_list": self._hyperedges,
|
||||
"labels": self._node_labels,
|
||||
}
|
||||
|
||||
def __getitem__(self, key: str):
|
||||
return self._content[key]
|
||||
|
||||
def process_label_txt(self, data_str, delimiter="\n", transform_fun=str):
|
||||
data_str = data_str.strip()
|
||||
data_lst = data_str.split(delimiter)
|
||||
final_lst = []
|
||||
for data in data_lst:
|
||||
data = data.strip()
|
||||
data = transform_fun(data)
|
||||
final_lst.append(data)
|
||||
return final_lst
|
||||
|
||||
@property
|
||||
def node_labels(self):
|
||||
return self._node_labels
|
||||
|
||||
@property
|
||||
def node_names(self):
|
||||
return self._node_names
|
||||
|
||||
@property
|
||||
def label_names(self):
|
||||
return self._label_names
|
||||
|
||||
@property
|
||||
def hyperedges(self):
|
||||
return self._hyperedges
|
||||
|
||||
def generate_hypergraph(
|
||||
self,
|
||||
hyperedges_path=None,
|
||||
node_labels_path=None,
|
||||
node_names_path=None,
|
||||
label_names_path=None,
|
||||
):
|
||||
def fun(data):
|
||||
data = int(data) - 1
|
||||
return data
|
||||
|
||||
hyperedges_info = request_text_from_url(hyperedges_path)
|
||||
hyperedges_info = hyperedges_info.strip()
|
||||
hyperedges_lst = hyperedges_info.split("\n")
|
||||
for hyperedge in hyperedges_lst:
|
||||
hyperedge = hyperedge.strip()
|
||||
hyperedge = [int(i) - 1 for i in hyperedge.split(",")]
|
||||
self._hyperedges.append(tuple(hyperedge))
|
||||
# print(self.hyperedges)
|
||||
|
||||
node_labels_info = request_text_from_url(node_labels_path)
|
||||
|
||||
process_node_labels_info = self.process_label_txt(
|
||||
node_labels_info, transform_fun=fun
|
||||
)
|
||||
self._node_labels = process_node_labels_info
|
||||
# print("process_node_labels_info:", process_node_labels_info)
|
||||
node_names_info = request_text_from_url(node_names_path)
|
||||
process_node_names_info = self.process_label_txt(node_names_info)
|
||||
self._node_names = process_node_names_info
|
||||
# print("process_node_names_info:", process_node_names_info)
|
||||
label_names_info = request_text_from_url(label_names_path)
|
||||
process_label_names_info = self.process_label_txt(label_names_info)
|
||||
self._label_names = process_label_names_info
|
||||
# print("process_label_names_info:", process_label_names_info)
|
||||
@@ -0,0 +1,104 @@
|
||||
import requests
|
||||
|
||||
from easygraph.utils.exception import EasyGraphError
|
||||
|
||||
|
||||
def request_text_from_url(url):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
except requests.ConnectionError:
|
||||
raise EasyGraphError("Connection Error!")
|
||||
|
||||
if r.ok:
|
||||
return r.text
|
||||
else:
|
||||
raise EasyGraphError(f"Error: HTTP response {r.status_code}")
|
||||
|
||||
|
||||
class trivago_clicks:
|
||||
def __init__(self, data_root=None):
|
||||
self.data_root = "https://" if data_root is not None else data_root
|
||||
self.hyperedges_path = "https://gitlab.com/easy-graph/easygraph-data-trivago-clicks/-/raw/main/hyperedges-trivago-clicks.txt?inline=false"
|
||||
self.node_labels_path = "https://gitlab.com/easy-graph/easygraph-data-trivago-clicks/-/raw/main/node-labels-trivago-clicks.txt?ref_type=heads&inline=false"
|
||||
# self.node_names_path = "https://gitlab.com/easy-graph/easygraph-data-trivago-clicks/-/raw/main/node-names-house-committees.txt?ref_type=heads&inline=false"
|
||||
self.label_names_path = "https://gitlab.com/easy-graph/easygraph-data-trivago-clicks/-/raw/main/label-names-trivago-clicks.txt?ref_type=heads&inline=false"
|
||||
self._hyperedges = []
|
||||
self._node_labels = []
|
||||
self._label_names = []
|
||||
self._node_names = []
|
||||
self.generate_hypergraph(
|
||||
hyperedges_path=self.hyperedges_path,
|
||||
node_labels_path=self.node_labels_path,
|
||||
# node_names_path=self.node_names_path,
|
||||
label_names_path=self.label_names_path,
|
||||
)
|
||||
self._content = {
|
||||
"num_classes": len(self._label_names),
|
||||
"num_vertices": len(self._node_labels),
|
||||
"num_edges": len(self._hyperedges),
|
||||
"edge_list": self._hyperedges,
|
||||
"labels": self._node_labels,
|
||||
}
|
||||
|
||||
def __getitem__(self, key: str):
|
||||
return self._content[key]
|
||||
|
||||
def process_label_txt(self, data_str, delimiter="\n", transform_fun=str):
|
||||
data_str = data_str.strip()
|
||||
data_lst = data_str.split(delimiter)
|
||||
final_lst = []
|
||||
for data in data_lst:
|
||||
data = data.strip()
|
||||
data = transform_fun(data)
|
||||
final_lst.append(data)
|
||||
return final_lst
|
||||
|
||||
@property
|
||||
def node_labels(self):
|
||||
return self._node_labels
|
||||
|
||||
"""
|
||||
@property
|
||||
def node_names(self):
|
||||
return self._node_names
|
||||
"""
|
||||
|
||||
@property
|
||||
def label_names(self):
|
||||
return self._label_names
|
||||
|
||||
@property
|
||||
def hyperedges(self):
|
||||
return self._hyperedges
|
||||
|
||||
def generate_hypergraph(
|
||||
self,
|
||||
hyperedges_path=None,
|
||||
node_labels_path=None,
|
||||
# node_names_path=None,
|
||||
label_names_path=None,
|
||||
):
|
||||
def fun(data):
|
||||
data = int(data) - 1
|
||||
return data
|
||||
|
||||
hyperedges_info = request_text_from_url(hyperedges_path)
|
||||
hyperedges_info = hyperedges_info.strip()
|
||||
hyperedges_lst = hyperedges_info.split("\n")
|
||||
for hyperedge in hyperedges_lst:
|
||||
hyperedge = hyperedge.strip()
|
||||
hyperedge = [int(i) - 1 for i in hyperedge.split(",")]
|
||||
self._hyperedges.append(tuple(hyperedge))
|
||||
# print(self.hyperedges)
|
||||
|
||||
node_labels_info = request_text_from_url(node_labels_path)
|
||||
|
||||
process_node_labels_info = self.process_label_txt(
|
||||
node_labels_info, transform_fun=fun
|
||||
)
|
||||
self._node_labels = process_node_labels_info
|
||||
# print("process_node_labels_info:", process_node_labels_info)
|
||||
# print("process_node_names_info:", process_node_names_info)
|
||||
label_names_info = request_text_from_url(label_names_path)
|
||||
process_label_names_info = self.process_label_txt(label_names_info)
|
||||
self._label_names = process_label_names_info
|
||||
@@ -0,0 +1,208 @@
|
||||
import requests
|
||||
|
||||
from easygraph.utils.exception import EasyGraphError
|
||||
|
||||
|
||||
def request_text_from_url(url):
|
||||
"""
|
||||
Requests text content from the given URL.
|
||||
|
||||
Args:
|
||||
url (str): The URL from which to request text data.
|
||||
|
||||
Returns:
|
||||
str: The text content of the response if the request is successful.
|
||||
|
||||
Raises:
|
||||
EasyGraphError: If a connection error occurs during the request or if the HTTP response status code is not OK.
|
||||
"""
|
||||
try:
|
||||
r = requests.get(url)
|
||||
except requests.ConnectionError:
|
||||
raise EasyGraphError("Connection Error!")
|
||||
|
||||
if r.ok:
|
||||
return r.text
|
||||
else:
|
||||
raise EasyGraphError(f"Error: HTTP response {r.status_code}")
|
||||
|
||||
|
||||
class walmart_trips:
|
||||
"""
|
||||
A class for loading and processing the Walmart trips hypergraph dataset.
|
||||
|
||||
This class fetches hyperedge, node label, and label name data from predefined URLs,
|
||||
processes the data, and generates a hypergraph representation. It also provides access
|
||||
to various dataset attributes through properties and indexing.
|
||||
|
||||
Attributes:
|
||||
data_root (str): The root URL for the data. If provided during initialization, it is set to "https://";
|
||||
otherwise, it is None.
|
||||
hyperedges_path (str): The URL of the file containing hyperedge information.
|
||||
node_labels_path (str): The URL of the file containing node label information.
|
||||
label_names_path (str): The URL of the file containing label name information.
|
||||
_hyperedges (list): A list of tuples representing hyperedges.
|
||||
_node_labels (list): A list of node labels.
|
||||
_label_names (list): A list of label names.
|
||||
_node_names (list): An empty list reserved for node names (currently unused).
|
||||
_content (dict): A dictionary containing dataset statistics and data, such as the number of classes,
|
||||
vertices, edges, the edge list, and node labels.
|
||||
"""
|
||||
|
||||
def __init__(self, data_root=None, local_path=None):
|
||||
"""
|
||||
Initializes an instance of the walmart_trips class.
|
||||
|
||||
Args:
|
||||
data_root (str, optional): The root URL for the data. If provided, it is set to "https://";
|
||||
otherwise, it is None. Defaults to None.
|
||||
local_path (str, optional): Currently unused. Defaults to None.
|
||||
"""
|
||||
self.data_root = "https://" if data_root is not None else data_root
|
||||
self.hyperedges_path = "https://gitlab.com/easy-graph/easygraph-data-walmart-trips/-/raw/main/hyperedges-walmart-trips.txt?inline=false"
|
||||
self.node_labels_path = "https://gitlab.com/easy-graph/easygraph-data-walmart-trips/-/raw/main/node-labels-walmart-trips.txt?ref_type=heads&inline=false"
|
||||
# self.node_names_path = "https://gitlab.com/easy-graph/easygraph-data-walmart-trips/-/raw/main/node-names-house-committees.txt?ref_type=heads&inline=false"
|
||||
self.label_names_path = "https://gitlab.com/easy-graph/easygraph-data-walmart-trips/-/raw/main/label-names-walmart-trips.txt?ref_type=heads&inline=false"
|
||||
self._hyperedges = []
|
||||
self._node_labels = []
|
||||
self._label_names = []
|
||||
self._node_names = []
|
||||
|
||||
self.generate_hypergraph(
|
||||
hyperedges_path=self.hyperedges_path,
|
||||
node_labels_path=self.node_labels_path,
|
||||
# node_names_path=self.node_names_path,
|
||||
label_names_path=self.label_names_path,
|
||||
)
|
||||
|
||||
self._content = {
|
||||
"num_classes": len(self._label_names),
|
||||
"num_vertices": len(self._node_labels),
|
||||
"num_edges": len(self._hyperedges),
|
||||
"edge_list": self._hyperedges,
|
||||
"labels": self._node_labels,
|
||||
}
|
||||
|
||||
def __getitem__(self, key: str):
|
||||
"""
|
||||
Retrieves a value from the _content dictionary using the specified key.
|
||||
|
||||
Args:
|
||||
key (str): The key used to access the _content dictionary.
|
||||
|
||||
Returns:
|
||||
Any: The value corresponding to the key in the _content dictionary.
|
||||
"""
|
||||
return self._content[key]
|
||||
|
||||
def process_label_txt(self, data_str, delimiter="\n", transform_fun=str):
|
||||
"""
|
||||
Processes a string containing label data into a list of transformed values.
|
||||
|
||||
Args:
|
||||
data_str (str): The input string containing label data.
|
||||
delimiter (str, optional): The delimiter used to split the input string. Defaults to "\n".
|
||||
transform_fun (callable, optional): A function used to transform each label value.
|
||||
Defaults to the str function.
|
||||
|
||||
Returns:
|
||||
list: A list of transformed label values.
|
||||
"""
|
||||
data_str = data_str.strip()
|
||||
data_lst = data_str.split(delimiter)
|
||||
final_lst = []
|
||||
for data in data_lst:
|
||||
data = data.strip()
|
||||
data = transform_fun(data)
|
||||
final_lst.append(data)
|
||||
return final_lst
|
||||
|
||||
@property
|
||||
def node_labels(self):
|
||||
"""
|
||||
Gets the list of node labels.
|
||||
|
||||
Returns:
|
||||
list: A list of node labels.
|
||||
"""
|
||||
return self._node_labels
|
||||
|
||||
"""
|
||||
@property
|
||||
def node_names(self):
|
||||
return self._node_names
|
||||
"""
|
||||
|
||||
@property
|
||||
def label_names(self):
|
||||
"""
|
||||
Gets the list of label names.
|
||||
|
||||
Returns:
|
||||
list: A list of label names.
|
||||
"""
|
||||
return self._label_names
|
||||
|
||||
@property
|
||||
def hyperedges(self):
|
||||
"""
|
||||
Gets the list of hyperedges.
|
||||
|
||||
Returns:
|
||||
list: A list of tuples representing hyperedges.
|
||||
"""
|
||||
return self._hyperedges
|
||||
|
||||
def generate_hypergraph(
|
||||
self,
|
||||
hyperedges_path=None,
|
||||
node_labels_path=None,
|
||||
# node_names_path=None,
|
||||
label_names_path=None,
|
||||
):
|
||||
"""
|
||||
Generates a hypergraph by fetching and processing data from the specified URLs.
|
||||
|
||||
Args:
|
||||
hyperedges_path (str, optional): The URL of the file containing hyperedge information.
|
||||
Defaults to None.
|
||||
node_labels_path (str, optional): The URL of the file containing node label information.
|
||||
Defaults to None.
|
||||
label_names_path (str, optional): The URL of the file containing label name information.
|
||||
Defaults to None.
|
||||
"""
|
||||
|
||||
def fun(data):
|
||||
"""
|
||||
Converts a string to an integer and subtracts 1.
|
||||
|
||||
Args:
|
||||
data (str): The input string to be converted.
|
||||
|
||||
Returns:
|
||||
int: The converted integer value minus 1.
|
||||
"""
|
||||
data = int(data) - 1
|
||||
return data
|
||||
|
||||
hyperedges_info = request_text_from_url(hyperedges_path)
|
||||
hyperedges_info = hyperedges_info.strip()
|
||||
hyperedges_lst = hyperedges_info.split("\n")
|
||||
for hyperedge in hyperedges_lst:
|
||||
hyperedge = hyperedge.strip()
|
||||
hyperedge = [int(i) - 1 for i in hyperedge.split(",")]
|
||||
self._hyperedges.append(tuple(hyperedge))
|
||||
# print(self.hyperedges)
|
||||
|
||||
node_labels_info = request_text_from_url(node_labels_path)
|
||||
|
||||
process_node_labels_info = self.process_label_txt(
|
||||
node_labels_info, transform_fun=fun
|
||||
)
|
||||
self._node_labels = process_node_labels_info
|
||||
# print("process_node_labels_info:", process_node_labels_info)
|
||||
# print("process_node_names_info:", process_node_names_info)
|
||||
label_names_info = request_text_from_url(label_names_path)
|
||||
process_label_names_info = self.process_label_txt(label_names_info)
|
||||
self._label_names = process_label_names_info
|
||||
# print("process_label_names_info:", process_label_names_info)
|
||||
@@ -0,0 +1,93 @@
|
||||
import easygraph as eg
|
||||
|
||||
from .graph_dataset_base import EasyGraphDataset
|
||||
from .utils import _set_labels
|
||||
from .utils import tensor
|
||||
|
||||
|
||||
""" KarateClubDataset for inductive learning. """
|
||||
|
||||
|
||||
class KarateClubDataset(EasyGraphDataset):
|
||||
"""Karate Club dataset for Node Classification
|
||||
|
||||
Zachary's karate club is a social network of a university
|
||||
karate club, described in the paper "An Information Flow
|
||||
Model for Conflict and Fission in Small Groups" by Wayne W. Zachary.
|
||||
The network became a popular example of community structure in
|
||||
networks after its use by Michelle Girvan and Mark Newman in 2002.
|
||||
Official website: `<http://konect.cc/networks/ucidata-zachary/>`_
|
||||
|
||||
Karate Club dataset statistics:
|
||||
|
||||
- Nodes: 34
|
||||
- Edges: 156
|
||||
- Number of Classes: 2
|
||||
|
||||
Parameters
|
||||
----------
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~eg.Graph` object and returns
|
||||
a transformed version. The :class:`~eg.Graph` object will be
|
||||
transformed before every access.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
num_classes : int
|
||||
Number of node classes
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> dataset = KarateClubDataset()
|
||||
>>> num_classes = dataset.num_classes
|
||||
>>> g = dataset[0]
|
||||
>>> labels = g.ndata['label']
|
||||
"""
|
||||
|
||||
def __init__(self, transform=None):
|
||||
super(KarateClubDataset, self).__init__(name="karate_club", transform=transform)
|
||||
|
||||
def process(self):
|
||||
import numpy as np
|
||||
|
||||
kc_graph = eg.get_graph_karateclub()
|
||||
label = np.asarray(
|
||||
[kc_graph.nodes[i]["club"] != "Mr. Hi" for i in kc_graph.nodes]
|
||||
).astype(np.int64)
|
||||
label = tensor(label)
|
||||
|
||||
kc_graph = _set_labels(kc_graph, label)
|
||||
kc_graph.ndata["label"] = label
|
||||
self._graph = kc_graph
|
||||
self._data = [kc_graph]
|
||||
|
||||
@property
|
||||
def num_classes(self):
|
||||
"""Number of classes."""
|
||||
return 2
|
||||
|
||||
def __getitem__(self, idx):
|
||||
r"""Get graph object
|
||||
|
||||
Parameters
|
||||
----------
|
||||
idx : int
|
||||
Item index, KarateClubDataset has only one graph object
|
||||
|
||||
Returns
|
||||
-------
|
||||
:class:`eg.Graph`
|
||||
|
||||
graph structure and labels.
|
||||
|
||||
- ``ndata['label']``: ground truth labels
|
||||
"""
|
||||
assert idx == 0, "This dataset has only one graph"
|
||||
if self._transform is None:
|
||||
return self._graph
|
||||
else:
|
||||
return self._transform(self._graph)
|
||||
|
||||
def __len__(self):
|
||||
r"""The number of graphs in the dataset."""
|
||||
return 1
|
||||
@@ -0,0 +1,217 @@
|
||||
"""PPIDataset for inductive learning."""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
|
||||
from easygraph.classes.directed_graph import DiGraph
|
||||
|
||||
from ..readwrite import json_graph
|
||||
from .graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from .utils import _get_dgl_url
|
||||
from .utils import data_type_dict
|
||||
from .utils import tensor
|
||||
|
||||
|
||||
class PPIDataset(EasyGraphBuiltinDataset):
|
||||
r"""Protein-Protein Interaction dataset for inductive node classification
|
||||
|
||||
A toy Protein-Protein Interaction network dataset. The dataset contains
|
||||
24 graphs. The average number of nodes per graph is 2372. Each node has
|
||||
50 features and 121 labels. 20 graphs for training, 2 for validation
|
||||
and 2 for testing.
|
||||
|
||||
Reference: `<http://snap.stanford.edu/graphsage/>`_
|
||||
|
||||
Statistics:
|
||||
|
||||
- Train examples: 20
|
||||
- Valid examples: 2
|
||||
- Test examples: 2
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mode : str
|
||||
Must be one of ('train', 'valid', 'test').
|
||||
Default: 'train'
|
||||
raw_dir : str
|
||||
Raw file directory to download/contains the input data directory.
|
||||
Default: ~/.eg/
|
||||
force_reload : bool
|
||||
Whether to reload the dataset.
|
||||
Default: False
|
||||
verbose : bool
|
||||
Whether to print out progress information.
|
||||
Default: True.
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~eg.DGLGraph` object and returns
|
||||
a transformed version. The :class:`~eg.DGLGraph` object will be
|
||||
transformed before every access.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
num_labels : int
|
||||
Number of labels for each node
|
||||
labels : Tensor
|
||||
Node labels
|
||||
features : Tensor
|
||||
Node features
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> dataset = PPIDataset(mode='valid')
|
||||
>>> num_labels = dataset.num_labels
|
||||
>>> for g in dataset:
|
||||
.... feat = g.ndata['feat']
|
||||
.... label = g.ndata['label']
|
||||
.... # your code here
|
||||
>>>
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mode="train",
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=False,
|
||||
transform=None,
|
||||
):
|
||||
assert mode in ["train", "valid", "test"]
|
||||
self.mode = mode
|
||||
_url = _get_dgl_url("dataset/ppi.zip")
|
||||
super(PPIDataset, self).__init__(
|
||||
name="ppi",
|
||||
url=_url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def process(self):
|
||||
graph_file = os.path.join(
|
||||
self.save_path, "ppi", "{}_graph.json".format(self.mode)
|
||||
)
|
||||
label_file = os.path.join(
|
||||
self.save_path, "ppi", "{}_labels.npy".format(self.mode)
|
||||
)
|
||||
feat_file = os.path.join(
|
||||
self.save_path, "ppi", "{}_feats.npy".format(self.mode)
|
||||
)
|
||||
graph_id_file = os.path.join(
|
||||
self.save_path, "ppi", "{}_graph_id.npy".format(self.mode)
|
||||
)
|
||||
|
||||
g_data = json.load(open(graph_file))
|
||||
self._labels = np.load(label_file)
|
||||
self._feats = np.load(feat_file)
|
||||
self.graph = DiGraph(json_graph.node_link_graph(g_data))
|
||||
graph_id = np.load(graph_id_file)
|
||||
|
||||
# lo, hi means the range of graph ids for different portion of the dataset,
|
||||
# 20 graphs for training, 2 for validation and 2 for testing.
|
||||
lo, hi = 1, 21
|
||||
if self.mode == "valid":
|
||||
lo, hi = 21, 23
|
||||
elif self.mode == "test":
|
||||
lo, hi = 23, 25
|
||||
|
||||
graph_masks = []
|
||||
self.graphs = []
|
||||
for g_id in range(lo, hi):
|
||||
g_mask = np.where(graph_id == g_id)[0]
|
||||
graph_masks.append(g_mask)
|
||||
g = self.graph.nodes_subgraph(g_mask)
|
||||
g.ndata["feat"] = tensor(
|
||||
self._feats[g_mask], dtype=data_type_dict()["float32"]
|
||||
)
|
||||
g.ndata["label"] = tensor(
|
||||
self._labels[g_mask], dtype=data_type_dict()["float32"]
|
||||
)
|
||||
self.graphs.append(g)
|
||||
|
||||
def has_cache(self):
|
||||
graph_list_path = os.path.join(
|
||||
self.save_path, "{}_eg_graph_list.bin".format(self.mode)
|
||||
)
|
||||
g_path = os.path.join(self.save_path, "{}_eg_graph.bin".format(self.mode))
|
||||
info_path = os.path.join(self.save_path, "{}_info.pkl".format(self.mode))
|
||||
return (
|
||||
os.path.exists(graph_list_path)
|
||||
and os.path.exists(g_path)
|
||||
and os.path.exists(info_path)
|
||||
)
|
||||
|
||||
def save(self):
|
||||
graph_list_path = os.path.join(
|
||||
self.save_path, "{}_eg_graph_list.bin".format(self.mode)
|
||||
)
|
||||
g_path = os.path.join(self.save_path, "{}_eg_graph.bin".format(self.mode))
|
||||
info_path = os.path.join(self.save_path, "{}_info.pkl".format(self.mode))
|
||||
# save_graphs(graph_list_path, self.graphs)
|
||||
# save_graphs(g_path, self.graph)
|
||||
# save_info(info_path, {'labels': self._labels, 'feats': self._feats})
|
||||
|
||||
# def load(self):
|
||||
# graph_list_path = os.path.join(self.save_path, '{}_eg_graph_list.bin'.format(self.mode))
|
||||
# g_path = os.path.join(self.save_path, '{}_eg_graph.bin'.format(self.mode))
|
||||
# info_path = os.path.join(self.save_path, '{}_info.pkl'.format(self.mode))
|
||||
# self.graphs = load_graphs(graph_list_path)[0]
|
||||
# g, _ = load_graphs(g_path)
|
||||
# self.graph = g[0]
|
||||
# info = load_info(info_path)
|
||||
# self._labels = info['labels']
|
||||
# self._feats = info['feats']
|
||||
|
||||
@property
|
||||
def num_labels(self):
|
||||
return 121
|
||||
|
||||
def __len__(self):
|
||||
"""Return number of samples in this dataset."""
|
||||
return len(self.graphs)
|
||||
|
||||
def __getitem__(self, item):
|
||||
"""Get the item^th sample.
|
||||
|
||||
Parameters
|
||||
---------
|
||||
item : int
|
||||
The sample index.
|
||||
|
||||
Returns
|
||||
-------
|
||||
:class:`eg.Graph`
|
||||
graph structure, node features and node labels.
|
||||
|
||||
- ``ndata['feat']``: node features
|
||||
- ``ndata['label']``: node labels
|
||||
"""
|
||||
if self._transform is None:
|
||||
return self.graphs[item]
|
||||
else:
|
||||
return self._transform(self.graphs[item])
|
||||
|
||||
|
||||
class LegacyPPIDataset(PPIDataset):
|
||||
"""Legacy version of PPI Dataset"""
|
||||
|
||||
def __getitem__(self, item):
|
||||
"""Get the item^th sample.
|
||||
|
||||
Parameters
|
||||
---------
|
||||
idx : int
|
||||
The sample index.
|
||||
|
||||
Returns
|
||||
-------
|
||||
(eg.DGLGraph, Tensor, Tensor)
|
||||
The graph, features and its label.
|
||||
"""
|
||||
if self._transform is None:
|
||||
g = self.graphs[item]
|
||||
else:
|
||||
g = self._transform(self.graphs[item])
|
||||
return g, g.ndata["feat"], g.ndata["label"]
|
||||
@@ -0,0 +1,104 @@
|
||||
import os
|
||||
|
||||
import easygraph as eg
|
||||
import numpy as np
|
||||
import scipy.sparse as sp
|
||||
|
||||
from easygraph.classes.graph import Graph
|
||||
|
||||
from .graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from .utils import data_type_dict
|
||||
from .utils import download
|
||||
from .utils import extract_archive
|
||||
from .utils import tensor
|
||||
|
||||
|
||||
class RedditDataset(EasyGraphBuiltinDataset):
|
||||
r"""Reddit posts graph (Sept 2014) for community (subreddit) classification.
|
||||
|
||||
Statistics:
|
||||
- Nodes: ~232,965
|
||||
- Edges: ~114 million (approx.)
|
||||
- Features per node: 602
|
||||
- Classes: number of subreddit communities
|
||||
|
||||
Data are split by post-day: first 20 days train, then validation (30%), test (rest).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
self_loop : bool
|
||||
Add self-loop edges if True.
|
||||
raw_dir, force_reload, verbose, transform : same as EasyGraphBuiltinDataset
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
self_loop=False,
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=True,
|
||||
transform=None,
|
||||
):
|
||||
name = "reddit"
|
||||
url = "https://data.dgl.ai/dataset/reddit.zip"
|
||||
self.self_loop = self_loop
|
||||
super().__init__(
|
||||
name=name,
|
||||
url=url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def process(self):
|
||||
# Expect two files extracted: reddit_data.npz & reddit_graph.npz
|
||||
data = np.load(os.path.join(self.raw_path, "reddit_data.npz"))
|
||||
feat = data["feature"] # shape [N, 602]
|
||||
labels = data["label"] # shape [N]
|
||||
split = data["node_types"] # 1=train,2=val,3=test
|
||||
|
||||
# Load adjacency
|
||||
adj = sp.load_npz(os.path.join(self.raw_path, "reddit_graph.npz"))
|
||||
src, dst = adj.nonzero()
|
||||
if self.self_loop:
|
||||
self_loops = np.arange(adj.shape[0])
|
||||
src = np.concatenate([src, self_loops])
|
||||
dst = np.concatenate([dst, self_loops])
|
||||
edges = list(zip(src, dst))
|
||||
|
||||
# Build graph
|
||||
g = eg.Graph()
|
||||
g.add_edges_from(edges)
|
||||
|
||||
# Assign node features, labels, and masks
|
||||
for i in range(feat.shape[0]):
|
||||
g.add_node(
|
||||
i,
|
||||
feat=feat[i],
|
||||
label=int(labels[i]),
|
||||
train_mask=(split[i] == 1),
|
||||
val_mask=(split[i] == 2),
|
||||
test_mask=(split[i] == 3),
|
||||
)
|
||||
|
||||
self._g = g
|
||||
self._num_classes = int(np.max(labels) + 1)
|
||||
|
||||
if self.verbose:
|
||||
print("Loaded Reddit dataset:")
|
||||
print(f" NumNodes: {g.number_of_nodes()}")
|
||||
print(f" NumEdges: {g.number_of_edges()}")
|
||||
print(f" NumFeats: {feat.shape[1]}")
|
||||
print(f" NumClasses: {self._num_classes}")
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "RedditDataset only contains one graph"
|
||||
return self._g if self.transform is None else self.transform(self._g)
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
|
||||
@property
|
||||
def num_classes(self):
|
||||
return self._num_classes
|
||||
@@ -0,0 +1,107 @@
|
||||
"""RoadNet-CA Dataset
|
||||
|
||||
This dataset represents the road network of California.
|
||||
Nodes correspond to intersections, and edges represent roads connecting them.
|
||||
|
||||
The data is undirected and unweighted. No features or labels are provided.
|
||||
|
||||
Statistics:
|
||||
- Nodes: 1,965,206
|
||||
- Edges: 2,766,607
|
||||
- Features: None
|
||||
- Labels: None
|
||||
|
||||
Reference:
|
||||
J. Leskovec and A. Krevl, “SNAP Datasets: Stanford Large Network Dataset Collection,”
|
||||
https://snap.stanford.edu/data/roadNet-CA.html
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import easygraph as eg
|
||||
|
||||
from easygraph.classes.graph import Graph
|
||||
|
||||
from .graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from .utils import download
|
||||
|
||||
|
||||
class RoadNetCADataset(EasyGraphBuiltinDataset):
|
||||
r"""Road network of California (RoadNet-CA)
|
||||
|
||||
Nodes are road intersections and edges are roads connecting them.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_dir : str, optional
|
||||
Directory to store the raw downloaded files. Default: None
|
||||
force_reload : bool, optional
|
||||
Whether to re-download and process the dataset. Default: False
|
||||
verbose : bool, optional
|
||||
Whether to print detailed processing logs. Default: True
|
||||
transform : callable, optional
|
||||
Optional transform to apply on the graph.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from easygraph.datasets import RoadNetCADataset
|
||||
>>> dataset = RoadNetCADataset()
|
||||
>>> g = dataset[0]
|
||||
>>> print("Nodes:", g.number_of_nodes())
|
||||
>>> print("Edges:", g.number_of_edges())
|
||||
"""
|
||||
|
||||
def __init__(self, raw_dir=None, force_reload=False, verbose=True, transform=None):
|
||||
name = "roadNet-CA"
|
||||
url = "https://snap.stanford.edu/data/roadNet-CA.txt.gz"
|
||||
super(RoadNetCADataset, self).__init__(
|
||||
name=name,
|
||||
url=url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def download(self):
|
||||
r"""Download and decompress the .txt.gz file."""
|
||||
compressed_path = os.path.join(self.raw_dir, self.name + ".txt.gz")
|
||||
extracted_path = os.path.join(self.raw_path, self.name + ".txt")
|
||||
|
||||
download(self.url, path=compressed_path)
|
||||
|
||||
if not os.path.exists(self.raw_path):
|
||||
os.makedirs(self.raw_path)
|
||||
|
||||
with gzip.open(compressed_path, "rb") as f_in:
|
||||
with open(extracted_path, "wb") as f_out:
|
||||
shutil.copyfileobj(f_in, f_out)
|
||||
|
||||
def process(self):
|
||||
graph = eg.Graph() # Undirected road network
|
||||
edge_list_path = os.path.join(self.raw_path, self.name + ".txt")
|
||||
|
||||
with open(edge_list_path, "r") as f:
|
||||
for line in f:
|
||||
if line.startswith("#") or line.strip() == "":
|
||||
continue
|
||||
u, v = map(int, line.strip().split())
|
||||
graph.add_edge(u, v)
|
||||
|
||||
self._g = graph
|
||||
self._num_nodes = graph.number_of_nodes()
|
||||
self._num_edges = graph.number_of_edges()
|
||||
|
||||
if self.verbose:
|
||||
print("Finished loading RoadNet-CA dataset.")
|
||||
print(f" NumNodes: {self._num_nodes}")
|
||||
print(f" NumEdges: {self._num_edges}")
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "RoadNetCADataset only contains one graph"
|
||||
return self._g if self._transform is None else self._transform(self._g)
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
Binary file not shown.
@@ -0,0 +1,65 @@
|
||||
import gzip
|
||||
import os
|
||||
|
||||
import easygraph as eg
|
||||
|
||||
from easygraph.datasets.graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from easygraph.datasets.utils import download
|
||||
from easygraph.datasets.utils import extract_archive
|
||||
|
||||
|
||||
class TwitterEgoDataset(EasyGraphBuiltinDataset):
|
||||
r"""
|
||||
Twitter Ego Network Dataset
|
||||
|
||||
The Twitter dataset was collected from public sources and contains a large ego-network of Twitter users.
|
||||
The combined network includes 81K edges among 81K users.
|
||||
|
||||
Source: J. McAuley and J. Leskovec, Stanford SNAP, 2012
|
||||
URL: https://snap.stanford.edu/data/egonets-Twitter.html
|
||||
File used: https://snap.stanford.edu/data/twitter_combined.txt.gz
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super(TwitterEgoDataset, self).__init__(
|
||||
name="twitter_ego",
|
||||
url="https://snap.stanford.edu/data/twitter_combined.txt.gz",
|
||||
force_reload=False,
|
||||
)
|
||||
|
||||
def download(self):
|
||||
gz_path = os.path.join(self.raw_path, "twitter_combined.txt.gz")
|
||||
download(self.url, path=gz_path)
|
||||
extract_archive(gz_path, self.raw_path)
|
||||
|
||||
def process(self):
|
||||
import gzip
|
||||
|
||||
import easygraph as eg
|
||||
|
||||
gz_path = os.path.join(self.raw_path, "twitter_combined.txt.gz")
|
||||
txt_path = os.path.join(self.raw_path, "twitter_combined.txt")
|
||||
|
||||
if not os.path.exists(txt_path):
|
||||
with gzip.open(gz_path, "rt") as f_in, open(txt_path, "w") as f_out:
|
||||
f_out.writelines(f_in)
|
||||
|
||||
G = eg.Graph()
|
||||
edge_count = 0
|
||||
with open(txt_path, "r") as f:
|
||||
for line in f:
|
||||
u, v = map(int, line.strip().split())
|
||||
G.add_edge(u, v)
|
||||
edge_count += 1
|
||||
|
||||
self._graphs = [G]
|
||||
self._graph = G
|
||||
self._processed = True
|
||||
|
||||
def __getitem__(self, idx):
|
||||
if self._graph is not None:
|
||||
return self._graph
|
||||
elif self._graphs:
|
||||
return self._graphs[idx]
|
||||
else:
|
||||
return None
|
||||
@@ -0,0 +1,358 @@
|
||||
import errno
|
||||
import hashlib
|
||||
import numbers
|
||||
import os
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
import torch as th
|
||||
|
||||
|
||||
__all__ = [
|
||||
"download",
|
||||
"extract_archive",
|
||||
"get_download_dir",
|
||||
"makedirs",
|
||||
"generate_mask_tensor",
|
||||
]
|
||||
|
||||
import warnings
|
||||
|
||||
from easygraph.utils.download import _retry
|
||||
|
||||
|
||||
def _get_eg_url(file_url):
|
||||
"""Get EasyGraph online url for download."""
|
||||
eg_repo_url = "https://gitlab.com/easy-graph/"
|
||||
repo_url = eg_repo_url
|
||||
if repo_url[-1] != "/":
|
||||
repo_url = repo_url + "/"
|
||||
return repo_url + file_url
|
||||
|
||||
|
||||
def _get_dgl_url(file_url):
|
||||
"""Get DGL online url for download."""
|
||||
dgl_repo_url = "https://data.dgl.ai/"
|
||||
repo_url = os.environ.get("DGL_REPO", dgl_repo_url)
|
||||
if repo_url[-1] != "/":
|
||||
repo_url = repo_url + "/"
|
||||
return repo_url + file_url
|
||||
|
||||
|
||||
def _set_labels(G, labels):
|
||||
index = 0
|
||||
for node in G.nodes:
|
||||
G.add_node(node, label=labels[index])
|
||||
index += 1
|
||||
return G
|
||||
|
||||
|
||||
def _set_features(G, features):
|
||||
index = 0
|
||||
for node in G.nodes:
|
||||
G.add_node(node, feat=features[index])
|
||||
index += 1
|
||||
return G
|
||||
|
||||
|
||||
def nonzero_1d(input):
|
||||
x = th.nonzero(input, as_tuple=False).squeeze()
|
||||
return x if x.dim() == 1 else x.view(-1)
|
||||
|
||||
|
||||
def tensor(data, dtype=None):
|
||||
if isinstance(data, numbers.Number):
|
||||
data = [data]
|
||||
if isinstance(data, list) and len(data) > 0 and isinstance(data[0], th.Tensor):
|
||||
# prevent GPU->CPU->GPU copies
|
||||
if data[0].ndim == 0:
|
||||
# zero dimension scalar tensors
|
||||
return th.stack(data)
|
||||
if isinstance(data, th.Tensor):
|
||||
return th.as_tensor(data, dtype=dtype, device=data.device)
|
||||
else:
|
||||
return th.as_tensor(data, dtype=dtype)
|
||||
|
||||
|
||||
def data_type_dict():
|
||||
return {
|
||||
"float16": th.float16,
|
||||
"float32": th.float32,
|
||||
"float64": th.float64,
|
||||
"uint8": th.uint8,
|
||||
"int8": th.int8,
|
||||
"int16": th.int16,
|
||||
"int32": th.int32,
|
||||
"int64": th.int64,
|
||||
"bool": th.bool,
|
||||
}
|
||||
|
||||
|
||||
def download(
|
||||
url,
|
||||
path=None,
|
||||
overwrite=True,
|
||||
sha1_hash=None,
|
||||
retries=5,
|
||||
verify_ssl=True,
|
||||
log=True,
|
||||
):
|
||||
"""Download a given URL.
|
||||
|
||||
Codes borrowed from mxnet/gluon/utils.py
|
||||
|
||||
Parameters
|
||||
----------
|
||||
url : str
|
||||
URL to download.
|
||||
path : str, optional
|
||||
Destination path to store downloaded file. By default stores to the
|
||||
current directory with the same name as in url.
|
||||
overwrite : bool, optional
|
||||
Whether to overwrite the destination file if it already exists.
|
||||
By default always overwrites the downloaded file.
|
||||
sha1_hash : str, optional
|
||||
Expected sha1 hash in hexadecimal digits. Will ignore existing file when hash is specified
|
||||
but doesn't match.
|
||||
retries : integer, default 5
|
||||
The number of times to attempt downloading in case of failure or non 200 return codes.
|
||||
verify_ssl : bool, default True
|
||||
Verify SSL certificates.
|
||||
log : bool, default True
|
||||
Whether to print the progress for download
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The file path of the downloaded file.
|
||||
"""
|
||||
if path is None:
|
||||
fname = url.split("/")[-1]
|
||||
# Empty filenames are invalid
|
||||
assert fname, (
|
||||
"Can't construct file-name from this URL. "
|
||||
"Please set the `path` option manually."
|
||||
)
|
||||
else:
|
||||
path = os.path.expanduser(path)
|
||||
if os.path.isdir(path):
|
||||
fname = os.path.join(path, url.split("/")[-1])
|
||||
else:
|
||||
fname = path
|
||||
assert retries >= 0, "Number of retries should be at least 0"
|
||||
|
||||
if not verify_ssl:
|
||||
warnings.warn(
|
||||
"Unverified HTTPS request is being made (verify_ssl=False). "
|
||||
"Adding certificate verification is strongly advised."
|
||||
)
|
||||
|
||||
if (
|
||||
overwrite
|
||||
or not os.path.exists(fname)
|
||||
or (sha1_hash and not check_sha1(fname, sha1_hash))
|
||||
):
|
||||
dirname = os.path.dirname(os.path.abspath(os.path.expanduser(fname)))
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
while retries + 1 > 0:
|
||||
# Disable pyling too broad Exception
|
||||
# pylint: disable=W0703
|
||||
try:
|
||||
if log:
|
||||
print("Downloading %s from %s..." % (fname, url))
|
||||
r = requests.get(url, stream=True, verify=verify_ssl)
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError("Failed downloading url %s" % url)
|
||||
with open(fname, "wb") as f:
|
||||
for chunk in r.iter_content(chunk_size=1024):
|
||||
if chunk: # filter out keep-alive new chunks
|
||||
f.write(chunk)
|
||||
if sha1_hash and not check_sha1(fname, sha1_hash):
|
||||
raise UserWarning(
|
||||
"File {} is downloaded but the content hash does not match."
|
||||
" The repo may be outdated or download may be incomplete. "
|
||||
'If the "repo_url" is overridden, consider switching to '
|
||||
"the default repo.".format(fname)
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
retries -= 1
|
||||
if retries <= 0:
|
||||
raise e
|
||||
else:
|
||||
if log:
|
||||
print(
|
||||
"download failed, retrying, {} attempt{} left".format(
|
||||
retries, "s" if retries > 1 else ""
|
||||
)
|
||||
)
|
||||
|
||||
return fname
|
||||
|
||||
|
||||
def extract_archive(file, target_dir, overwrite=False):
|
||||
"""Extract archive file.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file : str
|
||||
Absolute path of the archive file.
|
||||
target_dir : str
|
||||
Target directory of the archive to be uncompressed.
|
||||
overwrite : bool, default True
|
||||
Whether to overwrite the contents inside the directory.
|
||||
By default always overwrites.
|
||||
"""
|
||||
if os.path.exists(target_dir) and not overwrite:
|
||||
return
|
||||
print("Extracting file to {}".format(target_dir))
|
||||
if file.endswith(".tar.gz") or file.endswith(".tar") or file.endswith(".tgz"):
|
||||
import tarfile
|
||||
|
||||
with tarfile.open(file, "r") as archive:
|
||||
archive.extractall(path=target_dir)
|
||||
elif file.endswith(".gz"):
|
||||
import gzip
|
||||
import shutil
|
||||
|
||||
with gzip.open(file, "rb") as f_in:
|
||||
target_file = os.path.join(target_dir, os.path.basename(file)[:-3])
|
||||
with open(target_file, "wb") as f_out:
|
||||
shutil.copyfileobj(f_in, f_out)
|
||||
elif file.endswith(".zip"):
|
||||
import zipfile
|
||||
|
||||
with zipfile.ZipFile(file, "r") as archive:
|
||||
archive.extractall(path=target_dir)
|
||||
else:
|
||||
raise Exception("Unrecognized file type: " + file)
|
||||
|
||||
|
||||
def get_download_dir():
|
||||
"""Get the absolute path to the download directory.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dirname : str
|
||||
Path to the download directory
|
||||
"""
|
||||
default_dir = os.path.join(os.path.expanduser("~"), ".EasyGraphData")
|
||||
dirname = os.environ.get("EG_DOWNLOAD_DIR", default_dir)
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
return dirname
|
||||
|
||||
|
||||
def makedirs(path):
|
||||
try:
|
||||
os.makedirs(os.path.expanduser(os.path.normpath(path)))
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST and os.path.isdir(path):
|
||||
raise e
|
||||
|
||||
|
||||
def check_sha1(filename, sha1_hash):
|
||||
"""Check whether the sha1 hash of the file content matches the expected hash.
|
||||
|
||||
Codes borrowed from mxnet/gluon/utils.py
|
||||
|
||||
Parameters
|
||||
----------
|
||||
filename : str
|
||||
Path to the file.
|
||||
sha1_hash : str
|
||||
Expected sha1 hash in hexadecimal digits.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
Whether the file content matches the expected hash.
|
||||
"""
|
||||
sha1 = hashlib.sha1()
|
||||
with open(filename, "rb") as f:
|
||||
while True:
|
||||
data = f.read(1048576)
|
||||
if not data:
|
||||
break
|
||||
sha1.update(data)
|
||||
|
||||
return sha1.hexdigest() == sha1_hash
|
||||
|
||||
|
||||
def generate_mask_tensor(mask):
|
||||
"""Generate mask tensor according to different backend
|
||||
For torch, it will create a bool tensor
|
||||
Parameters
|
||||
----------
|
||||
mask: numpy ndarray
|
||||
input mask tensor
|
||||
"""
|
||||
assert isinstance(
|
||||
mask, np.ndarray
|
||||
), "input for generate_mask_tensor should be an numpy ndarray"
|
||||
return tensor(mask, dtype=data_type_dict()["bool"])
|
||||
|
||||
|
||||
def deprecate_property(old, new):
|
||||
warnings.warn(
|
||||
"Property {} will be deprecated, please use {} instead.".format(old, new)
|
||||
)
|
||||
|
||||
|
||||
def check_file(file_path: Path, md5: str):
|
||||
r"""Check if a file is valid.
|
||||
|
||||
Args:
|
||||
``file_path`` (``Path``): The local path of the file.
|
||||
``md5`` (``str``): The md5 of the file.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: Not found the file.
|
||||
"""
|
||||
if not file_path.exists():
|
||||
raise FileNotFoundError(f"{file_path} does not exist.")
|
||||
else:
|
||||
with open(file_path, "rb") as f:
|
||||
data = f.read()
|
||||
cur_md5 = hashlib.md5(data).hexdigest()
|
||||
return cur_md5 == md5
|
||||
|
||||
|
||||
def download_file(url: str, file_path: Path):
|
||||
r"""Download a file from a url.
|
||||
|
||||
Args:
|
||||
``url`` (``str``): the url of the file
|
||||
``file_path`` (``str``): the path to the file
|
||||
"""
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
r = requests.get(url, stream=True, verify=True)
|
||||
if r.status_code != 200:
|
||||
raise requests.HTTPError(f"{url} is not accessible.")
|
||||
with open(file_path, "wb") as f:
|
||||
for chunk in r.iter_content(chunk_size=1024):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
|
||||
|
||||
@_retry(3)
|
||||
def download_and_check(url: str, file_path: Path, md5: str):
|
||||
r"""Download a file from a url and check its integrity.
|
||||
|
||||
Args:
|
||||
``url`` (``str``): The url of the file.
|
||||
``file_path`` (``Path``): The path to the file.
|
||||
``md5`` (``str``): The md5 of the file.
|
||||
"""
|
||||
if not file_path.exists():
|
||||
download_file(url, file_path)
|
||||
if not check_file(file_path, md5):
|
||||
file_path.unlink()
|
||||
raise ValueError(
|
||||
f"{file_path} is corrupted. We will delete it, and try to download it"
|
||||
" again."
|
||||
)
|
||||
return True
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Web-Google Dataset
|
||||
|
||||
This dataset is a web graph based on Google's web pages and their hyperlink
|
||||
structure, as crawled by the Stanford WebBase project in 2002.
|
||||
|
||||
Each node represents a web page, and a directed edge from u to v indicates
|
||||
a hyperlink from page u to page v.
|
||||
|
||||
Statistics:
|
||||
- Nodes: 875713
|
||||
- Edges: 5105039
|
||||
- Features: None
|
||||
- Labels: None
|
||||
|
||||
Reference:
|
||||
J. Leskovec, A. Rajaraman, J. Ullman, “Mining of Massive Datasets.”
|
||||
Dataset from SNAP: https://snap.stanford.edu/data/web-Google.html
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import easygraph as eg
|
||||
|
||||
from easygraph.classes.graph import Graph
|
||||
|
||||
from .graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from .utils import download
|
||||
from .utils import extract_archive
|
||||
|
||||
|
||||
class WebGoogleDataset(EasyGraphBuiltinDataset):
|
||||
r"""Web-Google hyperlink network dataset.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
raw_dir : str, optional
|
||||
Directory to store the raw downloaded files. Default: None
|
||||
force_reload : bool, optional
|
||||
Whether to re-download and process the dataset. Default: False
|
||||
verbose : bool, optional
|
||||
Whether to print detailed processing logs. Default: True
|
||||
transform : callable, optional
|
||||
Optional transform to apply on the graph.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from easygraph.datasets import WebGoogleDataset
|
||||
>>> dataset = WebGoogleDataset()
|
||||
>>> g = dataset[0]
|
||||
>>> print("Nodes:", g.number_of_nodes())
|
||||
>>> print("Edges:", g.number_of_edges())
|
||||
"""
|
||||
|
||||
def __init__(self, raw_dir=None, force_reload=False, verbose=True, transform=None):
|
||||
name = "web-Google"
|
||||
url = "https://snap.stanford.edu/data/web-Google.txt.gz"
|
||||
super(WebGoogleDataset, self).__init__(
|
||||
name=name,
|
||||
url=url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def download(self):
|
||||
r"""Download and extract .gz edge list."""
|
||||
if self.url is not None:
|
||||
file_path = os.path.join(self.raw_dir, self.name + ".txt.gz")
|
||||
download(self.url, path=file_path)
|
||||
extract_archive(file_path, self.raw_path)
|
||||
|
||||
def process(self):
|
||||
graph = eg.DiGraph() # Web-Google is directed
|
||||
edge_list_path = os.path.join(self.raw_path, self.name + ".txt")
|
||||
|
||||
with open(edge_list_path, "r") as f:
|
||||
for line in f:
|
||||
if line.startswith("#") or line.strip() == "":
|
||||
continue
|
||||
u, v = map(int, line.strip().split())
|
||||
graph.add_edge(u, v)
|
||||
|
||||
self._g = graph
|
||||
self._num_nodes = graph.number_of_nodes()
|
||||
self._num_edges = graph.number_of_edges()
|
||||
|
||||
if self.verbose:
|
||||
print("Finished loading Web-Google dataset.")
|
||||
print(f" NumNodes: {self._num_nodes}")
|
||||
print(f" NumEdges: {self._num_edges}")
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "WebGoogleDataset only contains one graph"
|
||||
return self._g if self._transform is None else self._transform(self._g)
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
|
||||
def download(self):
|
||||
r"""Download and decompress the .txt.gz file."""
|
||||
if self.url is not None:
|
||||
compressed_path = os.path.join(self.raw_dir, self.name + ".txt.gz")
|
||||
extracted_path = os.path.join(self.raw_path, self.name + ".txt")
|
||||
|
||||
# Download .gz file
|
||||
download(self.url, path=compressed_path)
|
||||
|
||||
# Ensure output directory exists
|
||||
if not os.path.exists(self.raw_path):
|
||||
os.makedirs(self.raw_path)
|
||||
|
||||
# Decompress manually
|
||||
with gzip.open(compressed_path, "rb") as f_in:
|
||||
with open(extracted_path, "wb") as f_out:
|
||||
shutil.copyfileobj(f_in, f_out)
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Wikipedia Top Categories Dataset (wiki-topcats)
|
||||
|
||||
This dataset is a directed graph of Wikipedia articles restricted to
|
||||
top-level categories (at least 100 articles), capturing the largest
|
||||
strongly connected component.
|
||||
|
||||
Statistics:
|
||||
- Nodes: 1,791,489
|
||||
- Edges: 28,511,807
|
||||
- Categories: 17,364
|
||||
- Overlapping labels per node
|
||||
|
||||
Source:
|
||||
H. Yin, A. Benson, J. Leskovec, D. Gleich.
|
||||
"Local Higher-order Graph Clustering", KDD 2017
|
||||
Data: https://snap.stanford.edu/data/wiki-topcats.html
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import os
|
||||
|
||||
import easygraph as eg
|
||||
|
||||
from easygraph.datasets.graph_dataset_base import EasyGraphBuiltinDataset
|
||||
from easygraph.datasets.utils import download
|
||||
from easygraph.datasets.utils import extract_archive
|
||||
|
||||
|
||||
class WikiTopCatsDataset(EasyGraphBuiltinDataset):
|
||||
"""Wikipedia Top Categories Snapshot from 2011 (SNAP)"""
|
||||
|
||||
def __init__(self, raw_dir=None, force_reload=False, verbose=True, transform=None):
|
||||
super(WikiTopCatsDataset, self).__init__(
|
||||
name="wiki_topcats",
|
||||
url="https://snap.stanford.edu/data/wiki-topcats.txt.gz",
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
)
|
||||
|
||||
def download(self):
|
||||
# Download the main graph file
|
||||
gz_path = os.path.join(self.raw_dir, "wiki-topcats.txt.gz")
|
||||
download(self.url, path=gz_path)
|
||||
|
||||
# Also download category info and page names
|
||||
cat_url = "https://snap.stanford.edu/data/wiki-topcats-categories.txt.gz"
|
||||
names_url = "https://snap.stanford.edu/data/wiki-topcats-page-names.txt.gz"
|
||||
download(
|
||||
cat_url, path=os.path.join(self.raw_dir, "wiki-topcats-categories.txt.gz")
|
||||
)
|
||||
download(
|
||||
names_url, path=os.path.join(self.raw_dir, "wiki-topcats-page-names.txt.gz")
|
||||
)
|
||||
|
||||
def process(self):
|
||||
raw = self.raw_dir
|
||||
|
||||
# Decompress and read edges
|
||||
edge_gz = os.path.join(raw, "wiki-topcats.txt.gz")
|
||||
edge_txt = os.path.join(raw, "wiki-topcats.txt")
|
||||
if not os.path.exists(edge_txt):
|
||||
with gzip.open(edge_gz, "rt") as fin, open(edge_txt, "w") as fout:
|
||||
fout.writelines(fin)
|
||||
G = eg.DiGraph()
|
||||
edge_count = 0
|
||||
with open(edge_txt, "r") as f:
|
||||
for line in f:
|
||||
u, v = map(int, line.strip().split())
|
||||
G.add_edge(u, v)
|
||||
edge_count += 1
|
||||
if self.verbose:
|
||||
print(f"Loaded graph: {G.number_of_nodes()} nodes, {edge_count} edges")
|
||||
|
||||
# Compress node names
|
||||
names_gz = os.path.join(raw, "wiki-topcats-page-names.txt.gz")
|
||||
names = {}
|
||||
with gzip.open(names_gz, "rt") as f:
|
||||
for idx, line in enumerate(f):
|
||||
names[idx] = line.strip()
|
||||
|
||||
# Load categories
|
||||
cats_gz = os.path.join(raw, "wiki-topcats-categories.txt.gz")
|
||||
labels = {} # mapping: node -> list of category strings
|
||||
with gzip.open(cats_gz, "rt") as f:
|
||||
for idx, line in enumerate(f):
|
||||
categories = line.strip().split(";")
|
||||
categories = [cat.strip() for cat in categories if cat.strip()]
|
||||
labels[idx] = categories
|
||||
|
||||
# Attach node features: empty, and node labels
|
||||
for n in G.nodes:
|
||||
G.add_node(n, name=names.get(n, ""), label=labels.get(n, []))
|
||||
|
||||
self._graph = G
|
||||
self._graphs = [G]
|
||||
self._processed = True
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0
|
||||
return self._graph
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
Reference in New Issue
Block a user