chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,4 @@
|
||||
from .email_enron import *
|
||||
from .email_eu import *
|
||||
from .hospital_lyon import *
|
||||
from .load_dataset import *
|
||||
@@ -0,0 +1,86 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
from easygraph.convert import dict_to_hypergraph
|
||||
from easygraph.datasets.dynamic.load_dataset import request_json_from_url
|
||||
from easygraph.datasets.graph_dataset_base import EasyGraphDataset
|
||||
from easygraph.datasets.utils import _get_eg_url
|
||||
from easygraph.datasets.utils import tensor
|
||||
|
||||
|
||||
class Email_Enron(EasyGraphDataset):
|
||||
_urls = {
|
||||
"email-enron": (
|
||||
"easygraph-data-email-enron/-/raw/main/email-enron.json?inline=false"
|
||||
),
|
||||
"email-eu": "easygraph-data-email-eu/-/raw/main/email-eu.json?inline=false",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=True,
|
||||
transform=None,
|
||||
save_dir="./",
|
||||
):
|
||||
name = "email-enron"
|
||||
self.url = _get_eg_url(self._urls[name])
|
||||
super(Email_Enron, self).__init__(
|
||||
name=name,
|
||||
url=self.url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
save_dir=save_dir,
|
||||
)
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return self._url
|
||||
|
||||
@property
|
||||
def save_name(self):
|
||||
return self.name
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "This dataset has only one graph"
|
||||
if self._transform is None:
|
||||
return self._g
|
||||
else:
|
||||
return self._transform(self._g)
|
||||
|
||||
def load(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".json")
|
||||
with open(graph_path, "r") as f:
|
||||
self.load_data = json.load(f)
|
||||
|
||||
def has_cache(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".json")
|
||||
if os.path.exists(graph_path):
|
||||
return True
|
||||
return False
|
||||
|
||||
def download(self):
|
||||
if self.has_cache():
|
||||
self.load()
|
||||
else:
|
||||
root = self.raw_dir
|
||||
data = request_json_from_url(self.url)
|
||||
with open(os.path.join(root, self.save_name + ".json"), "w") as f:
|
||||
json.dump(data, f)
|
||||
self.load_data = data
|
||||
|
||||
def process(self):
|
||||
"""Loads input data from data directory and transfer to target graph for better analysis"""
|
||||
|
||||
self._g, edge_feature_list = dict_to_hypergraph(self.load_data, is_dynamic=True)
|
||||
|
||||
self._g.ndata["hyperedge_feature"] = tensor(
|
||||
range(1, len(edge_feature_list) + 1)
|
||||
)
|
||||
|
||||
@url.setter
|
||||
def url(self, value):
|
||||
self._url = value
|
||||
@@ -0,0 +1,81 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
from easygraph.convert import dict_to_hypergraph
|
||||
from easygraph.datasets.dynamic.load_dataset import request_json_from_url
|
||||
from easygraph.datasets.graph_dataset_base import EasyGraphDataset
|
||||
from easygraph.datasets.utils import _get_eg_url
|
||||
from easygraph.datasets.utils import tensor
|
||||
|
||||
|
||||
class Email_Eu(EasyGraphDataset):
|
||||
_urls = {
|
||||
"email-eu": "easygraph-data-email-eu/-/raw/main/email-eu.json?inline=false",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=True,
|
||||
transform=None,
|
||||
save_dir="./",
|
||||
):
|
||||
name = "email-eu"
|
||||
self.url = _get_eg_url(self._urls[name])
|
||||
super(Email_Eu, self).__init__(
|
||||
name=name,
|
||||
url=self.url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
save_dir=save_dir,
|
||||
)
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return self._url
|
||||
|
||||
@property
|
||||
def save_name(self):
|
||||
return self.name
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "This dataset has only one graph"
|
||||
if self._transform is None:
|
||||
return self._g
|
||||
else:
|
||||
return self._transform(self._g)
|
||||
|
||||
def load(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".json")
|
||||
with open(graph_path, "r") as f:
|
||||
self.load_data = json.load(f)
|
||||
|
||||
def has_cache(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".json")
|
||||
if os.path.exists(graph_path):
|
||||
return True
|
||||
return False
|
||||
|
||||
def download(self):
|
||||
if self.has_cache():
|
||||
self.load()
|
||||
else:
|
||||
root = self.raw_dir
|
||||
data = request_json_from_url(self.url)
|
||||
with open(os.path.join(root, self.save_name + ".json"), "w") as f:
|
||||
json.dump(data, f)
|
||||
self.load_data = data
|
||||
|
||||
def process(self):
|
||||
"""Loads input data from data directory and transfer to target graph for better analysis"""
|
||||
self._g, edge_feature_list = dict_to_hypergraph(self.load_data, is_dynamic=True)
|
||||
self._g.ndata["hyperedge_feature"] = tensor(
|
||||
range(1, len(edge_feature_list) + 1)
|
||||
)
|
||||
|
||||
@url.setter
|
||||
def url(self, value):
|
||||
self._url = value
|
||||
@@ -0,0 +1,133 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
from easygraph.classes.hypergraph import Hypergraph
|
||||
from easygraph.datasets.dynamic.load_dataset import request_json_from_url
|
||||
from easygraph.datasets.graph_dataset_base import EasyGraphDataset
|
||||
from easygraph.datasets.utils import _get_eg_url
|
||||
from easygraph.datasets.utils import tensor
|
||||
|
||||
|
||||
class Hospital_Lyon(EasyGraphDataset):
|
||||
_urls = {
|
||||
"hospital_lyon": (
|
||||
"easygraph-data-hospital-lyon/-/raw/main/hospital-lyon.json?ref_type=heads&inline=false"
|
||||
),
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
raw_dir=None,
|
||||
force_reload=False,
|
||||
verbose=True,
|
||||
transform=None,
|
||||
save_dir="./",
|
||||
):
|
||||
name = "hospital_lyon"
|
||||
self.url = _get_eg_url(self._urls[name])
|
||||
super(Hospital_Lyon, self).__init__(
|
||||
name=name,
|
||||
url=self.url,
|
||||
raw_dir=raw_dir,
|
||||
force_reload=force_reload,
|
||||
verbose=verbose,
|
||||
transform=transform,
|
||||
save_dir=save_dir,
|
||||
)
|
||||
|
||||
def preprocess(self, data, max_order=None, is_dynamic=True):
|
||||
# The index of the nodes in this dataset are not continuous and therefore require special processing
|
||||
timestamp_lst = list()
|
||||
node_data = data["node-data"]
|
||||
node_num = len(node_data)
|
||||
G = Hypergraph(num_v=node_num)
|
||||
id = 0
|
||||
name_dict = {}
|
||||
for k, v in data["node-data"].items():
|
||||
name_dict[k] = id
|
||||
v["name"] = k
|
||||
G.v_property[id] = v
|
||||
id = id + 1
|
||||
e_property_dict = data["edge-data"]
|
||||
rows = []
|
||||
cols = []
|
||||
edge_flag_dict = {}
|
||||
edge_id = 0
|
||||
for id, edge in data["edge-dict"].items():
|
||||
if max_order and len(edge) > max_order + 1:
|
||||
continue
|
||||
|
||||
try:
|
||||
id = int(id)
|
||||
except ValueError as e:
|
||||
raise TypeError(
|
||||
f"Failed to convert the edge with ID {id} to type int."
|
||||
) from e
|
||||
|
||||
try:
|
||||
edge = [name_dict[n] for n in edge]
|
||||
rows.extend(edge)
|
||||
cols.extend(len(edge) * [edge_id])
|
||||
edge_id += 1
|
||||
except ValueError as e:
|
||||
raise TypeError(f"Failed to convert nodes to type int.") from e
|
||||
if is_dynamic:
|
||||
G.add_hyperedges(
|
||||
e_list=edge,
|
||||
e_property=e_property_dict[str(id)],
|
||||
group_name=e_property_dict[str(id)]["timestamp"],
|
||||
)
|
||||
timestamp_lst.append(e_property_dict[str(id)]["timestamp"])
|
||||
else:
|
||||
G.add_hyperedges(e_list=edge, e_property=e_property_dict[str(id)])
|
||||
G._rows = rows
|
||||
G._cols = cols
|
||||
return G, timestamp_lst
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return self._url
|
||||
|
||||
@property
|
||||
def save_name(self):
|
||||
return self.name
|
||||
|
||||
def __getitem__(self, idx):
|
||||
assert idx == 0, "This dataset has only one graph"
|
||||
if self._transform is None:
|
||||
return self._g
|
||||
else:
|
||||
return self._transform(self._g)
|
||||
|
||||
def load(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".json")
|
||||
with open(graph_path, "r") as f:
|
||||
self.load_data = json.load(f)
|
||||
|
||||
def has_cache(self):
|
||||
graph_path = os.path.join(self.save_path, self.save_name + ".json")
|
||||
if os.path.exists(graph_path):
|
||||
return True
|
||||
return False
|
||||
|
||||
def download(self):
|
||||
if self.has_cache():
|
||||
self.load()
|
||||
else:
|
||||
root = self.raw_dir
|
||||
data = request_json_from_url(self.url)
|
||||
with open(os.path.join(root, self.save_name + ".json"), "w") as f:
|
||||
json.dump(data, f)
|
||||
self.load_data = data
|
||||
|
||||
def process(self):
|
||||
"""Loads input data from data directory and transfer to target graph for better analysis"""
|
||||
|
||||
self._g, edge_feature_list = self.preprocess(self.load_data, is_dynamic=True)
|
||||
self._g.ndata["hyperedge_feature"] = tensor(
|
||||
range(1, len(edge_feature_list) + 1)
|
||||
)
|
||||
|
||||
@url.setter
|
||||
def url(self, value):
|
||||
self._url = value
|
||||
@@ -0,0 +1,94 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
from warnings import warn
|
||||
|
||||
import requests
|
||||
|
||||
from easygraph.convert import dict_to_hypergraph
|
||||
from easygraph.utils.exception import EasyGraphError
|
||||
|
||||
|
||||
__all__ = [
|
||||
"load_dynamic_hypergraph_dataset",
|
||||
]
|
||||
|
||||
dataset_index_url = "https://gitlab.com/easy-graph/easygraph-data/-/raw/main/dataset_index.json?inline=false"
|
||||
|
||||
|
||||
def request_json_from_url(url):
|
||||
try:
|
||||
r = requests.get(url)
|
||||
except requests.ConnectionError:
|
||||
raise EasyGraphError("Connection Error!")
|
||||
|
||||
if r.ok:
|
||||
return r.json()
|
||||
else:
|
||||
raise EasyGraphError(f"Error: HTTP response {r.status_code}")
|
||||
|
||||
|
||||
def _request_from_eg_data(dataset=None, cache=True):
|
||||
"""Request a dataset from eg-data.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
dataset : str, optional
|
||||
Dataset name. Valid options are the top-level tags of the
|
||||
index.json file in the xgi-data repository. If None, prints
|
||||
the list of available datasets.
|
||||
cache : bool, optional
|
||||
Whether or not to cache the output
|
||||
|
||||
Returns
|
||||
-------
|
||||
Data
|
||||
The requested data loaded from a json file.
|
||||
|
||||
Raises
|
||||
------
|
||||
EasyGraphError
|
||||
If the HTTP request is not successful or the dataset does not exist.
|
||||
|
||||
|
||||
"""
|
||||
|
||||
index_data = request_json_from_url(dataset_index_url)
|
||||
|
||||
key = dataset.lower()
|
||||
if key not in index_data:
|
||||
print("Valid dataset names:")
|
||||
print(*index_data, sep="\n")
|
||||
raise EasyGraphError("Must choose a valid dataset name!")
|
||||
|
||||
return request_json_from_url(index_data[key]["url"])
|
||||
|
||||
|
||||
def load_dynamic_hypergraph_dataset(
|
||||
dataset=None,
|
||||
local_read=False,
|
||||
path="",
|
||||
max_order=None,
|
||||
):
|
||||
index_datasets = request_json_from_url(dataset_index_url)
|
||||
if dataset is None:
|
||||
print("Please refer to available list")
|
||||
|
||||
print(*index_datasets, sep="\n")
|
||||
return
|
||||
|
||||
if local_read:
|
||||
cfp = os.path.join(path, dataset + ".json")
|
||||
if os.path.exists(cfp):
|
||||
data = json.load(open(cfp, "r"))
|
||||
return dict_to_hypergraph(data, max_order=max_order)
|
||||
else:
|
||||
warn(
|
||||
f"No local copy was found at {cfp}. The data is requested "
|
||||
"from the xgi-data repository instead. To download a local "
|
||||
"copy, use `download_xgi_data`."
|
||||
)
|
||||
data = _request_from_eg_data(dataset)
|
||||
return dict_to_hypergraph(
|
||||
data, max_order=max_order, is_dynamic=index_datasets[dataset]["is_dynamic"]
|
||||
)
|
||||
Reference in New Issue
Block a user