chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,255 @@
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import scipy.sparse as sp
|
||||
|
||||
from .. import backend as F
|
||||
from ..convert import graph
|
||||
from .dgl_dataset import DGLBuiltinDataset
|
||||
from .utils import _get_dgl_url, load_graphs, load_info, save_graphs, save_info
|
||||
|
||||
|
||||
class FakeNewsDataset(DGLBuiltinDataset):
|
||||
r"""Fake News Graph Classification dataset.
|
||||
|
||||
The dataset is composed of two sets of tree-structured fake/real
|
||||
news propagation graphs extracted from Twitter. Different from
|
||||
most of the benchmark datasets for the graph classification task,
|
||||
the graphs in this dataset are directed tree-structured graphs where
|
||||
the root node represents the news, the leaf nodes are Twitter users
|
||||
who retweeted the root news. Besides, the node features are encoded
|
||||
user historical tweets using different pretrained language models:
|
||||
|
||||
- bert: the 768-dimensional node feature composed of Twitter user historical tweets encoded by the bert-as-service
|
||||
- content: the 310-dimensional node feature composed of a 300-dimensional “spacy” vector plus a 10-dimensional “profile” vector
|
||||
- profile: the 10-dimensional node feature composed of ten Twitter user profile attributes.
|
||||
- spacy: the 300-dimensional node feature composed of Twitter user historical tweets encoded by the spaCy word2vec encoder.
|
||||
|
||||
Reference: <https://github.com/safe-graph/GNN-FakeNews>
|
||||
|
||||
Note: this dataset is for academic use only, and commercial use is prohibited.
|
||||
|
||||
Statistics:
|
||||
|
||||
Politifact:
|
||||
|
||||
- Graphs: 314
|
||||
- Nodes: 41,054
|
||||
- Edges: 40,740
|
||||
- Classes:
|
||||
|
||||
- Fake: 157
|
||||
- Real: 157
|
||||
|
||||
- Node feature size:
|
||||
|
||||
- bert: 768
|
||||
- content: 310
|
||||
- profile: 10
|
||||
- spacy: 300
|
||||
|
||||
Gossipcop:
|
||||
|
||||
- Graphs: 5,464
|
||||
- Nodes: 314,262
|
||||
- Edges: 308,798
|
||||
- Classes:
|
||||
|
||||
- Fake: 2,732
|
||||
- Real: 2,732
|
||||
|
||||
- Node feature size:
|
||||
|
||||
- bert: 768
|
||||
- content: 310
|
||||
- profile: 10
|
||||
- spacy: 300
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
Name of the dataset (gossipcop, or politifact)
|
||||
feature_name : str
|
||||
Name of the feature (bert, content, profile, or spacy)
|
||||
raw_dir : str
|
||||
Specifying the directory that will store the
|
||||
downloaded data or the directory that
|
||||
already stores the input data.
|
||||
Default: ~/.dgl/
|
||||
transform : callable, optional
|
||||
A transform that takes in a :class:`~dgl.DGLGraph` object and returns
|
||||
a transformed version. The :class:`~dgl.DGLGraph` object will be
|
||||
transformed before every access.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
name : str
|
||||
Name of the dataset (gossipcop, or politifact)
|
||||
num_classes : int
|
||||
Number of label classes
|
||||
num_graphs : int
|
||||
Number of graphs
|
||||
graphs : list
|
||||
A list of DGLGraph objects
|
||||
labels : Tensor
|
||||
Graph labels
|
||||
feature_name : str
|
||||
Name of the feature (bert, content, profile, or spacy)
|
||||
feature : Tensor
|
||||
Node features
|
||||
train_mask : Tensor
|
||||
Mask of training set
|
||||
val_mask : Tensor
|
||||
Mask of validation set
|
||||
test_mask : Tensor
|
||||
Mask of testing set
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> dataset = FakeNewsDataset('gossipcop', 'bert')
|
||||
>>> graph, label = dataset[0]
|
||||
>>> num_classes = dataset.num_classes
|
||||
>>> feat = dataset.feature
|
||||
>>> labels = dataset.labels
|
||||
"""
|
||||
file_urls = {
|
||||
"gossipcop": "dataset/FakeNewsGOS.zip",
|
||||
"politifact": "dataset/FakeNewsPOL.zip",
|
||||
}
|
||||
|
||||
def __init__(self, name, feature_name, raw_dir=None, transform=None):
|
||||
assert name in [
|
||||
"gossipcop",
|
||||
"politifact",
|
||||
], "Only supports 'gossipcop' or 'politifact'."
|
||||
url = _get_dgl_url(self.file_urls[name])
|
||||
|
||||
assert feature_name in [
|
||||
"bert",
|
||||
"content",
|
||||
"profile",
|
||||
"spacy",
|
||||
], "Only supports 'bert', 'content', 'profile', or 'spacy'"
|
||||
self.feature_name = feature_name
|
||||
super(FakeNewsDataset, self).__init__(
|
||||
name=name, url=url, raw_dir=raw_dir, transform=transform
|
||||
)
|
||||
|
||||
def process(self):
|
||||
"""process raw data to graph, labels and masks"""
|
||||
self.labels = F.tensor(
|
||||
np.load(os.path.join(self.raw_path, "graph_labels.npy"))
|
||||
)
|
||||
num_graphs = self.labels.shape[0]
|
||||
|
||||
node_graph_id = np.load(
|
||||
os.path.join(self.raw_path, "node_graph_id.npy")
|
||||
)
|
||||
edges = np.genfromtxt(
|
||||
os.path.join(self.raw_path, "A.txt"), delimiter=",", dtype=int
|
||||
)
|
||||
src = edges[:, 0]
|
||||
dst = edges[:, 1]
|
||||
g = graph((src, dst))
|
||||
|
||||
node_idx_list = []
|
||||
for idx in range(np.max(node_graph_id) + 1):
|
||||
node_idx = np.where(node_graph_id == idx)
|
||||
node_idx_list.append(node_idx[0])
|
||||
|
||||
self.graphs = [g.subgraph(node_idx) for node_idx in node_idx_list]
|
||||
|
||||
train_idx = np.load(os.path.join(self.raw_path, "train_idx.npy"))
|
||||
val_idx = np.load(os.path.join(self.raw_path, "val_idx.npy"))
|
||||
test_idx = np.load(os.path.join(self.raw_path, "test_idx.npy"))
|
||||
train_mask = np.zeros(num_graphs, dtype=np.bool_)
|
||||
val_mask = np.zeros(num_graphs, dtype=np.bool_)
|
||||
test_mask = np.zeros(num_graphs, dtype=np.bool_)
|
||||
train_mask[train_idx] = True
|
||||
val_mask[val_idx] = True
|
||||
test_mask[test_idx] = True
|
||||
self.train_mask = F.tensor(train_mask)
|
||||
self.val_mask = F.tensor(val_mask)
|
||||
self.test_mask = F.tensor(test_mask)
|
||||
|
||||
feature_file = "new_" + self.feature_name + "_feature.npz"
|
||||
self.feature = F.tensor(
|
||||
sp.load_npz(os.path.join(self.raw_path, feature_file)).todense()
|
||||
)
|
||||
|
||||
def save(self):
|
||||
"""save the graph list and the labels"""
|
||||
save_graphs(str(self.graph_path), self.graphs)
|
||||
save_info(
|
||||
self.info_path,
|
||||
{
|
||||
"label": self.labels,
|
||||
"feature": self.feature,
|
||||
"train_mask": self.train_mask,
|
||||
"val_mask": self.val_mask,
|
||||
"test_mask": self.test_mask,
|
||||
},
|
||||
)
|
||||
|
||||
@property
|
||||
def graph_path(self):
|
||||
return os.path.join(self.save_path, self.name + "_dgl_graph.bin")
|
||||
|
||||
@property
|
||||
def info_path(self):
|
||||
return os.path.join(self.save_path, self.name + "_dgl_graph.pkl")
|
||||
|
||||
def has_cache(self):
|
||||
"""check whether there are processed data in `self.save_path`"""
|
||||
return os.path.exists(self.graph_path) and os.path.exists(
|
||||
self.info_path
|
||||
)
|
||||
|
||||
def load(self):
|
||||
"""load processed data from directory `self.save_path`"""
|
||||
graphs, _ = load_graphs(str(self.graph_path))
|
||||
info = load_info(str(self.info_path))
|
||||
self.graphs = graphs
|
||||
self.labels = info["label"]
|
||||
self.feature = info["feature"]
|
||||
|
||||
self.train_mask = info["train_mask"]
|
||||
self.val_mask = info["val_mask"]
|
||||
self.test_mask = info["test_mask"]
|
||||
|
||||
@property
|
||||
def num_classes(self):
|
||||
"""Number of classes for each graph, i.e. number of prediction tasks."""
|
||||
return 2
|
||||
|
||||
@property
|
||||
def num_graphs(self):
|
||||
"""Number of graphs."""
|
||||
return self.labels.shape[0]
|
||||
|
||||
def __getitem__(self, i):
|
||||
r"""Get graph and label by index
|
||||
|
||||
Parameters
|
||||
----------
|
||||
i : int
|
||||
Item index
|
||||
|
||||
Returns
|
||||
-------
|
||||
(:class:`dgl.DGLGraph`, Tensor)
|
||||
"""
|
||||
if self._transform is None:
|
||||
g = self.graphs[i]
|
||||
else:
|
||||
g = self._transform(self.graphs[i])
|
||||
return g, self.labels[i]
|
||||
|
||||
def __len__(self):
|
||||
r"""Number of graphs in the dataset.
|
||||
|
||||
Return
|
||||
-------
|
||||
int
|
||||
"""
|
||||
return len(self.graphs)
|
||||
Reference in New Issue
Block a user