wehub-resource-sync
222 lines
8.3 KiB
Python
222 lines
8.3 KiB
Python
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Copyright (c) 2017 Kamran Kowsari
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this dataset and associated documentation files (the "Dataset"), to deal
|
|
# in the dataset without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Dataset, and to permit persons to whom the dataset is
|
|
# furnished to do so, subject to the following conditions:
|
|
|
|
import collections
|
|
import os
|
|
|
|
from paddle.dataset.common import md5file
|
|
from paddle.utils.download import get_path_from_url
|
|
|
|
from paddlenlp.datasets import DatasetBuilder
|
|
from paddlenlp.utils.env import DATA_HOME
|
|
|
|
__all__ = ["WOS"]
|
|
|
|
|
|
class WOS(DatasetBuilder):
|
|
"""
|
|
Web of Science(WOS) dataset contains abstracts of published papers from Web of Science.
|
|
More information please refer to 'https://data.mendeley.com/datasets/9rw3vkcfy4/2'.
|
|
"""
|
|
|
|
lazy = False
|
|
URL = "https://bj.bcebos.com/paddlenlp/datasets/wos.tar.gz"
|
|
MD5 = "15c8631ed6a474f471f480c31a6bbcda"
|
|
META_INFO = collections.namedtuple("META_INFO", ("file", "md5"))
|
|
SPLITS = {
|
|
"train": META_INFO(os.path.join("wos", "train.tsv"), "e0153a1ef502235edf2bb138afcfef99"),
|
|
"dev": META_INFO(os.path.join("wos", "dev.tsv"), "fcfc283349b353c3e1123fdd20429de9 "),
|
|
"test": META_INFO(os.path.join("wos", "test.tsv"), "6fe2068aada7f17220d521dd11c73aee"),
|
|
}
|
|
|
|
def _get_data(self, mode, **kwargs):
|
|
"""Check and download Dataset"""
|
|
default_root = os.path.join(DATA_HOME, self.__class__.__name__)
|
|
filename, data_hash = self.SPLITS[mode]
|
|
fullname = os.path.join(default_root, filename)
|
|
if not os.path.exists(fullname) or (data_hash and not md5file(fullname) == data_hash):
|
|
|
|
get_path_from_url(self.URL, default_root, self.MD5)
|
|
|
|
return fullname
|
|
|
|
def _read(self, filename, *args):
|
|
|
|
with open(filename, "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line_stripped = line.split("\t")
|
|
|
|
example = {"sentence": line_stripped[0].strip()}
|
|
for i in range(len(line_stripped) - 1):
|
|
example["level {}".format(i + 1)] = line_stripped[i + 1].strip().split(",")
|
|
|
|
yield example
|
|
|
|
def get_labels(self):
|
|
"""
|
|
Return labels of the WOS.
|
|
"""
|
|
return [
|
|
"CS",
|
|
"ECE",
|
|
"Psychology",
|
|
"MAE",
|
|
"Civil",
|
|
"Medical",
|
|
"biochemistry",
|
|
"CS##Computer vision",
|
|
"CS##Machine learning",
|
|
"CS##network security",
|
|
"CS##Cryptography",
|
|
"CS##Operating systems",
|
|
"CS##Computer graphics",
|
|
"CS##Image processing",
|
|
"CS##Parallel computing",
|
|
"CS##Relational databases",
|
|
"CS##Software engineering",
|
|
"CS##Distributed computing",
|
|
"CS##Structured Storage",
|
|
"CS##Symbolic computation",
|
|
"CS##Algorithm design",
|
|
"CS##Computer programming",
|
|
"CS##Data structures",
|
|
"CS##Bioinformatics",
|
|
"ECE##Electricity",
|
|
"ECE##Lorentz force law",
|
|
"ECE##Electrical circuits",
|
|
"ECE##Voltage law",
|
|
"ECE##Digital control",
|
|
"ECE##System identification",
|
|
"ECE##Electrical network",
|
|
"ECE##Microcontroller",
|
|
"ECE##Electrical generator/Analog signal processing",
|
|
"ECE##Electric motor",
|
|
"ECE##Satellite radio",
|
|
"ECE##Control engineering",
|
|
"ECE##Signal-flow graph",
|
|
"ECE##State space representation",
|
|
"ECE##PID controller",
|
|
"ECE##Operational amplifier",
|
|
"Psychology##Prejudice",
|
|
"Psychology##Social cognition",
|
|
"Psychology##Person perception",
|
|
"Psychology##Nonverbal communication",
|
|
"Psychology##Prosocial behavior",
|
|
"Psychology##Leadership",
|
|
"Psychology##Eating disorders",
|
|
"Psychology##Depression",
|
|
"Psychology##Borderline personality disorder",
|
|
"Psychology##Seasonal affective disorder",
|
|
"Medical##Schizophrenia",
|
|
"Psychology##Antisocial personality disorder",
|
|
"Psychology##Media violence",
|
|
"Psychology##Prenatal development",
|
|
"Psychology##Child abuse",
|
|
"Psychology##Gender roles",
|
|
"Psychology##False memories",
|
|
"Psychology##Attention",
|
|
"Psychology##Problem-solving",
|
|
"MAE##computer-aided design",
|
|
"MAE##Hydraulics",
|
|
"MAE##Manufacturing engineering",
|
|
"MAE##Machine design",
|
|
"MAE##Fluid mechanics",
|
|
"MAE##Internal combustion engine",
|
|
"MAE##Thermodynamics",
|
|
"MAE##Materials Engineering",
|
|
"MAE##Strength of materials",
|
|
"Civil##Ambient Intelligence",
|
|
"Civil##Geotextile",
|
|
"Civil##Remote Sensing",
|
|
"Civil##Rainwater Harvesting",
|
|
"Civil##Water Pollution",
|
|
"Civil##Suspension Bridge",
|
|
"Civil##Stealth Technology",
|
|
"Civil##Green Building",
|
|
"Civil##Solar Energy",
|
|
"Civil##Construction Management",
|
|
"Civil##Smart Material",
|
|
"Medical##Addiction",
|
|
"Medical##Allergies",
|
|
"Medical##Alzheimer's Disease",
|
|
"Medical##Ankylosing Spondylitis",
|
|
"Medical##Anxiety",
|
|
"Medical##Asthma",
|
|
"Medical##Atopic Dermatitis",
|
|
"Medical##Atrial Fibrillation",
|
|
"Medical##Autism",
|
|
"Medical##Skin Care",
|
|
"Medical##Bipolar Disorder",
|
|
"Medical##Birth Control",
|
|
"Medical##Children's Health",
|
|
"Medical##Crohn's Disease",
|
|
"Medical##Dementia",
|
|
"Medical##Diabetes",
|
|
"Medical##Weight Loss",
|
|
"Medical##Digestive Health",
|
|
"Medical##Emergency Contraception",
|
|
"Medical##Mental Health",
|
|
"Medical##Fungal Infection",
|
|
"Medical##Headache",
|
|
"Medical##Healthy Sleep",
|
|
"Medical##Heart Disease",
|
|
"Medical##Hepatitis C",
|
|
"Medical##Hereditary Angioedema",
|
|
"Medical##HIV/AIDS",
|
|
"Medical##Hypothyroidism",
|
|
"Medical##Idiopathic Pulmonary Fibrosis",
|
|
"Medical##Irritable Bowel Syndrome",
|
|
"Medical##Kidney Health",
|
|
"Medical##Low Testosterone",
|
|
"Medical##Lymphoma",
|
|
"Medical##Medicare",
|
|
"Medical##Menopause",
|
|
"Medical##Migraine",
|
|
"Medical##Multiple Sclerosis",
|
|
"Medical##Myelofibrosis",
|
|
"Medical##Cancer",
|
|
"Medical##Osteoarthritis",
|
|
"Medical##Osteoporosis",
|
|
"Medical##Overactive Bladder",
|
|
"Medical##Parenting",
|
|
"Medical##Parkinson's Disease",
|
|
"Medical##Polycythemia Vera",
|
|
"Medical##Psoriasis",
|
|
"Medical##Psoriatic Arthritis",
|
|
"Medical##Rheumatoid Arthritis",
|
|
"Medical##Senior Health",
|
|
"Medical##Smoking Cessation",
|
|
"Medical##Sports Injuries",
|
|
"Medical##Sprains and Strains",
|
|
"Medical##Stress Management",
|
|
"biochemistry##Molecular biology",
|
|
"biochemistry##Cell biology",
|
|
"biochemistry##Human Metabolism",
|
|
"biochemistry##Immunology",
|
|
"biochemistry##Genetics",
|
|
"biochemistry##Enzymology",
|
|
"biochemistry##Polymerase chain reaction",
|
|
"biochemistry##Northern blotting",
|
|
"biochemistry##Southern blotting",
|
|
]
|