chore: import upstream snapshot with attribution

docs/api/common/configurable.rst

@@ -0,0 +1,11 @@
+.. _api/configurable:
+
+configurable
+====================
+
+
+.. autoclass:: hanlp_common.configurable.Configurable
+	:members:
+
+.. autoclass:: hanlp_common.configurable.AutoConfigurable
+	:members:

docs/api/common/conll.rst

@@ -0,0 +1,14 @@
+.. _api/conll:
+
+conll
+====================
+
+
+.. autoclass:: hanlp_common.conll.CoNLLWord
+	:members:
+
+.. autoclass:: hanlp_common.conll.CoNLLUWord
+	:members:
+
+.. autoclass:: hanlp_common.conll.CoNLLSentence
+	:members:

docs/api/common/constant.rst

@@ -0,0 +1,6 @@
+constant
+====================
+
+
+.. automodule:: hanlp_common.constant
+	:members:

docs/api/common/document.rst

@@ -0,0 +1,9 @@
+.. _api/document:
+
+document
+====================
+
+.. currentmodule:: hanlp_common
+
+.. autoclass:: hanlp_common.document.Document
+	:members:

docs/api/common/index.md

@@ -0,0 +1,11 @@
+# hanlp_common
+
+Common APIs shared between `hanlp` and `restful`.
+
+```{toctree}
+document
+conll
+configurable
+constant
+```
+

docs/api/hanlp/common/component.rst

@@ -0,0 +1,7 @@
+component
+=================
+
+.. currentmodule:: hanlp.common
+
+.. autoclass:: hanlp.common.component.Component
+	:members:

docs/api/hanlp/common/dataset.md

@@ -0,0 +1,64 @@
+# dataset
+
+This module provides base definition for datasets, dataloaders and samplers.
+
+## datasets
+
+```{eval-rst}
+.. currentmodule:: hanlp.common
+
+.. autoclass:: hanlp.common.dataset.Transformable
+	:members:
+
+.. autoclass:: hanlp.common.dataset.TransformableDataset
+	:members:
+	:special-members:
+	:exclude-members: __init__, __repr__
+```
+
+## dataloaders
+
+```{eval-rst}
+.. currentmodule:: hanlp.common
+
+.. autoclass:: hanlp.common.dataset.PadSequenceDataLoader
+	:members:
+	:special-members:
+	:exclude-members: __init__, __repr__
+
+.. autoclass:: hanlp.common.dataset.PrefetchDataLoader
+	:members:
+	:special-members:
+	:exclude-members: __init__, __repr__
+```
+
+## samplers
+
+```{eval-rst}
+.. currentmodule:: hanlp.common
+
+.. autoclass:: hanlp.common.dataset.BucketSampler
+	:members:
+
+.. autoclass:: hanlp.common.dataset.KMeansSampler
+	:members:
+
+.. autoclass:: hanlp.common.dataset.SortingSampler
+	:members:
+```
+
+## sampler builders
+
+```{eval-rst}
+.. currentmodule:: hanlp.common
+
+.. autoclass:: hanlp.common.dataset.SamplerBuilder
+	:members:
+
+.. autoclass:: hanlp.common.dataset.SortingSamplerBuilder
+	:members:
+
+.. autoclass:: hanlp.common.dataset.KMeansSamplerBuilder
+	:members:
+
+```

docs/api/hanlp/common/index.md

@@ -0,0 +1,13 @@
+# common
+
+Common base classes.
+
+```{toctree}
+structure
+vocab
+transform
+dataset
+component
+torch_component
+```
+

docs/api/hanlp/common/structure.md

@@ -0,0 +1,12 @@
+# structure
+
+```{eval-rst}
+.. currentmodule:: hanlp.common
+
+.. autoclass:: hanlp.common.structure.ConfigTracker
+	:members:
+
+.. autoclass:: hanlp.common.structure.History
+	:members:
+
+```

docs/api/hanlp/common/torch_component.md

@@ -0,0 +1,9 @@
+# torch_component
+
+```{eval-rst}
+.. currentmodule:: hanlp.common.torch_component
+
+.. autoclass:: hanlp.common.torch_component.TorchComponent
+	:members:
+
+```

docs/api/hanlp/common/transform.md

@@ -0,0 +1,9 @@
+# transform
+
+```{eval-rst}
+.. currentmodule:: hanlp.common
+
+.. autoclass:: hanlp.common.transform.VocabDict
+	:members:
+
+```

docs/api/hanlp/common/vocab.md

@@ -0,0 +1,11 @@
+# vocab
+
+```{eval-rst}
+.. currentmodule:: hanlp.common
+
+.. autoclass:: hanlp.common.transform.Vocab
+	:members:
+	:special-members:
+	:exclude-members: __init__, __repr__, __call__, __str__
+
+```

docs/api/hanlp/components/classifiers.md

@@ -0,0 +1,9 @@
+# classifiers
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.classifiers
+
+.. autoclass:: hanlp.components.classifiers.transformer_classifier.TransformerClassifier
+	:members:
+
+```

docs/api/hanlp/components/eos.md

@@ -0,0 +1,9 @@
+# eos
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.eos
+
+.. autoclass:: hanlp.components.eos.ngram.NgramSentenceBoundaryDetector
+	:members:
+
+```

docs/api/hanlp/components/index.md

@@ -0,0 +1,18 @@
+# components
+
+NLP components.
+
+```{toctree}
+mtl/index
+classifiers
+eos
+tokenizers/index
+lemmatizer
+taggers/index
+ner/index
+parsers/index
+srl/index
+pipeline
+sts
+```
+

docs/api/hanlp/components/lemmatizer.md

@@ -0,0 +1,9 @@
+# lemmatizer
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.lemmatizer
+
+.. autoclass:: TransformerLemmatizer
+	:members:
+
+```

docs/api/hanlp/components/mtl/index.md

@@ -0,0 +1,9 @@
+# mtl
+
+Multi-Task Learning (MTL) framework.
+
+```{toctree}
+mtl
+tasks/index
+```
+

docs/api/hanlp/components/mtl/mtl.md

@@ -0,0 +1,11 @@
+# MultiTaskLearning
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.multi_task_learning.MultiTaskLearning
+	:members:
+	:special-members:
+	:exclude-members: __init__, __repr__
+
+```

docs/api/hanlp/components/mtl/tasks/constituency.md

@@ -0,0 +1,12 @@
+# con
+
+Constituency parsing.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.tasks.constituency.CRFConstituencyParsing
+	:members:
+	:exclude-members: execute_training_loop, fit_dataloader
+
+```

docs/api/hanlp/components/mtl/tasks/dep.md

@@ -0,0 +1,12 @@
+# dep
+
+Dependency parsing.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.tasks.dep.BiaffineDependencyParsing
+	:members:
+	:exclude-members: execute_training_loop, fit_dataloader
+
+```

docs/api/hanlp/components/mtl/tasks/index.md

@@ -0,0 +1,17 @@
+# tasks
+
+Multi-Task Learning (MTL) tasks.
+
+```{toctree}
+task
+constituency
+dep
+sdp
+ud
+lem
+pos
+tok
+ner/index
+srl/index
+```
+

docs/api/hanlp/components/mtl/tasks/lem.md

@@ -0,0 +1,12 @@
+# lem
+
+Lemmatization.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.tasks.lem.TransformerLemmatization
+	:members:
+	:exclude-members: execute_training_loop, fit_dataloader
+
+```

docs/api/hanlp/components/mtl/tasks/ner/biaffine_ner.md

@@ -0,0 +1,12 @@
+# biaffine_ner
+
+Biaffine Named Entity Recognition.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.tasks.ner.biaffine_ner.BiaffineNamedEntityRecognition
+	:members:
+	:exclude-members: execute_training_loop, fit_dataloader
+
+```

docs/api/hanlp/components/mtl/tasks/ner/index.md

@@ -0,0 +1,9 @@
+# ner
+
+Named Entity Recognition.
+
+```{toctree}
+tag_ner
+biaffine_ner
+```
+

docs/api/hanlp/components/mtl/tasks/ner/tag_ner.md

@@ -0,0 +1,12 @@
+# tag_ner
+
+Tagging based Named Entity Recognition.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.tasks.ner.tag_ner.TaggingNamedEntityRecognition
+	:members:
+	:exclude-members: execute_training_loop, fit_dataloader
+
+```

docs/api/hanlp/components/mtl/tasks/pos.md

@@ -0,0 +1,12 @@
+# pos
+
+Part-of-speech tagging.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.tasks.pos.TransformerTagging
+	:members:
+	:exclude-members: execute_training_loop, fit_dataloader
+
+```

docs/api/hanlp/components/mtl/tasks/sdp.md

@@ -0,0 +1,12 @@
+# sdp
+
+Semantic Dependency Parsing.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.tasks.sdp.BiaffineSemanticDependencyParsing
+	:members:
+	:exclude-members: execute_training_loop, fit_dataloader
+
+```

docs/api/hanlp/components/mtl/tasks/srl/bio_srl.md

@@ -0,0 +1,12 @@
+# bio_srl
+
+BIO Tagging based Semantic Role Labeling.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.tasks.srl.bio_srl.SpanBIOSemanticRoleLabeling
+	:members:
+	:exclude-members: execute_training_loop, fit_dataloader
+
+```

docs/api/hanlp/components/mtl/tasks/srl/index.md

@@ -0,0 +1,9 @@
+# srl
+
+Semantic Role Labeling.
+
+```{toctree}
+bio_srl
+rank_srl
+```
+

docs/api/hanlp/components/mtl/tasks/srl/rank_srl.md

@@ -0,0 +1,12 @@
+# rank_srl
+
+Span Ranking Semantic Role Labeling.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.tasks.srl.rank_srl.SpanRankingSemanticRoleLabeling
+	:members:
+	:exclude-members: execute_training_loop, fit_dataloader
+
+```

docs/api/hanlp/components/mtl/tasks/task.md

@@ -0,0 +1,10 @@
+# Task
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.tasks.Task
+	:members:
+	:exclude-members: execute_training_loop, fit_dataloader
+
+```

docs/api/hanlp/components/mtl/tasks/tok.md

@@ -0,0 +1,12 @@
+# tok
+
+Tokenization.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.tasks.tok.tag_tok.TaggingTokenization
+	:members:
+	:exclude-members: execute_training_loop, fit_dataloader
+
+```

docs/api/hanlp/components/mtl/tasks/ud.md

@@ -0,0 +1,12 @@
+# ud
+
+Universal Dependencies Parsing (lemmatization, features, PoS tagging and dependency parsing).
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.mtl
+
+.. autoclass:: hanlp.components.mtl.tasks.ud.UniversalDependenciesParsing
+	:members:
+	:exclude-members: execute_training_loop, fit_dataloader
+
+```

docs/api/hanlp/components/ner/biaffine_ner.md

@@ -0,0 +1,11 @@
+# biaffine_ner
+
+Biaffine Named Entity Recognition.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.ner.transformer_ner
+
+.. autoclass:: hanlp.components.ner.biaffine_ner.biaffine_ner.BiaffineNamedEntityRecognizer
+	:members:
+
+```

docs/api/hanlp/components/ner/index.md

@@ -0,0 +1,10 @@
+# ner
+
+Named Entity Recognition.
+
+```{toctree}
+transformer_ner
+rnn_ner
+biaffine_ner
+```
+

docs/api/hanlp/components/ner/rnn_ner.md

@@ -0,0 +1,11 @@
+# rnn_ner
+
+Tagging based Named Entity Recognition.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.ner.rnn_ner
+
+.. autoclass:: hanlp.components.ner.rnn_ner.RNNNamedEntityRecognizer
+	:members:
+
+```

docs/api/hanlp/components/ner/transformer_ner.md

@@ -0,0 +1,11 @@
+# transformer_ner
+
+Tagging based Named Entity Recognition.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.ner.transformer_ner
+
+.. autoclass:: hanlp.components.ner.transformer_ner.TransformerNamedEntityRecognizer
+	:members:
+
+```

docs/api/hanlp/components/parsers/biaffine_dep.md

@@ -0,0 +1,11 @@
+# biaffine_dep
+
+Biaffine dependency parser.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components
+
+.. autoclass:: hanlp.components.parsers.biaffine.biaffine_dep.BiaffineDependencyParser
+	:members:
+
+```

docs/api/hanlp/components/parsers/biaffine_sdp.md

@@ -0,0 +1,11 @@
+# biaffine_sdp
+
+Biaffine dependency parser.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components
+
+.. autoclass:: hanlp.components.parsers.biaffine.biaffine_sdp.BiaffineSemanticDependencyParser
+	:members:
+
+```

docs/api/hanlp/components/parsers/crf_constituency_parser.md

@@ -0,0 +1,11 @@
+# crf_constituency_parser
+
+Biaffine dependency parser.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components
+
+.. autoclass:: hanlp.components.parsers.constituency.crf_constituency_parser.CRFConstituencyParser
+	:members:
+
+```

docs/api/hanlp/components/parsers/index.md

@@ -0,0 +1,11 @@
+# parsers
+
+Parsers.
+
+```{toctree}
+biaffine_dep
+biaffine_sdp
+ud_parser
+crf_constituency_parser
+```
+

docs/api/hanlp/components/parsers/ud_parser.md

@@ -0,0 +1,11 @@
+# ud_parser
+
+Universal Dependencies Parsing (lemmatization, features, PoS tagging and dependency parsing).
+
+```{eval-rst}
+.. currentmodule:: hanlp.components
+
+.. autoclass:: hanlp.components.parsers.ud.ud_parser.UniversalDependenciesParser
+	:members:
+
+```

docs/api/hanlp/components/pipeline.md

@@ -0,0 +1,13 @@
+# pipeline
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.pipeline
+
+.. autoclass:: hanlp.components.pipeline.Pipe
+	:members:
+	
+.. autoclass:: hanlp.components.pipeline.Pipeline
+	:members:
+
+```
+

docs/api/hanlp/components/srl/index.md

@@ -0,0 +1,9 @@
+# srl
+
+Semantic Role Labelers.
+
+```{toctree}
+span_rank
+span_bio
+```
+

docs/api/hanlp/components/srl/span_bio.md

@@ -0,0 +1,11 @@
+# span_bio
+
+Span BIO tagging based SRL.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.srl.span_bio.span_bio
+
+.. autoclass:: SpanBIOSemanticRoleLabeler
+	:members:
+
+```

docs/api/hanlp/components/srl/span_rank.md

@@ -0,0 +1,11 @@
+# span_rank
+
+Span Rank based SRL.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.srl.span_rank.span_rank
+
+.. autoclass:: SpanRankingSemanticRoleLabeler
+	:members:
+
+```

docs/api/hanlp/components/sts.md

@@ -0,0 +1,10 @@
+# sts
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.sts
+
+.. autoclass:: hanlp.components.sts.transformer_sts.TransformerSemanticTextualSimilarity
+	:members:
+
+```
+

docs/api/hanlp/components/taggers/index.md

@@ -0,0 +1,9 @@
+# taggers
+
+Taggers.
+
+```{toctree}
+transformer_tagger
+rnn_tagger
+```
+

docs/api/hanlp/components/taggers/rnn_tagger.md

@@ -0,0 +1,11 @@
+# rnn_tagger
+
+RNN based tagger.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components
+
+.. autoclass:: hanlp.components.taggers.rnn_tagger.RNNTagger
+	:members:
+
+```

docs/api/hanlp/components/taggers/transformer_tagger.md

@@ -0,0 +1,11 @@
+# transformer_tagger
+
+Transformer based tagger.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components
+
+.. autoclass:: hanlp.components.taggers.transformers.transformer_tagger.TransformerTagger
+	:members:
+
+```

docs/api/hanlp/components/tokenizers/index.md

@@ -0,0 +1,9 @@
+# tokenizers
+
+Tokenizers.
+
+```{toctree}
+transformer
+multi_criteria
+```
+

docs/api/hanlp/components/tokenizers/multi_criteria.md

@@ -0,0 +1,11 @@
+# multi_criteria
+
+Transformer based Multi-Criteria Word tokenizer.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.tokenizers.multi_criteria_cws_transformer
+
+.. autoclass:: hanlp.components.tokenizers.multi_criteria_cws_transformer.MultiCriteriaTransformerTaggingTokenizer
+	:members:
+
+```

docs/api/hanlp/components/tokenizers/transformer.md

@@ -0,0 +1,11 @@
+# transformer
+
+Transformer based tokenizer.
+
+```{eval-rst}
+.. currentmodule:: hanlp.components.tokenizers.transformer
+
+.. autoclass:: hanlp.components.tokenizers.transformer.TransformerTaggingTokenizer
+	:members:
+
+```

docs/api/hanlp/datasets/constituency/constituency_dataset.md

@@ -0,0 +1,8 @@
+# constituency_dataset
+
+```{eval-rst}
+
+.. autoclass:: hanlp.datasets.parsing.loaders.constituency_dataset.ConstituencyDataset
+	:members:
+
+```

docs/api/hanlp/datasets/constituency/index.md

@@ -0,0 +1,9 @@
+# con
+
+Constituency parsing datasets.
+
+```{toctree}
+constituency_dataset
+resources
+```
+

docs/api/hanlp/datasets/constituency/resources.md

@@ -0,0 +1,52 @@
+# resources
+
+## Chinese Treebank
+
+
+### CTB8
+
+
+
+````{margin} **Discussion**
+```{seealso}
+About our data split on [our forum](https://bbs.hankcs.com/t/topic/3024).
+```
+````
+
+```{eval-rst}
+
+
+.. autodata:: hanlp.datasets.parsing.ctb8.CTB8_BRACKET_LINE_NOEC_TRAIN
+.. autodata:: hanlp.datasets.parsing.ctb8.CTB8_BRACKET_LINE_NOEC_DEV
+.. autodata:: hanlp.datasets.parsing.ctb8.CTB8_BRACKET_LINE_NOEC_TEST
+
+```
+
+### CTB9
+
+````{margin} **Discussion**
+```{seealso}
+About our data split on [our forum](https://bbs.hankcs.com/t/topic/3024).
+```
+````
+
+```{eval-rst}
+
+
+.. autodata:: hanlp.datasets.parsing.ctb9.CTB9_BRACKET_LINE_NOEC_TRAIN
+.. autodata:: hanlp.datasets.parsing.ctb9.CTB9_BRACKET_LINE_NOEC_DEV
+.. autodata:: hanlp.datasets.parsing.ctb9.CTB9_BRACKET_LINE_NOEC_TEST
+
+```
+
+## English Treebank
+
+### PTB
+
+```{eval-rst}
+
+.. autodata:: hanlp.datasets.parsing.ptb.PTB_TRAIN
+.. autodata:: hanlp.datasets.parsing.ptb.PTB_DEV
+.. autodata:: hanlp.datasets.parsing.ptb.PTB_TEST
+
+```

docs/api/hanlp/datasets/dep/conll_dataset.md

@@ -0,0 +1,10 @@
+# conll
+
+```{eval-rst}
+.. currentmodule:: hanlp.datasets.parsing.loaders.conll_dataset 
+
+
+.. autoclass:: CoNLLParsingDataset
+	:members:
+
+```

docs/api/hanlp/datasets/dep/index.md

@@ -0,0 +1,9 @@
+# dep
+
+Dependency parsing datasets.
+
+```{toctree}
+conll_dataset
+resources
+```
+

docs/api/hanlp/datasets/dep/resources.md

@@ -0,0 +1,120 @@
+# resources
+
+## PKU Multiview Treebank
+
+PKU Multi-view Chinese Treebank, released by PKU-ICL. It contains the sentences from People's Daily(19980101-19980110).
+The number of sentences in it is 14463.
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.parsing.pmt1
+    :members:
+
+```
+
+## Chinese Treebank
+
+### CTB5
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.parsing.ctb5
+    :members:
+
+```
+
+### CTB7
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.parsing.ctb7
+    :members:
+
+```
+
+### CTB8
+
+```{eval-rst}
+
+.. Attention::
+
+    We propose a new data split for CTB which is different from the academia conventions with the following 3 advantages.
+    
+    - Easy to reproduce. Files ending with ``8`` go to dev set, ending with ``9`` go to the test set, otherwise go to the training set.
+    - Full use of CTB8. The academia conventional split omits 50 gold files while we recall them.
+    - More balanced split across genres. Proportions of samples in each genres are similar.
+    
+    We also use Stanford Dependencies 3.3.0 which offers fine-grained relations and more grammars than the conventional
+    head finding rules introduced by :cite:`zhang-clark-2008-tale`.
+    
+    Therefore, scores on our preprocessed CTB8 are not directly comparable to those in most literatures. We have 
+    experimented the same model on the conventionally baked CTB8 and the scores could be 4~5 points higher. 
+    We believe it's worthy since HanLP is made for practical purposes, not just for producing pretty numbers.
+    
+```
+
+````{margin} **Discussion**
+```{seealso}
+We have a discussion on [our forum](https://bbs.hankcs.com/t/topic/3024).
+```
+````
+
+```{eval-rst}
+
+
+.. autodata:: hanlp.datasets.parsing.ctb8.CTB8_SD330_TRAIN
+.. autodata:: hanlp.datasets.parsing.ctb8.CTB8_SD330_DEV
+.. autodata:: hanlp.datasets.parsing.ctb8.CTB8_SD330_TEST
+
+```
+
+### CTB9
+
+```{eval-rst}
+
+.. Attention::
+
+    Similar preprocessing and splits with CTB8 are applied. See the notice above.
+    
+```
+
+```{eval-rst}
+
+
+.. autodata:: hanlp.datasets.parsing.ctb9.CTB9_SD330_TRAIN
+.. autodata:: hanlp.datasets.parsing.ctb9.CTB9_SD330_DEV
+.. autodata:: hanlp.datasets.parsing.ctb9.CTB9_SD330_TEST
+
+```
+
+## English Treebank
+
+### PTB
+
+```{eval-rst}
+
+.. autodata:: hanlp.datasets.parsing.ptb.PTB_SD330_TRAIN
+.. autodata:: hanlp.datasets.parsing.ptb.PTB_SD330_DEV
+.. autodata:: hanlp.datasets.parsing.ptb.PTB_SD330_TEST
+
+```
+
+## Universal Dependencies
+
+### Languages
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.parsing.ud.ud27
+    :members:
+
+```
+
+### Multilingual
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.parsing.ud.ud27m
+    :members:
+
+```

docs/api/hanlp/datasets/eos/eos.md

@@ -0,0 +1,9 @@
+# eos
+
+```{eval-rst}
+.. currentmodule:: hanlp.datasets.eos.eos
+
+.. autoclass:: SentenceBoundaryDetectionDataset
+	:members:
+
+```

docs/api/hanlp/datasets/eos/index.md

@@ -0,0 +1,9 @@
+# eos
+
+Sentence boundary detection datasets.
+
+```{toctree}
+eos
+resources
+```
+

docs/api/hanlp/datasets/eos/resources.md

@@ -0,0 +1,10 @@
+# resources
+
+## nn_eos
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.eos.loaders.nn_eos
+    :members:
+
+```

docs/api/hanlp/datasets/index.md

@@ -0,0 +1,26 @@
+# datasets
+
+```{eval-rst}
+NLP datasets grouped by tasks. For each task, we provide at least one ``torch.utils.data.Dataset`` compatible class
+and several open-source resources. Their file format and description can be found in their ``Dataset.load_file`` 
+documents. Their contents are split into ``TRAIN``, ``DEV`` and ``TEST`` portions, each of them is stored in
+a Python constant which can be fetched using :meth:`~hanlp.utils.io_util.get_resource`.  
+``` 
+
+````{margin} **Professionals use Linux**
+```{note}
+Many preprocessing scripts written by professionals make heavy use of Linux/Unix tool chains like shell, perl, gcc, 
+etc., which is not available or buggy on Windows. You may need a *nix evironment to run these scripts.
+```
+````
+
+```{toctree}
+eos/index
+tok/index
+pos/index
+ner/index
+dep/index
+srl/index
+constituency/index
+```
+

docs/api/hanlp/datasets/ner/index.md

@@ -0,0 +1,10 @@
+# ner
+
+NER datasets.
+
+```{toctree}
+tsv
+json
+resources
+```
+

docs/api/hanlp/datasets/ner/json.md

@@ -0,0 +1,9 @@
+# json
+
+```{eval-rst}
+.. currentmodule:: hanlp.datasets.ner.loaders.json_ner
+
+.. autoclass:: JsonNERDataset
+	:members:
+
+```

docs/api/hanlp/datasets/ner/resources.md

@@ -0,0 +1,50 @@
+# resources
+
+## CoNLL 2003
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.ner.conll03
+    :members:
+
+```
+
+## MSRA
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.ner.msra
+    :members:
+
+```
+
+## OntoNotes5
+
+```{eval-rst}
+
+.. autodata:: hanlp.datasets.srl.ontonotes5.chinese.ONTONOTES5_CONLL12_CHINESE_TRAIN
+.. autodata:: hanlp.datasets.srl.ontonotes5.chinese.ONTONOTES5_CONLL12_CHINESE_DEV
+.. autodata:: hanlp.datasets.srl.ontonotes5.chinese.ONTONOTES5_CONLL12_CHINESE_TEST
+
+.. autodata:: hanlp.datasets.srl.ontonotes5.chinese.ONTONOTES5_CONLL12_NER_CHINESE_TRAIN
+.. autodata:: hanlp.datasets.srl.ontonotes5.chinese.ONTONOTES5_CONLL12_NER_CHINESE_DEV
+.. autodata:: hanlp.datasets.srl.ontonotes5.chinese.ONTONOTES5_CONLL12_NER_CHINESE_TEST
+
+```
+
+## Resume
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.ner.resume
+    :members:
+```
+
+## Weibo
+
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.ner.weibo
+    :members:
+```

docs/api/hanlp/datasets/ner/tsv.md

@@ -0,0 +1,9 @@
+# tsv
+
+```{eval-rst}
+.. currentmodule:: hanlp.datasets.ner.loaders.tsv
+
+.. autoclass:: TSVTaggingDataset
+	:members:
+
+```

docs/api/hanlp/datasets/pos/index.md

@@ -0,0 +1,12 @@
+# pos
+
+PoS datasets. 
+
+```{eval-rst}
+PoS is a normal tagging task which uses :class:`hanlp.datasets.ner.loaders.tsv.TSVTaggingDataset` for loading.
+```
+
+```{toctree}
+resources
+```
+

docs/api/hanlp/datasets/pos/resources.md

@@ -0,0 +1,32 @@
+# resources
+
+## CTB5
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.pos.ctb5
+    :members:
+
+```
+
+## CTB8
+
+```{eval-rst}
+
+.. autodata:: hanlp.datasets.parsing.ctb8.CTB8_POS_TRAIN
+.. autodata:: hanlp.datasets.parsing.ctb8.CTB8_POS_DEV
+.. autodata:: hanlp.datasets.parsing.ctb8.CTB8_POS_TEST
+
+```
+
+## CTB9
+
+
+```{eval-rst}
+
+
+.. autodata:: hanlp.datasets.parsing.ctb9.CTB9_POS_TRAIN
+.. autodata:: hanlp.datasets.parsing.ctb9.CTB9_POS_DEV
+.. autodata:: hanlp.datasets.parsing.ctb9.CTB9_POS_TEST
+
+```

docs/api/hanlp/datasets/srl/conll2012_dataset.md

@@ -0,0 +1,8 @@
+# conll2012_dataset
+
+```{eval-rst}
+
+.. autoclass:: hanlp.datasets.srl.loaders.conll2012.CoNLL2012SRLDataset
+	:members:
+
+```

docs/api/hanlp/datasets/srl/index.md

@@ -0,0 +1,9 @@
+# srl
+
+Semantic Role Labeling datasets.
+
+```{toctree}
+conll2012_dataset
+resources
+```
+

docs/api/hanlp/datasets/srl/resources.md

@@ -0,0 +1,16 @@
+# resources
+
+## OntoNotes 5
+
+### Chinese
+
+```{eval-rst}
+
+.. autodata:: hanlp.datasets.srl.ontonotes5.chinese.ONTONOTES5_CONLL12_CHINESE_TRAIN
+    :noindex:
+.. autodata:: hanlp.datasets.srl.ontonotes5.chinese.ONTONOTES5_CONLL12_CHINESE_DEV
+    :noindex:
+.. autodata:: hanlp.datasets.srl.ontonotes5.chinese.ONTONOTES5_CONLL12_CHINESE_TEST
+    :noindex:
+
+```

docs/api/hanlp/datasets/tok/index.md

@@ -0,0 +1,10 @@
+# tok
+
+Tokenization datasets.
+
+```{toctree}
+txt
+mcws_dataset
+resources
+```
+

docs/api/hanlp/datasets/tok/mcws_dataset.md

@@ -0,0 +1,9 @@
+# mcws_dataset
+
+```{eval-rst}
+.. currentmodule:: hanlp.datasets.tokenization.loaders.multi_criteria_cws.mcws_dataset
+
+.. autoclass:: MultiCriteriaTextTokenizingDataset
+	:members:
+
+```

docs/api/hanlp/datasets/tok/resources.md

@@ -0,0 +1,76 @@
+# resources
+
+## sighan2005
+
+[The Second International Chinese Word Segmentation Bakeoff](http://sighan.cs.uchicago.edu/bakeoff2005/) took place over the summer of 2005.
+
+### pku
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.tokenization.sighan2005.pku
+    :members:
+
+```
+
+### msr
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.tokenization.sighan2005.msr
+    :members:
+
+```
+
+### as
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.tokenization.sighan2005.as_
+    :members:
+
+```
+
+### cityu
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.tokenization.sighan2005.cityu
+    :members:
+
+```
+
+## CTB6
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.tokenization.ctb6
+    :members:
+
+```
+
+## CTB8
+
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.parsing.ctb8
+
+.. autodata:: CTB8_CWS_TRAIN
+.. autodata:: CTB8_CWS_DEV
+.. autodata:: CTB8_CWS_TEST
+
+```
+
+## CTB9
+
+
+```{eval-rst}
+
+.. automodule:: hanlp.datasets.parsing.ctb9
+
+.. autodata:: CTB9_CWS_TRAIN
+.. autodata:: CTB9_CWS_DEV
+.. autodata:: CTB9_CWS_TEST
+
+```

docs/api/hanlp/datasets/tok/txt.md

@@ -0,0 +1,9 @@
+# txt
+
+```{eval-rst}
+.. currentmodule:: hanlp.datasets.tokenization.loaders.txt
+
+.. autoclass:: TextTokenizingDataset
+	:members:
+
+```

docs/api/hanlp/hanlp.rst

@@ -0,0 +1,10 @@
+.. _api/main:
+
+hanlp
+==========
+
+.. currentmodule:: hanlp
+
+.. autofunction:: load
+
+.. autofunction:: pipeline

docs/api/hanlp/index.md

@@ -0,0 +1,13 @@
+# hanlp
+
+Core APIs for `hanlp`.
+
+```{toctree}
+hanlp
+common/index
+components/index
+pretrained/index
+datasets/index
+utils/index
+layers/index
+```

docs/api/hanlp/layers/decoders/biaffine_ner.md

@@ -0,0 +1,9 @@
+# biaffine_ner
+
+
+```{eval-rst}
+
+.. autoclass:: hanlp.components.ner.biaffine_ner.biaffine_ner_model.BiaffineNamedEntityRecognitionDecoder
+	:members:
+
+```

docs/api/hanlp/layers/decoders/index.md

@@ -0,0 +1,7 @@
+# decoders
+
+```{toctree}
+linear_crf
+biaffine_ner
+```
+

docs/api/hanlp/layers/decoders/linear_crf.md

@@ -0,0 +1,9 @@
+# linear_crf
+
+
+```{eval-rst}
+
+.. autoclass:: hanlp.components.mtl.tasks.pos.LinearCRFDecoder
+	:members:
+
+```

docs/api/hanlp/layers/embeddings/char_cnn.md

@@ -0,0 +1,12 @@
+# char_cnn
+
+
+```{eval-rst}
+
+.. autoclass:: hanlp.layers.embeddings.char_cnn.CharCNN
+	:members:
+
+.. autoclass:: hanlp.layers.embeddings.char_cnn.CharCNNEmbedding
+	:members:
+
+```

docs/api/hanlp/layers/embeddings/char_rnn.md

@@ -0,0 +1,12 @@
+# char_rnn
+
+
+```{eval-rst}
+
+.. autoclass:: hanlp.layers.embeddings.char_rnn.CharRNN
+	:members:
+
+.. autoclass:: hanlp.layers.embeddings.char_rnn.CharRNNEmbedding
+	:members:
+
+```

docs/api/hanlp/layers/embeddings/embedding.md

@@ -0,0 +1,15 @@
+# embedding
+
+
+```{eval-rst}
+
+.. autoclass:: hanlp.layers.embeddings.embedding.Embedding
+	:members:
+
+.. autoclass:: hanlp.layers.embeddings.embedding.ConcatModuleList
+	:members:
+
+.. autoclass:: hanlp.layers.embeddings.embedding.EmbeddingList
+	:members:
+
+```

docs/api/hanlp/layers/embeddings/fasttext.md

@@ -0,0 +1,11 @@
+# fasttext
+
+```{eval-rst}
+
+.. autoclass:: hanlp.layers.embeddings.fast_text.FastTextEmbedding
+	:members:
+
+.. autoclass:: hanlp.layers.embeddings.fast_text.FastTextEmbeddingModule
+	:members:
+
+```

docs/api/hanlp/layers/embeddings/index.md

@@ -0,0 +1,11 @@
+# embeddings
+
+```{toctree}
+embedding
+word2vec
+fasttext
+char_cnn
+char_rnn
+transformer
+```
+

docs/api/hanlp/layers/embeddings/transformer.md

@@ -0,0 +1,12 @@
+# transformer
+
+
+```{eval-rst}
+
+.. autoclass:: hanlp.layers.embeddings.contextual_word_embedding.ContextualWordEmbedding
+	:members:
+
+.. autoclass:: hanlp.layers.embeddings.contextual_word_embedding.ContextualWordEmbeddingModule
+	:members:
+
+```

docs/api/hanlp/layers/embeddings/word2vec.md

@@ -0,0 +1,11 @@
+# word2vec
+
+```{eval-rst}
+
+.. autoclass:: hanlp.layers.embeddings.word2vec.Word2VecEmbedding
+	:members:
+
+.. autoclass:: hanlp.layers.embeddings.word2vec.Word2VecEmbeddingModule
+	:members:
+
+```

docs/api/hanlp/layers/index.md

@@ -0,0 +1,8 @@
+# layers
+
+```{toctree}
+embeddings/index
+transformers/index
+decoders/index
+```
+

docs/api/hanlp/layers/transformers/encoder.md

@@ -0,0 +1,9 @@
+# encoder
+
+
+```{eval-rst}
+
+.. autoclass:: hanlp.layers.transformers.encoder.TransformerEncoder
+	:members:
+
+```

docs/api/hanlp/layers/transformers/index.md

@@ -0,0 +1,7 @@
+# transformers
+
+```{toctree}
+encoder
+tokenizer
+```
+

docs/api/hanlp/layers/transformers/tokenizer.md

@@ -0,0 +1,9 @@
+# tokenizer
+
+
+```{eval-rst}
+
+.. autoclass:: hanlp.transform.transformer_tokenizer.TransformerSequenceTokenizer
+	:members:
+
+```

docs/api/hanlp/pretrained/amr.md

@@ -0,0 +1,48 @@
+---
+jupytext:
+  formats: ipynb,md:myst
+  text_representation:
+    extension: .md
+    format_name: myst
+    format_version: '0.8'
+    jupytext_version: 1.4.2
+kernelspec:
+  display_name: Python 3
+  language: python
+  name: python3
+---
+# amr
+
+AMR captures “who is doing what to whom” in a sentence. Each sentence is represented as a rooted, directed, acyclic graph with labels on edges (relations) and leaves (concepts).
+Before loading an AMR model, make sure to install HanLP with the `amr` dependencies:
+
+```shell
+pip install hanlp[amr] -U
+```
+
+To parse a raw sentence into AMR:
+
+```{eval-rst}
+.. margin:: Batching is Faster
+
+    .. Hint:: Parse multiple sentences at once for faster speed! 
+```
+
+
+```{code-cell} ipython3
+:tags: [output_scroll]
+import hanlp
+
+amr_parser = hanlp.load(hanlp.pretrained.amr.AMR3_SEQ2SEQ_BART_LARGE)
+amr = amr_parser('The boy wants the girl to believe him.')
+print(amr)
+```
+
+All the pre-trained parsers and their scores are listed below.
+
+```{eval-rst}
+
+.. automodule:: hanlp.pretrained.amr
+    :members:
+
+```

docs/api/hanlp/pretrained/amr2text.md

@@ -0,0 +1,54 @@
+---
+jupytext:
+  formats: ipynb,md:myst
+  text_representation:
+    extension: .md
+    format_name: myst
+    format_version: '0.8'
+    jupytext_version: 1.4.2
+kernelspec:
+  display_name: Python 3
+  language: python
+  name: python3
+---
+# amr2text
+
+AMR captures “who is doing what to whom” in a sentence. Each sentence is represented as a rooted, directed, acyclic graph with labels on edges (relations) and leaves (concepts).
+The goal of AMR-to-Text Generation is to recover the original sentence realization given an AMR. This task can be seen as the reverse of the structured prediction found in AMR parsing.
+Before loading an AMR model, make sure to install HanLP with the `amr` dependencies:
+
+```shell
+pip install hanlp[amr] -U
+```
+
+To generate a sentence given an AMR:
+
+```{eval-rst}
+.. margin:: Batching is Faster
+
+    .. Hint:: Generate multiple sentences at once for faster speed! 
+```
+
+
+```{code-cell} ipython3
+:tags: [output_scroll]
+import hanlp
+
+generation = hanlp.load(hanlp.pretrained.amr2text.AMR3_GRAPH_PRETRAIN_GENERATION)
+print(generation('''
+(z0 / want-01
+    :ARG0 (z1 / boy)
+    :ARG1 (z2 / believe-01
+              :ARG0 (z3 / girl)
+              :ARG1 z1))
+'''))
+```
+
+All the pre-trained parsers and their scores are listed below.
+
+```{eval-rst}
+
+.. automodule:: hanlp.pretrained.amr2text
+    :members:
+
+```

docs/api/hanlp/pretrained/constituency.md

@@ -0,0 +1,62 @@
+---
+jupytext:
+  formats: ipynb,md:myst
+  text_representation:
+    extension: .md
+    format_name: myst
+    format_version: '0.8'
+    jupytext_version: 1.4.2
+kernelspec:
+  display_name: Python 3
+  language: python
+  name: python3
+---
+
+# constituency
+
+Constituency Parsing is the process of analyzing the sentences by breaking down it into sub-phrases also known as constituents.
+
+To parse a tokenized sentence into constituency tree, first load a parser:
+
+```{eval-rst}
+.. margin:: Batching is Faster
+
+    .. Hint:: To speed up, parse multiple sentences at once, and use a GPU.
+```
+
+```{code-cell} ipython3
+:tags: [output_scroll]
+import hanlp
+
+con = hanlp.load(hanlp.pretrained.constituency.CTB9_CON_FULL_TAG_ELECTRA_SMALL)
+```
+
+Then parse a sequence or multiple sequences of tokens to it. 
+
+```{code-cell} ipython3
+:tags: [output_scroll]
+tree = con(["2021年", "HanLPv2.1", "带来", "最", "先进", "的", "多", "语种", "NLP", "技术", "。"])
+```
+
+The constituency tree is a nested list of constituencies:
+
+```{code-cell} ipython3
+:tags: [output_scroll]
+tree
+```
+
+You can `str` or `print` it to get its bracketed form:
+
+```{code-cell} ipython3
+:tags: [output_scroll]
+print(tree)
+```
+
+All the pre-trained parsers and their scores are listed below.
+
+```{eval-rst}
+
+.. automodule:: hanlp.pretrained.constituency
+    :members:
+
+```

docs/api/hanlp/pretrained/dep.md

@@ -0,0 +1,8 @@
+# dep
+
+```{eval-rst}
+
+.. automodule:: hanlp.pretrained.dep
+    :members:
+
+```

docs/api/hanlp/pretrained/eos.md

@@ -0,0 +1,9 @@
+# eos
+
+
+```{eval-rst}
+
+.. automodule:: hanlp.pretrained.eos
+    :members:
+
+```

docs/api/hanlp/pretrained/fasttext.md

@@ -0,0 +1,8 @@
+# fasttext
+
+```{eval-rst}
+
+.. automodule:: hanlp.pretrained.fasttext
+    :members:
+
+```

docs/api/hanlp/pretrained/glove.md

@@ -0,0 +1,8 @@
+# glove
+
+```{eval-rst}
+
+.. automodule:: hanlp.pretrained.glove
+    :members:
+
+```

docs/api/hanlp/pretrained/index.md

@@ -0,0 +1,27 @@
+# pretrained
+
+```{eval-rst}
+NLP components grouped by tasks. For each task, we provide at least one :class:`~hanlp.common.component.Component` 
+compatible class and several pretrained models. Each of them is stored in a Python constant which can be fetched 
+using :meth:`hanlp.load`.  
+``` 
+
+```{toctree}
+mtl
+eos
+tok
+pos
+ner
+dep
+constituency
+srl
+sdp
+amr
+amr2text
+sts
+word2vec
+glove
+fasttext
+mlm
+```
+